django-streaming 0.6.0__tar.gz → 0.7.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {django_streaming-0.6.0 → django_streaming-0.7.1}/.gitignore +1 -0
- {django_streaming-0.6.0 → django_streaming-0.7.1}/PKG-INFO +1 -1
- {django_streaming-0.6.0 → django_streaming-0.7.1}/src/streaming/__cli__.py +51 -53
- {django_streaming-0.6.0 → django_streaming-0.7.1}/src/streaming/backends/rabbitmq.py +17 -17
- django_streaming-0.7.1/src/streaming/callbacks.py +22 -0
- {django_streaming-0.6.0 → django_streaming-0.7.1}/src/streaming/config.py +1 -0
- {django_streaming-0.6.0 → django_streaming-0.7.1}/src/streaming/event.py +4 -2
- django_streaming-0.7.1/src/streaming/exceptions.py +35 -0
- django_streaming-0.6.0/src/streaming/types.pyi → django_streaming-0.7.1/src/streaming/types.py +1 -1
- django_streaming-0.7.1/src/streaming/utils.py +117 -0
- {django_streaming-0.6.0 → django_streaming-0.7.1}/src/streaming/version.py +2 -2
- django_streaming-0.6.0/src/streaming/exceptions.py +0 -27
- django_streaming-0.6.0/src/streaming/utils.py +0 -50
- {django_streaming-0.6.0 → django_streaming-0.7.1}/LICENSE +0 -0
- {django_streaming-0.6.0 → django_streaming-0.7.1}/README.md +0 -0
- {django_streaming-0.6.0 → django_streaming-0.7.1}/pyproject.toml +0 -0
- {django_streaming-0.6.0 → django_streaming-0.7.1}/src/streaming/__init__.py +0 -0
- {django_streaming-0.6.0 → django_streaming-0.7.1}/src/streaming/apps.py +0 -0
- {django_streaming-0.6.0 → django_streaming-0.7.1}/src/streaming/backends/__init__.py +0 -0
- {django_streaming-0.6.0 → django_streaming-0.7.1}/src/streaming/backends/_base.py +0 -0
- {django_streaming-0.6.0 → django_streaming-0.7.1}/src/streaming/backends/console.py +0 -0
- {django_streaming-0.6.0 → django_streaming-0.7.1}/src/streaming/backends/debug.py +0 -0
- {django_streaming-0.6.0 → django_streaming-0.7.1}/src/streaming/manager.py +0 -0
- {django_streaming-0.6.0 → django_streaming-0.7.1}/src/streaming/py.typed +0 -0
|
@@ -1,33 +1,34 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import logging
|
|
3
|
+
from collections.abc import Iterable
|
|
3
4
|
from typing import TYPE_CHECKING
|
|
4
5
|
|
|
5
6
|
import click
|
|
6
7
|
from click import ClickException
|
|
7
8
|
from colorama import Fore, Style
|
|
8
9
|
from django.core.exceptions import ImproperlyConfigured
|
|
9
|
-
from
|
|
10
|
+
from django.utils.module_loading import import_string
|
|
10
11
|
from pika.exceptions import ChannelClosedByBroker
|
|
11
|
-
from pika.spec import Basic, BasicProperties
|
|
12
12
|
|
|
13
13
|
from .backends import RabbitMQBackend, get_backend
|
|
14
|
-
from .
|
|
15
|
-
from .exceptions import AuthorizationError, StreamingConfigError
|
|
16
|
-
from .utils import make_event
|
|
14
|
+
from .config import CONFIG
|
|
15
|
+
from .exceptions import AuthorizationError, CallbackRetry, StreamingConfigError
|
|
16
|
+
from .utils import LevelFormatter, check_callback, make_event
|
|
17
17
|
|
|
18
18
|
if TYPE_CHECKING:
|
|
19
|
-
from .
|
|
19
|
+
from .event import Event
|
|
20
|
+
|
|
20
21
|
|
|
21
22
|
logger = logging.getLogger(__name__)
|
|
22
|
-
|
|
23
|
+
INFO_LINE = f"{Fore.YELLOW}%-16s: {Style.RESET_ALL}%s"
|
|
23
24
|
|
|
24
25
|
|
|
25
26
|
def _dump_info(backend: "RabbitMQBackend") -> None:
|
|
26
|
-
click.secho(
|
|
27
|
-
click.secho(
|
|
28
|
-
click.secho(
|
|
29
|
-
click.secho(
|
|
30
|
-
click.secho(
|
|
27
|
+
click.secho(INFO_LINE % ("Server", f"{backend.host}:{backend.port}"))
|
|
28
|
+
click.secho(INFO_LINE % ("VirtualHost", f"{backend.virtual_host}"))
|
|
29
|
+
click.secho(INFO_LINE % ("Exchange", f"{backend.exchange}"))
|
|
30
|
+
click.secho(INFO_LINE % ("Timeout", f"{backend.timeout}"))
|
|
31
|
+
click.secho(INFO_LINE % ("Client Name", f"{backend.client_name}"))
|
|
31
32
|
|
|
32
33
|
|
|
33
34
|
def assert_backend() -> "RabbitMQBackend":
|
|
@@ -38,14 +39,17 @@ def assert_backend() -> "RabbitMQBackend":
|
|
|
38
39
|
return backend
|
|
39
40
|
|
|
40
41
|
|
|
41
|
-
def configure_logging(debug: bool, loggers:
|
|
42
|
+
def configure_logging(debug: bool, loggers: Iterable[str] = ("streaming",)) -> None:
|
|
43
|
+
# This should be called only from a click command
|
|
44
|
+
formatter = LevelFormatter()
|
|
45
|
+
ch = logging.StreamHandler()
|
|
46
|
+
ch.setFormatter(formatter)
|
|
42
47
|
for log_name in loggers:
|
|
43
48
|
logr = logging.getLogger(log_name)
|
|
49
|
+
logr.handlers = []
|
|
44
50
|
if debug:
|
|
45
51
|
logr.setLevel(logging.DEBUG)
|
|
46
|
-
logr.addHandler(
|
|
47
|
-
else:
|
|
48
|
-
logr.handlers = []
|
|
52
|
+
logr.addHandler(ch)
|
|
49
53
|
|
|
50
54
|
|
|
51
55
|
@click.group()
|
|
@@ -77,9 +81,9 @@ def configure(queues: bool = False, debug: bool = False) -> None:
|
|
|
77
81
|
routing = backend.configure_queue_routing()
|
|
78
82
|
check.callback() # type: ignore[misc]
|
|
79
83
|
if routing:
|
|
80
|
-
click.secho(
|
|
84
|
+
click.secho(INFO_LINE % ("Queues", ""))
|
|
81
85
|
for k, v in routing.items():
|
|
82
|
-
click.secho(
|
|
86
|
+
click.secho(INFO_LINE % (f" {k}", "; ".join(v)))
|
|
83
87
|
except AuthorizationError as e:
|
|
84
88
|
click.secho(f"Unable to connect using {CONFIG.BROKER_URL}", fg="red", err=True)
|
|
85
89
|
raise ClickException(str(e)) from e
|
|
@@ -105,61 +109,55 @@ def send(routing_key: str, message: str, client_name: str, debug: bool) -> None:
|
|
|
105
109
|
payload = {
|
|
106
110
|
"message": message,
|
|
107
111
|
}
|
|
108
|
-
msg: Event = make_event(payload, key="Test")
|
|
112
|
+
msg: Event = make_event(message=payload, key="Test")
|
|
109
113
|
backend.publish(routing_key, msg)
|
|
110
114
|
click.secho(f"Sent: {msg}")
|
|
111
115
|
backend.disconnect()
|
|
112
116
|
|
|
113
117
|
|
|
114
|
-
|
|
115
|
-
|
|
118
|
+
@cli.command()
|
|
119
|
+
@click.option("-q", "--queues", multiple=True, help="Queue name to listen to")
|
|
120
|
+
@click.option("-cb", "--callback", default=None, help="User callback")
|
|
121
|
+
@click.option("--debug", is_flag=True, help="Debug mode")
|
|
122
|
+
def listen( # noqa PLR0913
|
|
123
|
+
queues: list[str], callback: str | None = None, debug: bool = False, dry_run: bool = False
|
|
124
|
+
) -> None:
|
|
125
|
+
"""Listens for streaming events."""
|
|
126
|
+
configure_logging(debug)
|
|
116
127
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
) -> None:
|
|
120
|
-
message: Event = Event.unmarshal(body)
|
|
121
|
-
click.echo(f"{Fore.GREEN}{message.timestamp} [{queue_name}]{Fore.LIGHTWHITE_EX} [{message.key}] {message.id} ")
|
|
122
|
-
extra: str | JSON
|
|
123
|
-
if payload:
|
|
124
|
-
if pretty:
|
|
125
|
-
extra = json.dumps(message.payload, indent=4)
|
|
126
|
-
else:
|
|
127
|
-
extra = message.payload
|
|
128
|
-
click.echo(f"{Fore.YELLOW}{extra}{Fore.RESET}")
|
|
128
|
+
if not callback:
|
|
129
|
+
callback = CONFIG.LISTEN_CALLBACK
|
|
129
130
|
|
|
130
131
|
try:
|
|
131
|
-
|
|
132
|
+
cb = import_string(callback)
|
|
133
|
+
if not check_callback(cb):
|
|
134
|
+
raise ClickException(f"Callback {callback} is not a valid callback")
|
|
135
|
+
except ImportError:
|
|
136
|
+
raise ClickException(f"Callback {callback} is not a valid callback") from None
|
|
137
|
+
|
|
138
|
+
backend = assert_backend()
|
|
139
|
+
try:
|
|
132
140
|
_dump_info(backend)
|
|
133
|
-
backend.
|
|
141
|
+
backend.connect(True)
|
|
142
|
+
backend.listen(cb, queues=queues, ack=not dry_run)
|
|
134
143
|
except ChannelClosedByBroker as e:
|
|
144
|
+
logger.debug(e)
|
|
135
145
|
click.secho(str(e), fg="red")
|
|
136
146
|
if "no queue" in str(e):
|
|
137
147
|
click.secho("Did you run 'stream configure --queues'", fg="red")
|
|
138
|
-
except
|
|
148
|
+
except CallbackRetry as e:
|
|
149
|
+
logger.debug(e)
|
|
150
|
+
except (StreamingConfigError, ImportError) as e:
|
|
151
|
+
logger.debug(e)
|
|
139
152
|
click.secho(str(e), fg="red")
|
|
140
153
|
click.get_current_context().exit(2)
|
|
141
|
-
except KeyboardInterrupt:
|
|
154
|
+
except KeyboardInterrupt as e:
|
|
155
|
+
logger.debug(e)
|
|
142
156
|
click.secho("Stopping listener.", fg="yellow")
|
|
143
157
|
finally:
|
|
144
158
|
backend.disconnect()
|
|
145
159
|
|
|
146
160
|
|
|
147
|
-
@cli.command()
|
|
148
|
-
@click.option("-q", "--queues", multiple=True, help="Queue name to listen to")
|
|
149
|
-
@click.option("--payload", default=False, is_flag=True, help="Print payload")
|
|
150
|
-
@click.option("--autoreload", "reload", is_flag=True, help="Enable auto-reloading.")
|
|
151
|
-
@click.option("--pretty", is_flag=True, help="Pretty-print payload.")
|
|
152
|
-
def listen(queues: list[str], payload: bool, reload: bool, pretty: bool) -> None:
|
|
153
|
-
"""Listens for streaming events."""
|
|
154
|
-
if reload:
|
|
155
|
-
from django.utils import autoreload
|
|
156
|
-
|
|
157
|
-
click.secho("Starting listener with autoreload...", fg="yellow")
|
|
158
|
-
autoreload.run_with_reloader(_listen, queues=queues, payload=payload, pretty=pretty)
|
|
159
|
-
else:
|
|
160
|
-
_listen(queues=queues, payload=payload, pretty=pretty)
|
|
161
|
-
|
|
162
|
-
|
|
163
161
|
@cli.command()
|
|
164
162
|
def purge() -> None:
|
|
165
163
|
"""Purges all messages from the configured queues."""
|
|
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING
|
|
|
8
8
|
import pika.channel
|
|
9
9
|
import pika.exceptions
|
|
10
10
|
from pika import PlainCredentials
|
|
11
|
-
from pika.exceptions import
|
|
11
|
+
from pika.exceptions import AMQPConnectionError, ConnectionClosed
|
|
12
12
|
from pika.exchange_type import ExchangeType
|
|
13
13
|
|
|
14
14
|
from streaming.config import CONFIG
|
|
@@ -17,8 +17,9 @@ from streaming.utils import exchange_exists
|
|
|
17
17
|
from ..event import Event
|
|
18
18
|
from ..exceptions import (
|
|
19
19
|
AuthorizationError,
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
CallbackError,
|
|
21
|
+
CallbackRetry,
|
|
22
|
+
CallbackSkipAck,
|
|
22
23
|
StreamingConfigError,
|
|
23
24
|
)
|
|
24
25
|
from ._base import BaseBackend
|
|
@@ -35,12 +36,9 @@ MAX_RETRIES = 5
|
|
|
35
36
|
|
|
36
37
|
|
|
37
38
|
class Callback:
|
|
38
|
-
def __init__(
|
|
39
|
-
self, queue_name: str, backend: "RabbitMQBackend", user_callback: "UserCallback", ack: bool = True
|
|
40
|
-
) -> None:
|
|
39
|
+
def __init__(self, queue_name: str, backend: "RabbitMQBackend", user_callback: "UserCallback") -> None:
|
|
41
40
|
self.backend = backend
|
|
42
41
|
self.user_callback: UserCallback = user_callback
|
|
43
|
-
self.ack = ack
|
|
44
42
|
self.queue_name = queue_name
|
|
45
43
|
|
|
46
44
|
def __call__(
|
|
@@ -48,14 +46,14 @@ class Callback:
|
|
|
48
46
|
) -> None:
|
|
49
47
|
try:
|
|
50
48
|
self.user_callback(self.queue_name, ch, method, properties, body)
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
49
|
+
ch.basic_ack(delivery_tag=method.delivery_tag) # type: ignore[arg-type]
|
|
50
|
+
except CallbackSkipAck as e:
|
|
51
|
+
logger.error(f"StreamingCallbackSkipAckException: {e}", exc_info=e)
|
|
52
|
+
except CallbackRetry:
|
|
54
53
|
evt: Event = Event.unmarshal(body)
|
|
55
|
-
logger.debug(f"StreamingCallbackError {evt.id}", exc_info=e)
|
|
56
54
|
retries = int(properties.headers.get("x-retries", 0)) # type: ignore[union-attr]
|
|
57
55
|
self.backend._handle_retry(evt, ch, method, retries)
|
|
58
|
-
except
|
|
56
|
+
except CallbackError as e:
|
|
59
57
|
logger.error(f"Callback failure: {e}", exc_info=e)
|
|
60
58
|
except Exception as e:
|
|
61
59
|
logger.exception(f"Unexpected exception occurred: {e}", exc_info=e)
|
|
@@ -98,10 +96,13 @@ class RabbitMQBackend(BaseBackend):
|
|
|
98
96
|
real_name = self.get_real_queue_name(alias)
|
|
99
97
|
logger.debug(f"Declaring queue '{real_name}'")
|
|
100
98
|
self.channel.queue_declare(queue=real_name, durable=True, arguments=None) # type: ignore[union-attr]
|
|
101
|
-
if binding_keys := config.get("
|
|
99
|
+
if binding_keys := config.get("binding_keys", []):
|
|
100
|
+
options = config.get("options", {})
|
|
102
101
|
for binding_key in binding_keys:
|
|
103
102
|
logger.debug("Listening on queue '%s' routed by '%s'", alias, binding_key)
|
|
104
|
-
self.channel.queue_bind(
|
|
103
|
+
self.channel.queue_bind( # type: ignore[union-attr]
|
|
104
|
+
exchange=self.exchange, queue=real_name, routing_key=binding_key, arguments=options
|
|
105
|
+
)
|
|
105
106
|
ret[alias].append(binding_key)
|
|
106
107
|
else:
|
|
107
108
|
ret[alias].append("warning no routing defined")
|
|
@@ -152,7 +153,7 @@ class RabbitMQBackend(BaseBackend):
|
|
|
152
153
|
def disconnect(self) -> None:
|
|
153
154
|
try:
|
|
154
155
|
if self._connection:
|
|
155
|
-
with suppress(
|
|
156
|
+
with suppress(ConnectionClosed, AttributeError, AMQPConnectionError):
|
|
156
157
|
logger.debug("Closing RabbitMQ connection.")
|
|
157
158
|
self._connection.close()
|
|
158
159
|
finally:
|
|
@@ -217,7 +218,6 @@ class RabbitMQBackend(BaseBackend):
|
|
|
217
218
|
if self.channel is None:
|
|
218
219
|
self.connect()
|
|
219
220
|
self.check_channel()
|
|
220
|
-
|
|
221
221
|
configured_queues = CONFIG.QUEUES
|
|
222
222
|
if not configured_queues:
|
|
223
223
|
logger.warning("No queues configured in settings.STREAMING['QUEUES']")
|
|
@@ -225,7 +225,7 @@ class RabbitMQBackend(BaseBackend):
|
|
|
225
225
|
queues_to_listen = queues or configured_queues.keys()
|
|
226
226
|
for queue_name in queues_to_listen:
|
|
227
227
|
real_queue_name = self.get_real_queue_name(queue_name)
|
|
228
|
-
_callback = Callback(queue_name, self, callback
|
|
228
|
+
_callback = Callback(queue_name, self, callback)
|
|
229
229
|
self.channel.basic_consume(queue=real_queue_name, on_message_callback=_callback, auto_ack=False) # type: ignore[union-attr]
|
|
230
230
|
|
|
231
231
|
logger.info("Waiting for messages. To exit press CTRL+C")
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
from colorama import Fore
|
|
5
|
+
from pika.adapters.blocking_connection import BlockingChannel
|
|
6
|
+
from pika.spec import Basic, BasicProperties
|
|
7
|
+
|
|
8
|
+
from streaming.event import Event
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def default_callback(
|
|
14
|
+
queue_name: str, ch: BlockingChannel, method: Basic.Deliver, properties: BasicProperties, body: bytes
|
|
15
|
+
) -> bool:
|
|
16
|
+
logger.debug("Invoking default callback")
|
|
17
|
+
message: Event = Event.unmarshal(body)
|
|
18
|
+
ack = f"{Fore.GREEN}ACK{Fore.RESET}"
|
|
19
|
+
sys.stdout.write(
|
|
20
|
+
f"{Fore.GREEN}{message.timestamp} [{queue_name}]{Fore.LIGHTWHITE_EX} [{message.key}] {message.id} {ack}\n"
|
|
21
|
+
)
|
|
22
|
+
return True
|
|
@@ -47,12 +47,14 @@ class Event:
|
|
|
47
47
|
)
|
|
48
48
|
|
|
49
49
|
@classmethod
|
|
50
|
-
def build(
|
|
50
|
+
def build(
|
|
51
|
+
cls, data: Any, value_type: "EventType" = "absolute", key: str = "N/A", message_id: str | None = None
|
|
52
|
+
) -> "Event":
|
|
51
53
|
if isinstance(data, str):
|
|
52
54
|
payload: JSON = {"message": data}
|
|
53
55
|
else:
|
|
54
56
|
payload = data
|
|
55
|
-
return cls(key=key, payload=payload, value_type=value_type)
|
|
57
|
+
return cls(key=key, payload=payload, value_type=value_type, message_id=message_id)
|
|
56
58
|
|
|
57
59
|
def as_dict(self) -> dict[str, Any]:
|
|
58
60
|
return {
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
logger = logging.getLogger(__name__)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class StreamingError(Exception):
|
|
7
|
+
pass
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class StreamingException(Exception): # noqa N818
|
|
11
|
+
pass
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class StreamingConfigError(StreamingError):
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class AuthorizationError(StreamingConfigError):
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class StreamingBackendError(StreamingError):
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class CallbackSkipAck(StreamingException): # noqa: N818
|
|
27
|
+
"""Consume exception but do not acknowledge the message."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class CallbackRetry(StreamingException): # noqa: N818
|
|
31
|
+
"""Message will be re-queued."""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class CallbackError(StreamingError): # noqa: N818
|
|
35
|
+
"""Message will be discarded."""
|
django_streaming-0.6.0/src/streaming/types.pyi → django_streaming-0.7.1/src/streaming/types.py
RENAMED
|
@@ -10,4 +10,4 @@ JSON = dict[str, JsonT]
|
|
|
10
10
|
|
|
11
11
|
EventType = Literal["absolute", "delta", "event"]
|
|
12
12
|
PikaCallback = Callable[[BlockingChannel, Basic.Deliver, BasicProperties, bytes], None]
|
|
13
|
-
UserCallback = Callable[[str, BlockingChannel, Basic.Deliver, BasicProperties, bytes],
|
|
13
|
+
UserCallback = Callable[[str, BlockingChannel, Basic.Deliver, BasicProperties, bytes], bool]
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import socket
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from inspect import signature
|
|
5
|
+
from typing import TYPE_CHECKING, Any, get_type_hints
|
|
6
|
+
|
|
7
|
+
from colorama import Fore
|
|
8
|
+
from django.core.serializers.json import DjangoJSONEncoder
|
|
9
|
+
from django.db import models
|
|
10
|
+
from pika.adapters.blocking_connection import BlockingChannel
|
|
11
|
+
from pika.exceptions import ChannelClosedByBroker
|
|
12
|
+
|
|
13
|
+
from streaming.types import JSON, EventType, UserCallback
|
|
14
|
+
|
|
15
|
+
if TYPE_CHECKING:
|
|
16
|
+
from streaming.event import Event
|
|
17
|
+
|
|
18
|
+
MINUTE = 60
|
|
19
|
+
HOUR = MINUTE * 60
|
|
20
|
+
DAY = HOUR * 24
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class LevelFormatter(logging.Formatter):
|
|
24
|
+
template = f"$color$%(levelname)s{Fore.RESET}: {Fore.LIGHTWHITE_EX}%(name)s{Fore.RESET} - %(message)s"
|
|
25
|
+
default = Fore.LIGHTWHITE_EX
|
|
26
|
+
colors = {
|
|
27
|
+
logging.DEBUG: Fore.MAGENTA,
|
|
28
|
+
logging.CRITICAL: Fore.RED,
|
|
29
|
+
logging.ERROR: Fore.RED,
|
|
30
|
+
logging.INFO: Fore.BLUE,
|
|
31
|
+
logging.WARN: Fore.YELLOW,
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
def format(self, record: logging.LogRecord) -> str:
|
|
35
|
+
color = self.colors.get(record.levelno, self.default)
|
|
36
|
+
log_fmt = self.template.replace("$color$", color)
|
|
37
|
+
formatter = logging.Formatter(log_fmt)
|
|
38
|
+
return formatter.format(record)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class StreamingJSONEncoder(DjangoJSONEncoder):
|
|
42
|
+
def default(self, o: Any) -> Any:
|
|
43
|
+
if isinstance(o, models.Model):
|
|
44
|
+
return str(o)
|
|
45
|
+
return super().default(o)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def parse_bool(value: Any) -> bool:
|
|
49
|
+
if isinstance(value, str):
|
|
50
|
+
return value.lower() in ("yes", "true", "1", "y", "t")
|
|
51
|
+
return value in [1, True]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def make_event(
|
|
55
|
+
message: "str|JSON", *, value_type: "EventType" = "absolute", key: str = "N/A", message_id: str | None = None
|
|
56
|
+
) -> "Event":
|
|
57
|
+
from streaming.event import Event
|
|
58
|
+
|
|
59
|
+
return Event.build(
|
|
60
|
+
key=key,
|
|
61
|
+
data=message,
|
|
62
|
+
value_type=value_type,
|
|
63
|
+
message_id=message_id,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def get_local_ip() -> str:
|
|
68
|
+
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
69
|
+
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
|
|
70
|
+
s.connect(("<broadcast>", 12345)) # 12345 is random port. 0 fails on Mac.
|
|
71
|
+
return str(s.getsockname()[0])
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def exchange_exists(channel: BlockingChannel | None, exchange_name: str) -> bool:
|
|
75
|
+
try:
|
|
76
|
+
channel.exchange_declare(exchange=exchange_name, passive=True) # type: ignore[union-attr]
|
|
77
|
+
return True
|
|
78
|
+
except (ChannelClosedByBroker, AttributeError):
|
|
79
|
+
return False
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def check_callback(
|
|
83
|
+
func: Callable[
|
|
84
|
+
[
|
|
85
|
+
Any,
|
|
86
|
+
],
|
|
87
|
+
Any,
|
|
88
|
+
],
|
|
89
|
+
) -> bool:
|
|
90
|
+
expected = UserCallback
|
|
91
|
+
try:
|
|
92
|
+
sig = signature(func)
|
|
93
|
+
params = list(sig.parameters.values())
|
|
94
|
+
hints = get_type_hints(func)
|
|
95
|
+
|
|
96
|
+
# For collections.abc.Callable, __args__ is ([arg1, arg2, ...], return_type)
|
|
97
|
+
expected_arg_types = expected.__args__[:-1] # type: ignore[attr-defined]
|
|
98
|
+
expected_return_type = expected.__args__[-1] # type: ignore[attr-defined]
|
|
99
|
+
|
|
100
|
+
if len(params) != len(expected_arg_types):
|
|
101
|
+
return False
|
|
102
|
+
return_hint = hints.get("return", type(None))
|
|
103
|
+
if expected_return_type is None:
|
|
104
|
+
expected_return_type = type(None)
|
|
105
|
+
|
|
106
|
+
for i, param in enumerate(params):
|
|
107
|
+
if hints.get(param.name) != expected_arg_types[i]:
|
|
108
|
+
return False
|
|
109
|
+
|
|
110
|
+
return bool(return_hint == expected_return_type)
|
|
111
|
+
|
|
112
|
+
except (TypeError, NameError, AttributeError, IndexError):
|
|
113
|
+
# TypeError -> not a callable or invalid annotations
|
|
114
|
+
# NameError -> forward refs not resolvable
|
|
115
|
+
# AttributeError -> in case expected is malformed
|
|
116
|
+
# IndexError -> if __args__ is not what we expect
|
|
117
|
+
return False
|
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '0.
|
|
32
|
-
__version_tuple__ = version_tuple = (0,
|
|
31
|
+
__version__ = version = '0.7.1'
|
|
32
|
+
__version_tuple__ = version_tuple = (0, 7, 1)
|
|
33
33
|
|
|
34
34
|
__commit_id__ = commit_id = None
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
|
|
3
|
-
logger = logging.getLogger(__name__)
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
class StreamingError(Exception):
|
|
7
|
-
pass
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
class StreamingConfigError(StreamingError):
|
|
11
|
-
pass
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
class AuthorizationError(StreamingConfigError):
|
|
15
|
-
pass
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
class StreamingBackendError(StreamingError):
|
|
19
|
-
pass
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
class StreamingCallbackRetryError(StreamingError):
|
|
23
|
-
"""Generic exception raised when a callback fails. Message will be re-queued."""
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
class StreamingCallbackFailure(StreamingError): # noqa: N818
|
|
27
|
-
"""Generic exception raised when a callback fails. Message will be discarded."""
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
import socket
|
|
2
|
-
from typing import TYPE_CHECKING, Any
|
|
3
|
-
|
|
4
|
-
from django.core.serializers.json import DjangoJSONEncoder
|
|
5
|
-
from django.db import models
|
|
6
|
-
from pika.adapters.blocking_connection import BlockingChannel
|
|
7
|
-
from pika.exceptions import ChannelClosedByBroker
|
|
8
|
-
|
|
9
|
-
if TYPE_CHECKING:
|
|
10
|
-
from streaming.event import Event
|
|
11
|
-
from streaming.types import JSON
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
MINUTE = 60
|
|
15
|
-
HOUR = MINUTE * 60
|
|
16
|
-
DAY = HOUR * 24
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
class StreamingJSONEncoder(DjangoJSONEncoder):
|
|
20
|
-
def default(self, o: Any) -> Any:
|
|
21
|
-
if isinstance(o, models.Model):
|
|
22
|
-
return str(o)
|
|
23
|
-
return super().default(o)
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
def parse_bool(value: Any) -> bool:
|
|
27
|
-
if isinstance(value, str):
|
|
28
|
-
return value.lower() in ("yes", "true", "1", "y", "t")
|
|
29
|
-
return value in [1, True]
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
def make_event(message: "str | JSON", *, key: str = "") -> "Event":
|
|
33
|
-
from streaming.event import Event
|
|
34
|
-
|
|
35
|
-
return Event.build(key=key, data=message, value_type="absolute")
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
def get_local_ip() -> str:
|
|
39
|
-
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
40
|
-
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
|
|
41
|
-
s.connect(("<broadcast>", 12345)) # 12345 is random port. 0 fails on Mac.
|
|
42
|
-
return str(s.getsockname()[0])
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
def exchange_exists(channel: BlockingChannel | None, exchange_name: str) -> bool:
|
|
46
|
-
try:
|
|
47
|
-
channel.exchange_declare(exchange=exchange_name, passive=True) # type: ignore[union-attr]
|
|
48
|
-
return True
|
|
49
|
-
except (ChannelClosedByBroker, AttributeError):
|
|
50
|
-
return False
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|