django-nativemojo 0.1.10__py3-none-any.whl → 0.1.16__py3-none-any.whl

mojo/apps/tasks/runner.py

@@ -1,174 +0,0 @@
-from importlib import import_module
-from concurrent.futures import ThreadPoolExecutor
-from .manager import TaskManager
-from mojo.tasks import manager
-import os
-from mojo.helpers import logit
-from mojo.helpers import daemon
-from mojo.helpers import paths
-import time
-
-
-class TaskEngine(daemon.Daemon):
-    """
-    The TaskEngine is responsible for managing and executing tasks across different channels.
-    It leverages a thread pool to execute tasks concurrently and uses a task manager to maintain task states.
-    """
-    def __init__(self, channels=["broadcast"], max_workers=5):
-        """
-        Initialize the TaskEngine.
-
-        Args:
-            channels (list): A list of channel names where tasks are queued.
-            max_workers (int, optional): The maximum number of threads available for task execution. Defaults to 5.
-        """
-        super().__init__("taskit", os.path.join(paths.VAR_ROOT, "taskit"))
-        self.manager = manager.TaskManager(channels)
-        self.channels = channels
-        if "broadcast" not in self.channels:
-            self.channels.append("broadcast")
-        self.max_workers = max_workers
-        self.executor = None
-        self.logger = logit.get_logger("taskit", "taskit.log")
-
-    def reset_running_tasks(self):
-        """
-        Reset tasks that are stuck in a running state by moving them back to the pending state.
-        """
-        for channel in self.channels:
-            for task_id in self.manager.get_running_ids(channel):
-                self.logger.info(f"moving task {task_id} from running to pending")
-                self.manager.remove_from_running(channel, task_id)
-                self.manager.add_to_pending(channel, task_id)
-
-    def queue_pending_tasks(self):
-        """
-        Queue all the pending tasks for execution.
-        """
-        for channel in self.channels:
-            for task_id in self.manager.get_pending_ids(channel):
-                self.queue_task(task_id)
-
-    def handle_message(self, message):
-        """
-        Handle incoming messages from the channels, decoding task identifiers and queuing them for execution.
-
-        Args:
-            message (dict): A dictionary with message data containing task information.
-        """
-        self.queue_task(message['data'].decode())
-
-    def on_run_task(self, task_id):
-        """
-        Execute a task based on its identifier by locating the relevant function and executing it.
-
-        Args:
-            task_id (str): The identifier of the task to be executed.
-        """
-        # this is a keep it thread safe with the redis connection
-        tman = TaskManager([])
-        task_data = tman.get_task(task_id)
-        if not task_data:
-            # this task has expired or no longer exists
-            self.logger.info(f"Task {task_id} has expired or no longer exists")
-            tman.remove_from_pending(task_id)
-            return
-        self.logger.info(f"Executing task {task_id}")
-        function_path = task_data.get('function')
-        module_name, func_name = function_path.rsplit('.', 1)
-        module = import_module(module_name)
-        func = getattr(module, func_name)
-        self.manager.remove_from_pending(task_id, task_data.channel)
-        self.manager.add_to_running(task_id, task_data.channel)
-
-        try:
-            task_data.started_at = time.time()
-            func(task_data)
-            task_data.completed_at = time.time()
-            task_data.elapsed_time = task_data.completed_at - task_data.started_at
-            tman.save_task(task_data)
-            tman.add_to_completed(task_data)
-            self.logger.info(f"Task {task_id} completed after {task_data.elapsed_time} seconds")
-        except Exception as e:
-            self.logger.error(f"Error executing task {task_id}: {str(e)}")
-            tman.add_to_errors(task_data, str(e))
-        finally:
-            tman.remove_from_running(task_id, task_data.channel)
-
-    def queue_task(self, task_id):
-        """
-        Submit a task for execution in the thread pool.
-
-        Args:
-            task_id (str): The identifier of the task to be queued.
-        """
-        self.logger.info(f"adding task {task_id}")
-        self.executor.submit(self.on_run_task, task_id)
-
-    def wait_for_all_tasks_to_complete(self, timeout=30):
-        """
-        Wait for all tasks submitted to the executor to complete.
-        """
-        self.executor.shutdown(wait=True, timeout=timeout)
-        # Check if there are still active threads
-        active_threads = [thread for thread in self.executor._threads if thread.is_alive()]
-        if active_threads:
-            self.logger.warning(f"shutdown issue, {len(active_threads)} tasks exceeded timeout")
-            self.executor.shutdown(wait=False)  # Stop accepting new tasks
-
-    def start_listening(self):
-        """
-        Listen for messages on the subscribed channels and handle them as they arrive.
-        """
-        self.logger.info("starting with channels...", self.channels)
-        self.reset_running_tasks()
-        self.queue_pending_tasks()
-        pubsub = self.manager.redis.pubsub()
-        channel_keys = {self.manager.get_channel_key(channel): self.handle_message for channel in self.channels}
-        pubsub.subscribe(**channel_keys)
-        for message in pubsub.listen():
-            if not self.running:
-                self.logger.info("shutting down, waiting for tasks to complete")
-                self.wait_for_all_tasks_to_complete()
-                self.logger.info("shutdown complete")
-                return
-            if message['type'] != 'message':
-                continue
-            self.handle_message(message)
-
-    def run(self):
-        self.executor = ThreadPoolExecutor(max_workers=self.max_workers)
-        self.start_listening()
-
-
-# HELPERS FOR RUNNING VIA CLI
-def get_args():
-    """
-    Setup the argument parser for command-line interface.
-
-    Returns:
-        Namespace: Parsed command-line arguments.
-    """
-    import argparse
-    parser = argparse.ArgumentParser(description="TaskEngine Background Service")
-    parser.add_argument("--start", action="store_true", help="Start the daemon")
-    parser.add_argument("--stop", action="store_true", help="Stop the daemon")
-    parser.add_argument("--foreground", "-f", action="store_true", help="Run in foreground mode")
-    parser.add_argument("-v", "--verbose", action="store_true",
-                        help="Enable verbose logging")
-    return parser, parser.parse_args()
-
-
-def main():
-    from mojo.helpers.settings import settings
-    parser, args = get_args()
-    daemon = TaskEngine(settings.TASKIT_CHANNELS)
-    if args.start:
-        daemon.start()
-    elif args.stop:
-        daemon.stop()
-    elif args.foreground:
-        print("Running in foreground mode...")
-        daemon.run()
-    else:
-        parser.print_help()

mojo/apps/tasks/tq_handlers.py

@@ -1,14 +0,0 @@
-from mojo.helpers import logit
-import time
-
-logger = logit.get_logger("ti_example", "ti_example.log")
-
-def run_example_task(task):
-    logger.info("Running example task with data", task)
-    time.sleep(task.data.get("duration", 5))
-
-
-def run_error_task(task):
-    logger.info("Running error task with data", task)
-    time.sleep(2)
-    raise Exception("Example error")

mojo/helpers/redis.py

@@ -1,10 +0,0 @@
-from redis import ConnectionPool, StrictRedis
-from mojo.helpers.settings import settings
-REDIS_POOL = None
-
-
-def get_connection():
-    global REDIS_POOL
-    if REDIS_POOL is None:
-        REDIS_POOL = ConnectionPool(**settings.REDIS_DB)
-    return StrictRedis(connection_pool=REDIS_POOL)

mojo/models/meta.py

@@ -1,262 +0,0 @@
-from objict import objict
-from django.db import models as dm
-import string
-from rest.encryption import ENCRYPTER, DECRYPTER
-from datetime import datetime, date
-
-class MetaDataBase(dm.Model):
-    class Meta:
-        abstract = True
-
-    category = dm.CharField(db_index=True, max_length=32, default=None, null=True, blank=True)
-    key = dm.CharField(db_index=True, max_length=80)
-    value_format = dm.CharField(max_length=16)
-    value = dm.TextField()
-    int_value = dm.IntegerField(default=None, null=True, blank=True)
-    float_value = dm.FloatField(default=None, null=True, blank=True)
-
-    def set_value(self, value):
-        self.value = str(value)
-        value_type = type(value)
-
-        if value_type is int or self.value in ["0", "1"]:
-            if value_type is int and value > 2147483647:
-                self.value_format = "S"
-                return
-            self.value_format = "I"
-            self.int_value = value
-        elif value_type is float:
-            self.value_format = "F"
-            self.float_value = value
-        elif isinstance(value, list):
-            self.value_format = "L"
-        elif isinstance(value, dict):
-            self.value_format = "O"
-        elif isinstance(value, str) and len(value) < 9 and value.isdigit():
-            self.value_format = "I"
-            self.int_value = int(value)
-        elif value in ["True", "true", "False", "false"]:
-            self.value_format = "I"
-            self.int_value = 1 if value.lower() == "true" else 0
-        elif isinstance(value, bool):
-            self.value_format = "I"
-            self.int_value = 1 if value else 0
-        else:
-            self.value_format = "S"
-
-    def get_strict_type(self, field_type):
-        try:
-            return field_type(self.value)
-        except (ValueError, TypeError):
-            if field_type is bool:
-                return self.int_value != 0 if self.value_format == 'I' else self.value.lower() in ['true', '1', 'y', 'yes']
-            elif field_type in [date, datetime]:
-                return rh.parseDate(self.value)
-            return self.value
-
-    def get_value(self, field_type=None):
-        if field_type:
-            return self.get_strict_type(field_type)
-        if self.value_format == 'I':
-            return self.int_value
-        elif self.value_format == 'F':
-            return self.float_value
-        elif self.value_format in ["L", "O"] and self.value:
-            try:
-                return eval(self.value)
-            except Exception:
-                pass
-        return self.value
-
-    def __str__(self):
-        return f"{self.category}.{self.key}={self.value}" if self.category else f"{self.key}={self.value}"
-
-class MetaDataModel:
-    def set_metadata(self, request, values=None):
-        if not self.id:
-            self.save()
-
-        values = values or request
-        if isinstance(values, list):
-            values = objict({k: v for item in values if isinstance(item, dict) for k, v in item.items()})
-
-        if not isinstance(values, dict):
-            raise ValueError(f"invalid metadata: {values}")
-
-        for key, value in values.items():
-            cat, key = key.split('.', 1) if '.' in key else (None, key)
-            self.set_property(key, value, cat, request=request)
-
-    def metadata(self):
-        return self.get_properties()
-
-    def remove_properties(self, category=None):
-        self.properties.filter(category=category).delete()
-
-    def get_properties(self, category=None):
-        result = {}
-        for prop in self.properties.all():
-            category_ = prop.category
-            key = prop.key
-
-            if not category_:
-                self._add_property_to_result(result, prop)
-                continue
-
-            props = self.get_field_props(category_)
-            if props.hidden:
-                continue
-
-            if category_ not in result:
-                result[category_] = {}
-
-            if category_ == "secrets":
-                masked_value = "*" * prop.int_value if prop.int_value else "******"
-                result[category_][key] = masked_value
-            else:
-                self._add_property_to_result(result[category_], prop)
-
-        return result.get(category, {}) if category else result
-
-    def _add_property_to_result(self, result_dict, prop):
-        props = self.get_field_props(prop.key)
-        if not props.hidden:
-            result_dict[prop.key] = prop.get_value()
-
-    def get_field_props(self, key):
-        self._init_field_props()
-        category, key = key.split('.', 1) if '.' in key else (None, key)
-        props = objict()
-
-        if self.__field_props:
-            cat_props = self.__field_props.get(category, {})
-            self._update_props_with_category(props, cat_props)
-
-            field_props = self.__field_props.get(key, {})
-            self._update_props_with_field(props, field_props)
-
-        return props
-
-    def _update_props_with_category(self, props, cat_props):
-        if cat_props:
-            props.notify = cat_props.get("notify")
-            props.requires = cat_props.get("requires")
-            props.hidden = cat_props.get("hidden", False)
-            on_change_name = cat_props.get("on_change")
-            if on_change_name:
-                props.on_change = getattr(self, on_change_name, None)
-
-    def _update_props_with_field(self, props, field_props):
-        props.notify = field_props.get("notify", props.notify)
-        props.requires = field_props.get("requires", props.requires)
-        props.hidden = field_props.get("hidden", props.hidden)
-        on_change_name = field_props.get("on_change")
-        if on_change_name:
-            props.on_change = getattr(self, on_change_name, None)
-
-    def check_field_perms(self, full_key, props, request=None):
-        if not props.requires:
-            return True
-        if not request or not request.member:
-            return False
-        if request.member.hasPermission(props.requires) or request.user.is_superuser:
-            return True
-
-        if props.notify and request.member:
-            subject = f"permission denied changing protected '{full_key}' field"
-            msg = f"permission denied changing protected field '{full_key}'\nby user: {request.user.username}\nfor: {self}"
-            request.member.notifyWithPermission(props.notify, subject, msg, email_only=True)
-        raise re.PermissionDeniedException(subject, 481)
-
-    def set_properties(self, data, category=None, request=None, using=None):
-        for k, v in data.items():
-            self.set_property(k, v, category, request=request, using=using)
-
-    def set_property(self, key, value, category=None, request=None, using=None, ascii_only=False, encrypted=False):
-        if ascii_only and isinstance(value, str):
-            value = ''.join(filter(lambda x: x in string.printable, value))
-
-        if using is None:
-            using = getattr(self.RestMeta, "DATABASE", None)
-
-        if request is None:
-            request = rh.getActiveRequest()
-
-        self._init_field_props()
-
-        if '.' in key:
-            category, key = key.split('.', 1)
-
-        full_key = f"{category}.{key}" if category else key
-        field_props = self.get_field_props(full_key)
-
-        if not self.check_field_perms(full_key, field_props, request):
-            return False
-
-        prop = self.properties.filter(category=category, key=key).last()
-        if not prop and (value is None or value == ""):
-            return False
-
-        has_changed, old_value = self._update_or_create_property(prop, category, key, value, encrypted, using)
-
-        if has_changed and field_props.on_change:
-            field_props.on_change(key, value, old_value, category)
-
-        self._notify_change_if_required(field_props, full_key, value, request)
-
-        if hasattr(self, "_recordRestChange"):
-            self._recordRestChange(f"metadata.{full_key}", old_value)
-
-        return has_changed
-
-    def _update_or_create_property(self, prop, category, key, value, encrypted, using):
-        has_changed = False
-        old_value = None
-
-        value_len = len(value) if encrypted else 0
-        if encrypted:
-            value = ENCRYPTER.encrypt(value)
-
-        if prop:
-            old_value = prop.get_value()
-            if value is None or value == "":
-                self.properties.filter(category=category, key=key).delete()
-                has_changed = True
-            else:
-                has_changed = str(value) != prop.value
-                if has_changed:
-                    prop.set_value(value)
-                    if encrypted:
-                        prop.int_value = value_len
-                    prop.save(using=using)
-        else:
-            has_changed = True
-            PropClass = self.get_fk_model("properties")
-            prop = PropClass(parent=self, key=key, category=category)
-            prop.set_value(value)
-            prop.save(using=using)
-
-        return has_changed, old_value
-
-    def _notify_change_if_required(self, field_props, full_key, value, request):
-        if field_props.notify and request and request.member:
-            username = request.member.username if request and request.member else "root"
-            truncated_value = "***" if value and len(str(value)) > 5 else value
-            msg = (f"protected field '{full_key}' changed to '{truncated_value}'\n"
-                   f"by user: {username}\nfor: {self}")
-            request.member.notifyWithPermission(field_props.notify, f"protected '{full_key}' field changed", msg, email_only=True)
-
-    def get_property(self, key, default=None, category=None, field_type=None, decrypted=False):
-        category, key = key.split('.', 1) if '.' in key else (category, key)
-
-        try:
-            prop_value = self.properties.get(category=category, key=key).get_value(field_type)
-            return DECRYPTER.decrypt(prop_value) if decrypted and prop_value else prop_value
-        except Exception:
-            return default
-
-    def set_secret_property(self, key, value):
-        return self.set_property(key, value, category="secrets", encrypted=True)
-
-    def get_secret_property(self, key, default=None):
-        return self.get_property(key, default, "secrets", decrypted=True)

mojo/ws4redis/README.md

@@ -1,174 +0,0 @@
-# Websocket HOWTO
-
-## Authentication
-
-### JWT
-
-Requires an existing JWT token that has gone through authentication process via rest
-
-```json
-{
-    "action": "auth",
-    "kind": "jwt",
-    "token": "..."
-}
-```
-
-
-
-### Model Authentication
-
-You can implement custom authentication flows via a model by using the WS4REDIS_AUTHENTICATORS in your django settings.py.
-
-##### WS4REDIS_AUTHENTICATORS
-
-```python
-WS4REDIS_AUTHENTICATORS = {
-    "mymodel": "myapp.MyModel"
-}
-```
-
-In your Model you will need to add the following class methods.
-
-This method is used by the async/websocket service to authenticate.
-If the model can authenticate the connection it should return dict with kind and pk of the model that is authenticaed.
-
-
-
-##### authWS4RedisConnection
-
-This method will authenticate the model, or return None if authentication failed.
-
-```python
-@classmethod
-def authWS4RedisConnection(cls, auth_data):
-    if auth_data and auth_data.token:
-        terminal = cls.objects.filter(token=auth_data.token).last()
-        if terminal is not None:
-            # we now return the terminal credentials to the framework
-            return UberDict(
-                kind="terminal",
-                pk=terminal.id,
-                uuid=terminal.tid,
-                token=auth_data.token,
-                only_one=True,  # only allows one connection at a time
-                instance=terminal)
-    return None
-```
-
-
-
-##### canPublishTo
-
-Add this to your Model to validate messages from this connection to be sent to this channel.
-
-```python
-@classmethod
-def canPublishTo(cls, credentials, msg):
-    if credentials:
-        return True
-    return False
-```
-
-
-
-##### WS4REDIS_CHANNELS
-
-Map channels to models
-
-```python
-WS4REDIS_CHANNELS = {
-    "group": "account.Group",
-    "chat": "chat.Room",
-}
-```
-
-
-
-##### onWS4RedisMessage
-
-Add this to your Model to allow for handling of messages sent to this channel.
-
-```python
-@classmethod
-def onWS4RedisMessage(cls, credentials, msg):
-    if msg.action == "status":
-      cls.createStatusRecord(msg)
-
-```
-
-
-
-### URL Params
-
-You can also use params in the url of the websocket.  
-
-**THIS IS NOT RECOMMENDED as the url params are not encrypted and can be easily snooped.**
-
-Include something like the follow in your django settings.py:
-
-```python
-def URL_AUTHENTICATOR(ws_con):
-    from objict import objict
-    token = ws_con.request.GET.get("token", None)
-    session_key = ws_con.request.GET.get("session_key", None)
-    if token is not None:
-      # this example assume the token is used for terminal auth
-      # you will still need to implement the Custom Auth flows to handle this
-    	ws_con.on_auth(objict(kind="terminal", token=token))
-		elif session_key is not None:
-      # or alternative is a session
-    	ws_con.on_auth(objict(kind="session", token=session_key))
-
-```
-
-
-
-## Subscribe
-
-```json
-{
-    "action": "subscribe",
-    "channel": "group",
-    "pk": 3,
-}
-```
-
-### Security
-
-In settins WS4REDIS_CHANNELS, map your channel to a model.
-The model should have a classmethod for canSubscribeTo that returns a list of pk they can subscribe to.
-
-
-## UnSubscribe
-
-```json
-{
-    "action": "unsubscribe",
-    "channel": "group",
-    "pk": 3,
-}
-```
-
-
-## Publish / Send To
-
-```json
-{
-    "action": "publish",
-    "channel": "group",
-    "pk": 3,
-    "message": "..."
-}
-```
-
-### Security
-
-In settins WS4REDIS_CHANNELS, map your channel to a model.
-The model should have a classmethod for canPublishTo that returns a list of pk they can publish to.
-
-
-## Custom Messages
-
-If an unknown action is sent with a channel then the framework will call onWS4RedisMessage on the channel model.
-

mojo/ws4redis/__init__.py

@@ -1,2 +0,0 @@
-# -*- coding: utf-8 -*-
-__version__ = '1.1.8'