django-eventspipe 0.1.0b1__tar.gz

django_eventspipe-0.1.0b1/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Giuseppe Corti <giuseppe.corti@protonmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

django_eventspipe-0.1.0b1/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.1
+Name: django_eventspipe
+Version: 0.1.0b1
+Summary: TODO
+Author-email: Giuseppe Corti <giuseppe.corti@pm.me>
+Project-URL: Homepage, https://github.com/r3vn/django_eventspipe
+Project-URL: Issues, https://github.com/r3vn/django_eventspipe/issues
+Classifier: Environment :: Web Environment
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: django>=5.0
+Requires-Dist: celery>=5.4
+
+TODO

django_eventspipe-0.1.0b1/README.md

@@ -0,0 +1 @@
+TODO

django_eventspipe-0.1.0b1/pyproject.toml

@@ -0,0 +1,30 @@
+[project]
+name = "django_eventspipe"
+version = "0.1.0b1"
+authors = [
+  { name="Giuseppe Corti", email="giuseppe.corti@pm.me" },
+]
+description = "TODO"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "django>=5.0",
+    "celery>=5.4"
+]
+classifiers = [
+    "Environment :: Web Environment",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Topic :: Software Development :: Libraries :: Application Frameworks",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+[project.urls]
+Homepage = "https://github.com/r3vn/django_eventspipe"
+Issues = "https://github.com/r3vn/django_eventspipe/issues"
+

django_eventspipe-0.1.0b1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

django_eventspipe-0.1.0b1/src/admin.py

@@ -0,0 +1,218 @@
+from django import forms
+from django.contrib import admin
+from django.utils.html import format_html
+from django.urls import reverse
+
+from .utils import linkify, PrettyJSONEncoder
+
+from .models import (
+    PipelineDefinition,
+    PipelineDefinitionTaskDefinition,
+    Task,
+    TaskDefinition,
+    Pipeline,
+    Artifact,
+    PipelineArtifact
+)
+
+class InlinePipeline(admin.TabularInline):
+    model = PipelineArtifact
+    extra = 0
+    ordering = ("pk",)
+    can_delete = False
+    readonly_fields = [
+        'pk',
+        'pipeline',
+    ]
+
+class InlineTask(admin.TabularInline):
+    model = Task
+    extra = 0
+    ordering = ("pk",)
+    can_delete = False
+    readonly_fields = [
+        '_status',
+        '_name',
+        'node',
+        'start_ts',
+        'end_ts'
+    ]
+    exclude = [
+        'definition',
+        'pipeline_definition',
+        'status'
+    ]
+
+    def _status(self, obj):
+        """
+        Read job's status properly
+        """
+        colors = ["#2196F3","#04AA6D","#f23232", "#f8f8f8", "#ff9800"]
+        return format_html(
+            "<span style=\"background-color:%s;display:block;text-align:center;font-size:0.6rem;padding:1px;max-width:55px;\"><b>%s</b></span>" 
+            % (
+                colors[obj.status], 
+                obj.get_status_display().upper()
+            )
+        )
+
+    def _name(self, obj):
+        return "%s" % obj.definition.task_definition.function
+
+class JsonOptionsForm(forms.ModelForm):
+    options = forms.JSONField(
+        encoder=PrettyJSONEncoder,
+        required=False
+    )
+
+class InlineTaskDefinition(admin.TabularInline):
+    model = PipelineDefinitionTaskDefinition
+    extra = 0
+    ordering = ("order",)
+    #form = JsonOptionsForm
+
+    exclude = ['options']
+
+    def _name(self, obj):
+        """
+        Get Job function name from JobDefinition object
+        """
+        return "%s" % obj.task_definition.function
+
+
+@admin.register(Pipeline)
+class PipelineAdmin(admin.ModelAdmin):
+    list_display = (
+        "pk",
+        "_status",
+        "name",
+        linkify("user"),
+        "start_ts",
+        "end_ts"
+    )
+    readonly_fields = []
+    inlines = [InlineTask]
+
+    # https://stackoverflow.com/a/19884095
+    def get_readonly_fields(self, request, obj=None):
+        return list(self.readonly_fields) + \
+               [field.name for field in obj._meta.fields] + \
+               [field.name for field in obj._meta.many_to_many]
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_delete_permission(self, request, obj=None):
+        return False
+
+    def _status(self, obj):
+        """
+        Read job's status properly
+        """
+        colors = ["#2196F3","#04AA6D","#f23232", "#f8f8f8"]
+        return format_html(
+            "<span style=\"background-color:%s;display:block;text-align:center;font-size:0.6rem;padding:1px;max-width:55px;\"><b>%s</b></span>" 
+            % (
+                colors[obj.status], 
+                obj.get_status_display().upper()
+            )
+        )
+
+@admin.register(PipelineDefinition)
+class PipelineDefinitionAdmin(admin.ModelAdmin):
+    list_display = (
+        "pk",
+        "enabled",
+        "event",
+        "filters",
+        "tasks_definition"
+    )
+    inlines = [InlineTaskDefinition]
+    form = JsonOptionsForm
+
+    @admin.action(description="Disable selected PipelineDefinition")
+    def disable_selection(self, request, queryset):
+        for object in queryset:
+
+            object.enabled = False
+            object.save()
+
+    @admin.action(description="Enable selected PipelineDefinition")
+    def enable_selection(self, request, queryset):
+        for object in queryset:
+
+            object.enabled = True
+            object.save()
+
+    @admin.action(description="Duplicate selected PipelineDefinition")
+    def duplicate_selection(self, request, queryset):
+        for object in queryset:
+
+            task_definitions = PipelineDefinitionTaskDefinition.objects.filter(pipeline_definition=object)
+
+            object.id = None
+            object.save()
+
+            for task_definition in task_definitions:
+                task_definition.id = None
+                task_definition.pipeline_definition = object
+                task_definition.save()
+
+    actions = [duplicate_selection, enable_selection, disable_selection]
+
+    def tasks_definition(self, obj):
+        """
+        Return a "pretty" tasks execution order view 
+        """
+        rt_string = ""
+        for definition in PipelineDefinitionTaskDefinition.objects.filter(pipeline_definition=obj).order_by('order'):
+            if definition.enabled:
+                job_name = definition.task_definition.function
+            else:
+                # task is disabled, Strikethrough the function name
+                job_name = "<s style=\"color:var(--delete-button-bg)\">%s</s>" % definition.task_definition.function
+
+            rt_string = "%s <span style=\"color:var(--link-fg)\">→</span> %s" % (rt_string, job_name)
+
+        return format_html(rt_string)
+
+@admin.register(TaskDefinition)
+class TaskDefinitionAdmin(admin.ModelAdmin):
+    list_display = (
+        "function",
+        "description"
+    )
+    ordering = ('function',)
+
+@admin.register(Artifact)
+class ArtifactAdmin(admin.ModelAdmin):
+    list_display = (
+        "pk",
+        "name",
+        "size",
+        "md5sum",
+        "timestamp",
+        "download"
+    )
+
+    inlines = [InlinePipeline]
+    readonly_fields = []
+
+    # https://stackoverflow.com/a/19884095
+    def get_readonly_fields(self, request, obj=None):
+        return list(self.readonly_fields) + \
+               [field.name for field in obj._meta.fields] + \
+               [field.name for field in obj._meta.many_to_many]
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_delete_permission(self, request, obj=None):
+        return False
+
+    def download(self, obj: Artifact) -> str:
+        url = reverse('get_artifact', args=(obj.pk,))
+        return format_html("<a class='button' href='%s'>📝 DOWNLOAD</a>" % url)
+
+    def size(self, obj: Artifact) -> str:
+        return "%s KB" % str(obj.get_size())

django_eventspipe-0.1.0b1/src/apps.py

@@ -0,0 +1,10 @@
+from django.apps import AppConfig
+
+
+class DjangoEventspipeConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'django_eventspipe'
+    verbose_name = 'Automations'
+
+    def ready(self):
+        import django_eventspipe.signals

django_eventspipe-0.1.0b1/src/decorators.py

@@ -0,0 +1,53 @@
+import traceback
+import platform
+
+from functools import wraps
+
+from .models import Pipeline, Task
+from .exceptions import TaskFailed
+
+def tracked_task(func):
+    @wraps(func)
+    def wrapper(self, context: dict[str, object], *args, **kw) -> dict[str, object]:
+        
+        # HACK: Get task name
+        if str(func.__name__) in str(func.__module__):
+            taskname = str(func.__module__)
+        else:
+            taskname = "%s.%s" % (func.__module__, func.__name__)
+
+        # Get Pipeline object
+        self.pipeline = Pipeline.objects.get(pk=context["pipeline"])
+        
+        # Get current pipeline's task
+        pipeline_task = Task.objects.get(
+            pipeline=self.pipeline, 
+            definition__task_definition__function=taskname
+        )
+
+        self.pipeline.log("executing '%s'..." % taskname)
+
+        # Start Task's tracking
+        pipeline_task.tracking_start(node=platform.node())
+
+        try:
+            # Execute Task
+            result = func(self, context, *args, **kw)
+
+            # Set Task's status as success
+            pipeline_task.tracking_update(1)
+
+        except Exception as E:
+            # Log the Exception
+            self.pipeline.log(str(traceback.format_exc()))
+
+            # Set Task's status as failed
+            pipeline_task.tracking_update(2)
+
+            raise TaskFailed
+
+        self.pipeline.log("'%s' execution complete." % taskname)
+
+        return result
+
+    return wrapper

django_eventspipe-0.1.0b1/src/django_eventspipe.egg-info/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.1
+Name: django_eventspipe
+Version: 0.1.0b1
+Summary: TODO
+Author-email: Giuseppe Corti <giuseppe.corti@pm.me>
+Project-URL: Homepage, https://github.com/r3vn/django_eventspipe
+Project-URL: Issues, https://github.com/r3vn/django_eventspipe/issues
+Classifier: Environment :: Web Environment
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: django>=5.0
+Requires-Dist: celery>=5.4
+
+TODO

django_eventspipe-0.1.0b1/src/django_eventspipe.egg-info/SOURCES.txt

@@ -0,0 +1,28 @@
+LICENSE.txt
+README.md
+pyproject.toml
+src/__init__.py
+src/admin.py
+src/apps.py
+src/decorators.py
+src/exceptions.py
+src/signals.py
+src/tests.py
+src/urls.py
+src/utils.py
+src/views.py
+src/django_eventspipe.egg-info/PKG-INFO
+src/django_eventspipe.egg-info/SOURCES.txt
+src/django_eventspipe.egg-info/dependency_links.txt
+src/django_eventspipe.egg-info/requires.txt
+src/django_eventspipe.egg-info/top_level.txt
+src/migrations/0001_initial.py
+src/migrations/__init__.py
+src/models/__init__.py
+src/models/artifact.py
+src/models/pipeline.py
+src/models/pipeline_artifact.py
+src/models/pipeline_definition.py
+src/models/pipeline_definition_task_definition.py
+src/models/task.py
+src/models/task_definition.py

django_eventspipe-0.1.0b1/src/django_eventspipe.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

django_eventspipe-0.1.0b1/src/django_eventspipe.egg-info/requires.txt

@@ -0,0 +1,2 @@
+django>=5.0
+celery>=5.4

django_eventspipe-0.1.0b1/src/django_eventspipe.egg-info/top_level.txt

@@ -0,0 +1,12 @@
+__init__
+admin
+apps
+decorators
+exceptions
+migrations
+models
+signals
+tests
+urls
+utils
+views

django_eventspipe-0.1.0b1/src/exceptions.py

@@ -0,0 +1,2 @@
+class TaskFailed(Exception):
+    pass

django_eventspipe-0.1.0b1/src/migrations/0001_initial.py

@@ -0,0 +1,91 @@
+# Generated by Django 5.0.6 on 2024-09-08 12:57
+
+import django.db.models.deletion
+import django_eventspipe.utils
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Artifact',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=1024)),
+                ('data', models.BinaryField(blank=True)),
+                ('timestamp', models.DateTimeField(auto_now_add=True)),
+                ('md5sum', models.CharField(blank=True, editable=False, max_length=32, unique=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='PipelineDefinition',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('event', models.CharField(max_length=256)),
+                ('filters', models.JSONField(blank=True, default=dict, null=True)),
+                ('options', models.JSONField(blank=True, default=dict, null=True)),
+                ('enabled', models.BooleanField(default=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='TaskDefinition',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('function', models.CharField(max_length=512, unique=True)),
+                ('description', models.TextField(blank=True, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Pipeline',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('node', models.CharField(default='undefined', max_length=256)),
+                ('name', models.CharField(max_length=128)),
+                ('status', models.IntegerField(choices=[(0, 'running'), (1, 'success'), (2, 'error'), (3, 'queued')], default=3)),
+                ('tasks_count', models.IntegerField(default=0)),
+                ('current_task', models.IntegerField(default=0)),
+                ('start_ts', models.DateTimeField(auto_now_add=True)),
+                ('end_ts', models.DateTimeField(blank=True, null=True)),
+                ('user', models.ForeignKey(on_delete=models.SET(django_eventspipe.utils.get_sentinel_user), to=settings.AUTH_USER_MODEL)),
+                ('definition', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='django_eventspipe.pipelinedefinition')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='PipelineArtifact',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('artifact', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='django_eventspipe.artifact')),
+                ('pipeline', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='django_eventspipe.pipeline')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='PipelineDefinitionTaskDefinition',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('enabled', models.BooleanField(default=True)),
+                ('order', models.IntegerField(default=20)),
+                ('pipeline_definition', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='django_eventspipe.pipelinedefinition')),
+                ('task_definition', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='django_eventspipe.taskdefinition')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Task',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('status', models.IntegerField(choices=[(0, 'running'), (1, 'success'), (2, 'error'), (3, 'queued'), (4, 'skipped')], default=2)),
+                ('node', models.CharField(default='undefined', max_length=256)),
+                ('start_ts', models.DateTimeField(blank=True, null=True)),
+                ('end_ts', models.DateTimeField(blank=True, null=True)),
+                ('definition', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='django_eventspipe.pipelinedefinitiontaskdefinition')),
+                ('pipeline', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='django_eventspipe.pipeline')),
+            ],
+        ),
+    ]

django_eventspipe-0.1.0b1/src/models/__init__.py

@@ -0,0 +1,7 @@
+from .task_definition import TaskDefinition
+from .artifact import Artifact
+from .pipeline_definition import PipelineDefinition
+from .pipeline_definition_task_definition import PipelineDefinitionTaskDefinition
+from .pipeline import Pipeline
+from .task import Task
+from .pipeline_artifact import PipelineArtifact

django_eventspipe-0.1.0b1/src/models/artifact.py

@@ -0,0 +1,29 @@
+import hashlib
+from django.db import models
+
+class Artifact(models.Model):
+    name       = models.CharField(max_length=1024)
+    data       = models.BinaryField(blank=True)
+    timestamp  = models.DateTimeField(auto_now_add=True)
+    md5sum     = models.CharField(max_length=32, blank=True, editable=False, unique=True)  # Unique MD5 checksum
+
+    @classmethod
+    def get_or_create(cls, name:str, data:bytes) -> object:
+        # Compute the MD5 checksum of the data
+        md5sum = hashlib.md5(data).hexdigest()
+
+        if cls.objects.filter(md5sum=md5sum).exists():
+            return cls.objects.get(md5sum=md5sum)
+
+        file = cls(name=name,data=data,md5sum=md5sum)
+        file.save()
+
+        return file
+        
+    def get_size(self) -> float:
+        """
+        Get size in KB of a file
+        """
+        return len(self.data) / 1000
+
+

django_eventspipe-0.1.0b1/src/models/pipeline.py

@@ -0,0 +1,173 @@
+import platform
+
+from celery import chain
+
+from django.db import models
+from django.contrib.auth.models import User
+from django.contrib.admin.models import LogEntry, ADDITION
+from django.contrib.contenttypes.models import ContentType
+from django.apps import apps
+from django.utils import timezone
+
+from django_eventspipe.utils import get_sentinel_user
+
+class Pipeline(models.Model):
+    
+    STATUS_CHOICES = [
+        (0, "running"),
+        (1, "success"),
+        (2, "error"),
+        (3, "queued")
+    ]
+
+    node         = models.CharField(max_length=256, default="undefined")
+    name         = models.CharField(max_length=128)
+    status       = models.IntegerField(default=3, choices=STATUS_CHOICES)
+    definition   = models.ForeignKey('django_eventspipe.PipelineDefinition', on_delete=models.SET_NULL, blank=True, null=True)
+    tasks_count  = models.IntegerField(default=0)
+    current_task = models.IntegerField(default=0)
+    start_ts     = models.DateTimeField(auto_now_add=True)
+    end_ts       = models.DateTimeField(blank=True, null=True)
+    user         = models.ForeignKey(User, on_delete=models.SET(get_sentinel_user))
+
+    @classmethod
+    def new_from_event(cls, event: dict) -> object | bool:
+        """
+        This function attempt to create and execute a `Pipeline` objects from a dict.
+        `Pipeline` objects require a `PipelineDefinition` object defined in order to start.
+        Filtered `PipelineDefinition` objects always have priority over generic ones,
+        this allow replacing 's definitions only when required. 
+        """
+        PipelineDefinition = apps.get_model("django_eventspipe.PipelineDefinition")
+
+        # Check for available info for this event
+        if "info" in event.keys():
+            pipeline_name = "%s %s" % (event["name"], event["info"])
+        else:
+            pipeline_name = event["name"]
+
+        # Create the pipeline objects
+        pipelines   = []
+        definitions = PipelineDefinition.get_definitions(event)
+
+        if len(definitions) <= 0:
+            # no such pipeline definition
+            return False
+
+        for definition in definitions:
+            # Create Pipeline object
+            pipeline = cls(
+                name=pipeline_name, 
+                user=User.objects.get(pk=event["user"]), 
+                node=platform.node(),
+                definition=definition
+            )
+            pipeline.save()
+
+            # Run Pipeline
+            pipeline.run(event)
+            pipelines.append(pipeline)
+
+        return pipelines
+
+    def __str__(self):
+        return "Pipeline #%d" % self.pk
+
+    def save_artifact(self, file_name: str, file_data: bytes) -> bool:
+        """
+        Save an artifact for this Pipeline
+        """
+
+        PipelineArtifact = apps.get_model("django_eventspipe.PipelineArtifact")
+
+        return PipelineArtifact.add_artifact(
+            pipeline  = self,
+            file_name = file_name, 
+            file_data = file_data,
+        )
+
+    def get_artifacts(self) -> dict[str, object]:
+        """
+        Get stored artifacts for this Pipeline
+        """
+        PipelineArtifact = apps.get_model("django_eventspipe.PipelineArtifact")
+
+        return PipelineArtifact.get_artifacts(self)
+
+    def run(self, event: dict) -> None:
+        """ 
+        Execute a `Pipeline`
+        """
+        Task = apps.get_model("django_eventspipe.Task")
+
+        # add initial pipeline's log entry
+        self.log("Event received %s" % str(event))
+
+        # Get initial context from event
+        context = {
+            "pipeline" : self.pk
+        }
+
+        # retrive context data from event
+        for data in event.keys():
+            context[data] = event[data]
+
+        # retrive context data from PipelineDefinitinon's options
+        for data in self.definition.options.keys():
+            context[data] = self.definition.options[data]
+
+        # Get Task definition for this event on this perimeter
+        pipeline_tasks   = Task.create_tasks(self)
+
+        # Get celery chain for this pipeline
+        tasks_chain_list = self.definition.get_tasks_chain(context) 
+
+        # No Tasks defined for this pipeline, exit
+        if len(pipeline_tasks) <= 0:
+            self.status = 1
+            self.tasks_count = 0
+            self.current_task = 0
+            self.end_ts = timezone.now()
+            self.save()
+
+            return None
+
+        # Set pipeline as queued
+        self.status      = 3
+        self.tasks_count = len(pipeline_tasks)
+        self.save()
+
+        # Start celery chain
+        pipeline_chain = chain(tasks_chain_list) 
+        pipeline_chain.apply_async()
+
+    def log(self, entry: str) -> None:
+        """
+        Add a LogEntry on a `Pipeline`
+        """
+        progress = "[%d/%d] " % (self.current_task, self.tasks_count) 
+        log_entry = "%s%s" % (progress, entry)
+
+        # Add an ADDITION logentry for this asset
+        LogEntry.objects.log_action(
+            user_id=self.user.pk,
+            content_type_id=ContentType.objects.get_for_model(Pipeline).pk,
+            object_id=self.id,
+            object_repr="Pipeline #%d" % self.id,
+            action_flag=ADDITION,
+            change_message=log_entry
+        )
+
+    def fail(self) -> None:
+        """
+        Set a `Pipeline` and his uncompleted `Task` objects as failed.
+        """
+        Task = apps.get_model("django_eventspipe.Task")
+
+        # Update Pipeline object
+        self.status = 2
+        self.end_ts = timezone.now()
+        self.save()
+
+        # Update queued Tasks
+        Task.pipeline_failed(self)

django_eventspipe-0.1.0b1/src/models/pipeline_artifact.py

@@ -0,0 +1,37 @@
+from django.db import models
+from django.apps import apps
+
+class PipelineArtifact(models.Model):
+    """
+    Artifact -> Pipeline association
+    """
+    pipeline = models.ForeignKey("django_eventspipe.Pipeline", on_delete=models.CASCADE)
+    artifact = models.ForeignKey("django_eventspipe.Artifact", on_delete=models.CASCADE)
+
+    @classmethod
+    def add_artifact(cls, pipeline: object, file_name: str, file_data: bytes) -> bool:
+        """
+        Add a new artifact to the database
+        """
+        Artifact = apps.get_model("django_eventspipe.Artifact")
+
+        artifact = Artifact.get_or_create(name=file_name, data=file_data)
+
+        pipeline_artifact = cls(pipeline=pipeline, artifact=artifact)
+        pipeline_artifact.save()
+
+        return True
+
+    @classmethod
+    def get_artifacts(cls, pipeline: object) -> dict[str, object]:
+        """
+        Get stored artifacts for a Pipeline
+        """
+
+        out = {}
+        artifacts = cls.objects.filter(pipeline=pipeline)
+
+        for artifact in artifacts:
+            out[artifact.artifact.name] = bytes(artifact.artifact.data)
+
+        return out

django_eventspipe-0.1.0b1/src/models/pipeline_definition.py

@@ -0,0 +1,77 @@
+from django.db import models
+from django.apps import apps
+from django.utils.module_loading import import_string
+
+class PipelineDefinition(models.Model):
+    event     = models.CharField(max_length=256)
+    filters   = models.JSONField(blank=True, null=True, default=dict)
+    options   = models.JSONField(blank=True, null=True, default=dict)
+    enabled   = models.BooleanField(default=True)
+
+    @classmethod
+    def get_definitions(cls, event):
+        """
+        Get pipeline definitions for a given event.
+        """
+        event_name = event.get("name")
+
+        # Use ORM to fetch enabled definitions for the event in one query
+        all_definitions = cls.objects.filter(event=event_name, enabled=True)
+
+        # Separate generic and custom definitions
+        generic_definitions = []
+        custom_definitions = []
+
+        for definition in all_definitions:
+            filters = definition.filters or {}
+            
+            # Check if filters exist, otherwise it's a generic definition
+            if filters:
+                match = all(
+                    event.get(key) == value 
+                    for key, value in filters.items()
+                    if key in event
+                )
+                if match:
+                    custom_definitions.append(definition)
+            else:
+                generic_definitions.append(definition)
+
+        # Return custom definitions if available, otherwise return generic ones
+        return custom_definitions if custom_definitions else generic_definitions
+
+    def get_defined_tasks(self) -> list:
+        """
+        Get Tasks defined for this PipelineDefinition
+        """
+        PipelineDefinitionTaskDefinition = apps.get_model('django_eventspipe.PipelineDefinitionTaskDefinition')
+
+        tasks = []
+        for definition in PipelineDefinitionTaskDefinition.objects.filter(pipeline_definition=self, enabled=True).order_by('order'):
+            tasks.append(definition)
+
+        return tasks
+
+    def get_tasks_chain(self, context) -> list:
+        """
+        Get Tasks defined for this PipelineDefinition as a celery chain
+        """
+        PipelineDefinitionTaskDefinition = apps.get_model('django_eventspipe.PipelineDefinitionTaskDefinition')
+
+        chain = []
+        first = True
+
+        for task_function in PipelineDefinitionTaskDefinition.objects.filter(
+            pipeline_definition=self, 
+            enabled=True
+        ).order_by('order').values_list('task_definition__function'):
+
+            if first:
+                signature = import_string(task_function[0]).s(context)
+                first = False
+            else:
+                signature = import_string(task_function[0]).s()
+
+            chain.append(signature)
+
+        return chain

django_eventspipe-0.1.0b1/src/models/pipeline_definition_task_definition.py

@@ -0,0 +1,8 @@
+from django.db import models
+
+class PipelineDefinitionTaskDefinition(models.Model):
+    pipeline_definition = models.ForeignKey('django_eventspipe.PipelineDefinition', on_delete=models.CASCADE)
+    task_definition     = models.ForeignKey('django_eventspipe.TaskDefinition', on_delete=models.CASCADE)
+    enabled             = models.BooleanField(default=True)
+    order               = models.IntegerField(default=20)
+

django_eventspipe-0.1.0b1/src/models/task.py

@@ -0,0 +1,86 @@
+from django.db import models 
+from django.utils import timezone
+
+class Task(models.Model):
+
+    STATUS_CHOICES = [
+        (0, "running"),
+        (1, "success"),
+        (2, "error"),
+        (3, "queued"),
+        (4, "skipped")
+    ]
+
+    status     = models.IntegerField(default=2, choices=STATUS_CHOICES)
+    node       = models.CharField(max_length=256, default="undefined")
+    pipeline   = models.ForeignKey('django_eventspipe.Pipeline', on_delete=models.CASCADE)
+    definition = models.ForeignKey('django_eventspipe.PipelineDefinitionTaskDefinition', null=True, on_delete=models.SET_NULL)
+    start_ts   = models.DateTimeField(blank=True, null=True)
+    end_ts     = models.DateTimeField(blank=True, null=True)
+
+    @classmethod
+    def create_tasks(cls, pipeline: object) -> list:
+        """
+        Create `Task` for a given pipeline
+        """
+        tasks = []
+
+        for defined_task in pipeline.definition.get_defined_tasks():
+            # Create Task
+            task = cls(
+                status     = 3,
+                pipeline   = pipeline,
+                definition = defined_task
+            )
+            task.save()
+            tasks.append(task)
+    
+        return tasks
+
+    @classmethod
+    def pipeline_failed(cls, pipeline: object) -> None:
+        """
+        Set all queued `Task` as failed.
+        """
+        tasks = cls.objects.all().filter(status=3, pipeline=pipeline) # kill both queued and running
+
+        for task in tasks:
+            task.status = 2   
+            task.save()
+
+    def tracking_start(self, node:str) -> None:
+        """
+        Start tracking a `Task`
+        """
+        self.status = 0
+        self.node = node
+        self.start_ts = timezone.now()
+        self.save()
+
+        # Increase Pipeline's current_task
+        self.pipeline.current_task += 1
+
+        # Set Pipeline's status as running
+        if self.pipeline.status != 0:
+            self.pipeline.status = 0
+
+        self.pipeline.save()
+
+    def tracking_update(self, status: int) -> None:
+        """
+        Update `Task`'s tracking data
+        """
+        # Update this object
+        self.status = status
+        self.end_ts = timezone.now()
+        self.save()
+
+        if status == 1:
+            # Check if pipeline is completed
+            if self.pipeline.current_task == self.pipeline.tasks_count:
+                self.pipeline.status = status
+                self.pipeline.end_ts = timezone.now()
+                self.pipeline.save()
+        else:
+            # Set pipeline and all queued Tasks as failed
+            self.pipeline.fail()

django_eventspipe-0.1.0b1/src/models/task_definition.py

@@ -0,0 +1,40 @@
+from django.db import models
+from django.utils.module_loading import import_string
+
+class TaskDefinition(models.Model):
+    function    = models.CharField(max_length=512, unique=True)
+    description = models.TextField(null=True, blank=True)
+
+    @classmethod
+    def check_definitions(cls, tasks: list) -> None:
+        """
+        Automatically import `Task` objects from celery
+        """
+        clean_tasks = []
+
+        # skip celery stuff
+        for task in tasks:
+            if "celery." in task: continue
+            clean_tasks.append(task)
+
+        # remove uknown `TaskDefinition`
+        for taskdef in cls.objects.all():
+            if taskdef.function not in clean_tasks:
+                taskdef.delete()
+
+        # import registered `Task` objects
+        for task in clean_tasks:
+            try:
+                imported_task = import_string(task)
+
+                if not cls.objects.filter(function=task).exists():
+                    # Get `Task` function and documentation
+                    new_task_def = cls(
+                        function=task, 
+                        description=imported_task.__doc__
+                    )
+                    new_task_def.save()
+            except: pass
+
+    def __str__(self) -> str:
+        return "%s" % self.function

django_eventspipe-0.1.0b1/src/signals.py

@@ -0,0 +1,15 @@
+from django.dispatch import Signal, receiver
+from .models import Pipeline
+
+event_signal = Signal()
+
+@receiver(event_signal)
+def create_pipelines(
+    sender: object, 
+    event: dict[str, object], 
+    **kwargs
+) -> None:
+    """
+    Signal handler to create `Pipeline` objects from an event
+    """
+    Pipeline.new_from_event(event)

django_eventspipe-0.1.0b1/src/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

django_eventspipe-0.1.0b1/src/urls.py

@@ -0,0 +1,7 @@
+from django.urls import path
+
+from . import views
+
+urlpatterns = [
+    path('artifact/<int:artifact_id>', views.get_artifact, name="get_artifact"),
+]

django_eventspipe-0.1.0b1/src/utils.py

@@ -0,0 +1,38 @@
+import json
+
+from django.urls import reverse
+from django.utils.html import format_html
+from django.contrib.auth import get_user_model
+from django.contrib.auth.models import User
+
+def get_sentinel_user() -> User:
+    """
+    Prevent pipeline removal when deleting an user
+    """
+    return get_user_model().objects.get_or_create(username="deleted")[0]
+
+# https://stackoverflow.com/a/53092940
+def linkify(field_name: str) -> str:
+    """
+    Converts a foreign key value into clickable links.
+    
+    If field_name is 'parent', link text will be str(obj.parent)
+    Link will be admin url for the admin url for obj.parent.id:change
+    """
+    def _linkify(obj):
+        linked_obj = getattr(obj, field_name)
+        if linked_obj is None:
+            return '-'
+        app_label = linked_obj._meta.app_label
+        model_name = linked_obj._meta.model_name
+        view_name = f'admin:{app_label}_{model_name}_change'
+        link_url = reverse(view_name, args=[linked_obj.pk])
+        return format_html('<a href="{}">{}</a>', link_url, linked_obj)
+
+    _linkify.short_description = field_name  # Sets column name
+    return _linkify
+
+# https://stackoverflow.com/a/72256767
+class PrettyJSONEncoder(json.JSONEncoder):
+    def __init__(self, *args, indent: int, sort_keys: bool, **kwargs):
+        super().__init__(*args, indent=2, sort_keys=True, **kwargs)

django_eventspipe-0.1.0b1/src/views.py

@@ -0,0 +1,16 @@
+from django.http import HttpRequest, HttpResponse
+from django.contrib.auth.decorators import login_required
+
+from .models import Artifact
+
+@login_required
+def get_artifact(request: HttpRequest, artifact_id: int) -> HttpResponse:
+    """
+    Get a file from Database
+    """
+    file = Artifact.objects.get(pk=artifact_id)
+
+    response = HttpResponse(file.data, content_type='application/octet-stream')
+    response['Content-Disposition']='attachment;filename=%s' % file.name
+
+    return response