flagsmith-common 3.8.1__tar.gz → 3.9.0__tar.gz

{flagsmith_common-3.8.1 → flagsmith_common-3.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flagsmith-common
-Version: 3.8.1
+Version: 3.9.0
 Summary: Flagsmith's common library
 Author: Matthew Elwell, Gagan Trivedi, Kim Gustyr, Zach Aysan, Francesco Lo Franco, Rodrigo López Dato, Evandro Myller, Wadii Zaim
 License-Expression: BSD-3-Clause
@@ -15,7 +15,7 @@ Requires-Dist: djangorestframework-recursive ; extra == 'common-core'
 Requires-Dist: djangorestframework ; extra == 'common-core'
 Requires-Dist: drf-spectacular>=0.28.0,<1 ; extra == 'common-core'
 Requires-Dist: drf-writable-nested ; extra == 'common-core'
-Requires-Dist: environs<15 ; extra == 'common-core'
+Requires-Dist: environs<16 ; extra == 'common-core'
 Requires-Dist: gunicorn>=19.1 ; extra == 'common-core'
 Requires-Dist: inflection ; extra == 'common-core'
 Requires-Dist: opentelemetry-api>=1.25,<2 ; extra == 'common-core'

{flagsmith_common-3.8.1 → flagsmith_common-3.9.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "flagsmith-common"
-version = "3.8.1"
+version = "3.9.0"
 description = "Flagsmith's common library"
 requires-python = ">=3.11,<4.0"
 dependencies = []
@@ -14,7 +14,7 @@ optional-dependencies = { test-tools = [
     "djangorestframework",
     "drf-spectacular (>=0.28.0,<1)",
     "drf-writable-nested",
-    "environs (<15)",
+    "environs (<16)",
     "gunicorn (>=19.1)",
     "inflection",
     "opentelemetry-api (>=1.25,<2)",

flagsmith_common-3.9.0/src/common/core/templates/docgen-events.md

@@ -0,0 +1,9 @@
+{% for event in flagsmith_events %}
+### `{{ event.name }}`
+
+Logged at `{{ event.level }}` from:
+{% for location in event.locations %} - `{{ location.path }}:{{ location.line }}`
+{% endfor %}
+Attributes:
+{% for attr in event.attributes %} - `{{ attr }}`
+{% endfor %}{% endfor %}

flagsmith_common-3.9.0/src/common/core/templates/docgen-metrics.md

@@ -0,0 +1,10 @@
+{% for metric in flagsmith_metrics %}
+### `{{ metric.name }}`
+
+{{ metric.type|title }}.
+
+{{ metric.documentation }}
+
+Labels:
+{% for label in metric.labels %} - `{{ label }}`
+{% endfor %}{% endfor %}

{flagsmith_common-3.8.1 → flagsmith_common-3.9.0}/src/task_processor/admin.py

@@ -16,7 +16,11 @@ class RecurringTaskAdmin(admin.ModelAdmin[RecurringTask]):
         "last_run_status",
         "last_run_finished_at",
         "is_locked",
+        "is_disabled",
+        "num_consecutive_failures",
     )
+    list_filter = ("is_disabled",)
+    actions = ("unlock", "enable")
     readonly_fields = ("args", "kwargs")
 
     def last_run_status(self, instance: RecurringTask) -> str | None:
@@ -36,3 +40,11 @@ class RecurringTaskAdmin(admin.ModelAdmin[RecurringTask]):
         queryset: QuerySet[RecurringTask],
     ) -> None:
         queryset.update(is_locked=False)
+
+    @admin.action(description="Re-enable selected tasks")
+    def enable(
+        self,
+        request: HttpRequest,
+        queryset: QuerySet[RecurringTask],
+    ) -> None:
+        queryset.update(is_disabled=False, num_consecutive_failures=0)

{flagsmith_common-3.8.1 → flagsmith_common-3.9.0}/src/task_processor/exceptions.py

@@ -26,3 +26,13 @@ class TaskBackoffError(TaskProcessingError):
 
 class TaskQueueFullError(Exception):
     pass
+
+
+class TaskAbandonedError(TaskProcessingError):
+    """
+    Marker error for recurring task runs whose worker died before
+    recording the result (process killed, OOM, host evicted, DB
+    connection lost during the post-execution save). Never raised —
+    used as the prefix in `error_details` so monitoring and log scrapers
+    can match on a single authoritative class name.
+    """

{flagsmith_common-3.8.1 → flagsmith_common-3.9.0}/src/task_processor/managers.py

@@ -14,5 +14,8 @@ class TaskManager(Manager["Task"]):
 
 
 class RecurringTaskManager(Manager["RecurringTask"]):
-    def get_tasks_to_process(self) -> "RawQuerySet[RecurringTask]":
-        return self.raw("SELECT * FROM get_recurringtasks_to_process()")
+    def get_task_to_process(self) -> "RecurringTask | None":
+        return next(
+            iter(self.raw("SELECT * FROM get_recurringtasks_to_process()")),
+            None,
+        )

flagsmith_common-3.9.0/src/task_processor/migrations/0015_add_is_disabled.py

@@ -0,0 +1,37 @@
+import os
+
+from django.db import migrations, models
+
+from common.migrations.helpers import PostgresOnlyRunSQL
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("task_processor", "0014_add_trace_context"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="recurringtask",
+            name="is_disabled",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="recurringtask",
+            name="num_consecutive_failures",
+            field=models.IntegerField(default=0),
+        ),
+        PostgresOnlyRunSQL.from_sql_file(
+            os.path.join(
+                os.path.dirname(__file__),
+                "sql",
+                "0015_get_recurringtasks_to_process.sql",
+            ),
+            reverse_sql=os.path.join(
+                os.path.dirname(__file__),
+                "sql",
+                "0013_get_recurringtasks_to_process.sql",
+            ),
+        ),
+    ]

flagsmith_common-3.9.0/src/task_processor/migrations/sql/0015_get_recurringtasks_to_process.sql

@@ -0,0 +1,33 @@
+CREATE OR REPLACE FUNCTION get_recurringtasks_to_process()
+RETURNS SETOF task_processor_recurringtask AS $$
+DECLARE
+    row_to_return task_processor_recurringtask;
+BEGIN
+    -- Select the tasks that needs to be processed
+    FOR row_to_return IN
+        SELECT *
+        FROM task_processor_recurringtask
+        -- Skip disabled tasks; add one minute to the timeout as a grace period for overhead
+        WHERE is_disabled = FALSE
+          AND (is_locked = FALSE OR (locked_at IS NOT NULL AND locked_at < NOW() - timeout + INTERVAL '1 minute'))
+        ORDER BY last_picked_at NULLS FIRST
+        LIMIT 1
+        -- Select for update to ensure that no other workers can select these tasks while in this transaction block
+        FOR UPDATE SKIP LOCKED
+    LOOP
+        -- Lock every selected task(by updating `is_locked` to true)
+        UPDATE task_processor_recurringtask
+        -- Lock this row by setting is_locked True, so that no other workers can select these tasks after this
+        -- transaction is complete (but the tasks are still being executed by the current worker)
+        SET is_locked = TRUE, locked_at = NOW(), last_picked_at = NOW()
+        WHERE id = row_to_return.id;
+        -- If we don't explicitly update the columns here, the client will receive a row
+        -- that is locked but still shows `is_locked` as `False` and `locked_at` as `None`.
+        row_to_return.is_locked := TRUE;
+        row_to_return.locked_at := NOW();
+        RETURN NEXT row_to_return;
+    END LOOP;
+
+    RETURN;
+END;
+$$ LANGUAGE plpgsql

{flagsmith_common-3.8.1 → flagsmith_common-3.9.0}/src/task_processor/models.py

@@ -1,3 +1,4 @@
+import logging
 import typing
 import uuid
 from datetime import datetime, timedelta
@@ -7,11 +8,13 @@ from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 from django.utils import timezone
 
-from task_processor.exceptions import TaskQueueFullError
+from task_processor.exceptions import TaskAbandonedError, TaskQueueFullError
 from task_processor.managers import RecurringTaskManager, TaskManager
 from task_processor.task_registry import get_task, registered_tasks
 from task_processor.types import TaskCallable, TraceContext
 
+logger = logging.getLogger(__name__)
+
 _django_json_encoder_default = DjangoJSONEncoder().default
 
 
@@ -151,6 +154,8 @@ class Task(AbstractBaseTask):
 
 
 class RecurringTask(AbstractBaseTask):
+    MAX_CONSECUTIVE_FAILURES = 4
+
     run_every = models.DurationField()
     first_run_time = models.TimeField(blank=True, null=True)
 
@@ -158,6 +163,8 @@ class RecurringTask(AbstractBaseTask):
     timeout = models.DurationField(default=timedelta(minutes=30))
 
     last_picked_at = models.DateTimeField(blank=True, null=True)
+    is_disabled = models.BooleanField(default=False)
+    num_consecutive_failures = models.IntegerField(default=0)
     objects: RecurringTaskManager = RecurringTaskManager()
 
     class Meta:
@@ -172,6 +179,42 @@ class RecurringTask(AbstractBaseTask):
         self.is_locked = False
         self.locked_at = None
 
+    def reconcile_abandoned_run(self) -> None:
+        # if for some reason the worker died before before writing the task run result
+        # we mark that run as explict failure here
+        abandoned_run = self.task_runs.filter(result__isnull=True).first()
+        if abandoned_run is None:
+            return
+        abandoned_run.finished_at = timezone.now()
+        abandoned_run.result = TaskResult.FAILURE.value
+        abandoned_run.error_details = (
+            f"{TaskAbandonedError.__name__}: "
+            "no result was written before the SQL reaper unlocked the task"
+        )
+        abandoned_run.save(
+            update_fields=["finished_at", "result", "error_details"],
+        )
+        logger.error(
+            "Recurring task '%s' was abandoned: %s",
+            self.task_identifier,
+            abandoned_run.error_details,
+        )
+
+    def mark_failure(self) -> None:
+        super().mark_failure()
+        self.num_consecutive_failures += 1
+        if self.num_consecutive_failures >= self.MAX_CONSECUTIVE_FAILURES:
+            self.is_disabled = True
+            logger.error(
+                "Recurring task '%s' auto-disabled after %d consecutive failures",
+                self.task_identifier,
+                self.num_consecutive_failures,
+            )
+
+    def mark_success(self) -> None:
+        super().mark_success()
+        self.num_consecutive_failures = 0
+
     @property
     def should_execute(self) -> bool:
         now = timezone.now()

{flagsmith_common-3.8.1 → flagsmith_common-3.9.0}/src/task_processor/processor.py

@@ -7,6 +7,7 @@ from datetime import timedelta
 from importlib.metadata import version
 
 from django.conf import settings
+from django.db import close_old_connections
 from django.utils import timezone
 from opentelemetry import context as otel_context
 from opentelemetry import propagate, trace
@@ -68,52 +69,66 @@ def run_tasks(database: str, num_tasks: int = 1) -> list[TaskRun]:
     return []
 
 
-def run_recurring_tasks(database: str) -> list[RecurringTaskRun]:
+def run_recurring_task(database: str) -> RecurringTaskRun | None:
     # NOTE: We will probably see a lot of delay in the execution of recurring tasks
     # if the tasks take longer then `run_every` to execute. This is not
     # a problem for now, but we should be mindful of this limitation
     task_manager: RecurringTaskManager = RecurringTask.objects.db_manager(database)
-    tasks = task_manager.get_tasks_to_process()
-    if tasks:
-        logger.debug(f"Running {len(tasks)} recurring task(s)")
-
-        task_runs = []
-
-        for task in tasks:
-            if not task.is_task_registered:
-                # This is necessary to ensure that old instances of the task processor,
-                # which may still be running during deployment, do not remove tasks added by new instances.
-                # Reference: https://github.com/Flagsmith/flagsmith/issues/2551
-                task_age = timezone.now() - task.created_at
-                if task_age > UNREGISTERED_RECURRING_TASK_GRACE_PERIOD:
-                    task.delete(using=database)
-                continue
-
-            if task.should_execute:
-                task, task_run = _run_task(task)
-                assert isinstance(task_run, RecurringTaskRun)
-                task_runs.append(task_run)
-            else:
-                task.unlock()
-
-        # update all tasks that were not deleted
-        to_update = [task for task in tasks if task.id]
-        RecurringTask.objects.using(database).bulk_update(
-            to_update,
-            fields=["is_locked", "locked_at"],
-        )
+    task = task_manager.get_task_to_process()
+    if task is None:
+        return None
+
+    logger.debug(f"Running recurring task '{task.task_identifier}'")
+
+    task.reconcile_abandoned_run()
+
+    if not task.is_task_registered:
+        # This is necessary to ensure that old instances of the task processor,
+        # which may still be running during deployment, do not remove tasks added by new instances.
+        # Reference: https://github.com/Flagsmith/flagsmith/issues/2551
+        task_age = timezone.now() - task.created_at
+        if task_age > UNREGISTERED_RECURRING_TASK_GRACE_PERIOD:
+            task.delete(using=database)
+        return None
+
+    task_run: RecurringTaskRun | None = None
+    if task.should_execute:
+        # Persist the task run before execution so that, if the worker is
+        # killed mid-task, we still have a row we can later mark as timed
+        # out when the task is unlocked by the timeout-based reaper in
+        # `get_recurringtasks_to_process`.
+        task_run = RecurringTaskRun(started_at=timezone.now(), task=task)
+        task_run.save(using=database)
+        task, run = _run_task(task, task_run=task_run)
+        assert run is task_run
+        # task.run() may have idled the DB connection past the server's
+        # session timeout; drop stale connections so the saves below open
+        # a fresh one. See Sentry FLAGSMITH-API-5EM.
+        close_old_connections()
+    else:
+        task.unlock()
+
+    task.save(
+        using=database,
+        update_fields=[
+            "is_locked",
+            "locked_at",
+            "is_disabled",
+            "num_consecutive_failures",
+        ],
+    )
 
-        if task_runs:
-            RecurringTaskRun.objects.using(database).bulk_create(task_runs)
-            logger.debug(f"Finished running {len(task_runs)} recurring task(s)")
+    if task_run:
+        task_run.save(using=database)
+        logger.debug(f"Finished running recurring task '{task.task_identifier}'")
+        return task_run
 
-        return task_runs
-
-    return []
+    return None
 
 
 def _run_task(
     task: T,
+    task_run: AnyTaskRun | None = None,
 ) -> typing.Tuple[T, AnyTaskRun]:
     assert settings.TASK_PROCESSOR_MODE, (
         "Attempt to run tasks in a non-task-processor environment"
@@ -129,7 +144,8 @@ def _run_task(
     logger.debug(
         f"Running task {task_identifier} id={task.pk} args={task.args} kwargs={task.kwargs}"
     )
-    task_run: AnyTaskRun = task.task_runs.model(started_at=timezone.now(), task=task)  # type: ignore[attr-defined]
+    if task_run is None:
+        task_run = task.task_runs.model(started_at=timezone.now(), task=task)  # type: ignore[attr-defined]
     result: str
     executor = None
 

{flagsmith_common-3.8.1 → flagsmith_common-3.9.0}/src/task_processor/threads.py

@@ -8,7 +8,7 @@ from django.conf import settings
 from django.db import close_old_connections
 from django.utils import timezone
 
-from task_processor.processor import run_recurring_tasks, run_tasks
+from task_processor.processor import run_recurring_task, run_tasks
 from task_processor.task_registry import initialise
 from task_processor.types import TaskProcessorConfig
 
@@ -111,7 +111,7 @@ class TaskRunner(Thread):
 
                 # Recurring tasks are only run on one database
                 if (database == "default") ^ database_is_separate:
-                    run_recurring_tasks(database)
+                    run_recurring_task(database)
             except Exception as exception:
                 # To prevent task threads from dying if they get an error retrieving the tasks from the
                 # database this will allow the thread to continue trying to retrieve tasks if it can

flagsmith_common-3.8.1/src/common/core/templates/docgen-events.md

@@ -1,20 +0,0 @@
----
-title: Events
-sidebar_label: Events
-sidebar_position: 30
----
-
-Flagsmith backend emits [OpenTelemetry events](https://opentelemetry.io/docs/specs/otel/logs/data-model/#events)
-that can be ingested to downstream observability systems and/or a data warehouse of your choice via OTLP.
-To learn how to configure this, see [OpenTelemetry](deployment-self-hosting/scaling-and-performance/opentelemetry).
-
-## Event catalogue
-{% for event in flagsmith_events %}
-### `{{ event.name }}`
-
-Logged at `{{ event.level }}` from:
-{% for location in event.locations %} - `{{ location.path }}:{{ location.line }}`
-{% endfor %}
-Attributes:
-{% for attr in event.attributes %} - `{{ attr }}`
-{% endfor %}{% endfor %}

flagsmith_common-3.8.1/src/common/core/templates/docgen-metrics.md

@@ -1,24 +0,0 @@
----
-title: Metrics
-sidebar_label: Metrics
-sidebar_position: 20
----
-
-## Prometheus
-
-To enable the Prometheus `/metrics` endpoint, set the `PROMETHEUS_ENABLED` environment variable to `true`.
-
-When enabled, Flagsmith serves the `/metrics` endpoint on port 9100.
-
-The metrics provided by Flagsmith are described below.
-
-{% for metric in flagsmith_metrics %}
-### `{{ metric.name }}`
-
-{{ metric.type|title }}.
-
-{{ metric.documentation }}
-
-Labels:
-{% for label in metric.labels %} - `{{ label }}`
-{% endfor %}{% endfor %}