ipfabric_netbox 4.3.0b7__tar.gz → 4.3.0b8__tar.gz

{ipfabric_netbox-4.3.0b7 → ipfabric_netbox-4.3.0b8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ipfabric_netbox
-Version: 4.3.0b7
+Version: 4.3.0b8
 Summary: NetBox plugin to sync IP Fabric data into NetBox
 License: MIT
 Keywords: netbox,ipfabric,plugin,sync

{ipfabric_netbox-4.3.0b7 → ipfabric_netbox-4.3.0b8}/ipfabric_netbox/__init__.py

@@ -6,7 +6,7 @@ class NetboxIPFabricConfig(PluginConfig):
     name = "ipfabric_netbox"
     verbose_name = "NetBox IP Fabric SoT Plugin"
     description = "Sync IP Fabric into NetBox"
-    version = "4.3.0b7"
+    version = "4.3.0b8"
     base_url = "ipfabric"
     min_version = "4.4.0"
 

{ipfabric_netbox-4.3.0b7 → ipfabric_netbox-4.3.0b8}/ipfabric_netbox/jobs.py

@@ -4,7 +4,6 @@ from datetime import timedelta
 from core.choices import DataSourceStatusChoices
 from core.choices import JobStatusChoices
 from core.exceptions import SyncError
-from core.models import Job
 from netbox.context_managers import event_tracking
 from rq.timeouts import JobTimeoutException
 from utilities.datetime import local_now
@@ -36,15 +35,15 @@ def sync_ipfabricsource(job, *args, **kwargs):
 
 
 def sync_ipfabric(job, *args, **kwargs):
-    obj = IPFabricSync.objects.get(pk=job.object_id)
+    sync = IPFabricSync.objects.get(pk=job.object_id)
 
     try:
         job.start()
-        obj.sync(job=job)
+        sync.sync(job=job)
         job.terminate()
     except Exception as e:
         job.terminate(status=JobStatusChoices.STATUS_ERRORED)
-        IPFabricSync.objects.filter(pk=obj.pk).update(
+        IPFabricSync.objects.filter(pk=sync.pk).update(
             status=DataSourceStatusChoices.FAILED
         )
         if type(e) in (SyncError, JobTimeoutException):
@@ -52,15 +51,38 @@ def sync_ipfabric(job, *args, **kwargs):
         else:
             raise e
     finally:
-        if obj.interval and not kwargs.get("adhoc"):
-            new_scheduled_time = local_now() + timedelta(minutes=obj.interval)
-            job = Job.enqueue(
-                sync_ipfabric,
-                name=f"{obj.name} - (scheduled)",
-                instance=obj,
-                user=obj.user,
-                schedule_at=new_scheduled_time,
-                interval=obj.interval,
+        if sync.interval and not kwargs.get("adhoc"):
+            new_scheduled_time = local_now() + timedelta(minutes=sync.interval)
+            # We want to create new Job only if scheduled time was before this Job started
+            # The current sync might have been changed while this job was running
+            sync.refresh_from_db()
+            if not sync.scheduled or (sync.scheduled and sync.scheduled > job.started):
+                logger.info(
+                    f"Not scheduling a new job for IPFabricSync {sync.pk} as the scheduled time was changed while the job was running."
+                )
+                return
+            # Update the sync object with the new scheduled time
+            # This also triggers the creation of a new Job
+            # Running in fake request context to ensure user is set for changelog
+            request = NetBoxFakeRequest(
+                {
+                    "META": {},
+                    "POST": sync.parameters,
+                    "GET": {},
+                    "FILES": {},
+                    "user": sync.user,
+                    "path": "",
+                    "id": job.job_id,
+                }
+            )
+
+            with event_tracking(request):
+                sync.scheduled = new_scheduled_time
+                sync.status = DataSourceStatusChoices.QUEUED
+                sync.full_clean()
+                sync.save()
+            logger.info(
+                f"Scheduled next sync for IPFabricSync {sync.pk} at {new_scheduled_time}."
             )
 
 

ipfabric_netbox-4.3.0b8/ipfabric_netbox/migrations/0020_clean_scheduled_jobs.py

@@ -0,0 +1,70 @@
+from datetime import timedelta
+from typing import TYPE_CHECKING
+
+from django.db import migrations
+from utilities.datetime import local_now
+
+if TYPE_CHECKING:
+    from django.apps import apps as apps_type
+    from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+
+
+def clean_scheduled_jobs(
+    apps: "apps_type", schema_editor: "BaseDatabaseSchemaEditor"
+) -> None:
+    Job = apps.get_model("core", "Job")
+    IPFabricSync = apps.get_model("ipfabric_netbox", "IPFabricSync")
+    ObjectType = apps.get_model("core", "ObjectType")
+
+    for sync in IPFabricSync.objects.all():
+        try:
+            scheduled_jobs = Job.objects.filter(
+                object_id=sync.id,
+                object_type=ObjectType.objects.get_for_model(sync),
+                scheduled__isnull=False,
+            ).order_by("scheduled")
+            if not scheduled_jobs.exists():
+                continue
+            if not sync.scheduled:
+                # Delete all scheduled jobs if the sync is not scheduled
+                scheduled_jobs.delete()
+                continue
+            if scheduled_jobs.count() == 1:
+                # Only one scheduled job exists, let's update scheduled time on the sync object
+                # This does not create a new job since sync is a Faked object in migration
+                sync.scheduled = scheduled_jobs.first().scheduled
+                sync.full_clean()
+                sync.save()
+                continue
+            # More than one scheduled job exists
+            # Find the one that is closest to scheduled + N * interval
+            interval = timedelta(minutes=sync.interval)
+            elapsed = local_now() - sync.scheduled
+            intervals_passed = (elapsed // interval) + 1
+            closest_future_scheduled = sync.scheduled + intervals_passed * interval
+            closest_job = min(
+                scheduled_jobs,
+                key=lambda job: abs(job.scheduled - closest_future_scheduled),
+            )
+            for job in scheduled_jobs:
+                if job != closest_job:
+                    job.delete()
+            sync.scheduled = closest_job.scheduled
+            sync.full_clean()
+            sync.save()
+        except Exception as err:
+            # Always be safe inside a migration
+            print(f"Error cleaning scheduled jobs for IPFabricSync {sync.id}: {err}")
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        (
+            "ipfabric_netbox",
+            "0019_alter_ipfabrictransformmap_options_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.RunPython(clean_scheduled_jobs, migrations.RunPython.noop),
+    ]

{ipfabric_netbox-4.3.0b7 → ipfabric_netbox-4.3.0b8}/ipfabric_netbox/models.py

@@ -8,6 +8,7 @@ from uuid import uuid4
 
 import httpx
 from core.choices import DataSourceStatusChoices
+from core.choices import JobStatusChoices
 from core.exceptions import SyncError
 from core.models import Job
 from core.models import ObjectType
@@ -710,32 +711,56 @@ class IPFabricSync(IPFabricClient, JobsMixin, TagsMixin, ChangeLoggedModel):
             pk__in=[tm.pk for tm in maps_by_target.values()]
         )
 
-    def enqueue_sync_job(self, adhoc=False, user=None):
-        # Set the status to "syncing"
-        self.status = DataSourceStatusChoices.QUEUED
-        IPFabricSync.objects.filter(pk=self.pk).update(status=self.status)
-
-        Job.enqueue(
-            import_string("ipfabric_netbox.jobs.sync_ipfabricsource"),
-            name=f"{self.name} Snapshot Sync (Pre Ingestion)",
-            instance=self.snapshot_data.source,
-            user=self.user,
-        )
+    def delete_scheduled_jobs(self) -> None:
+        Job.objects.filter(
+            object_type=ObjectType.objects.get_for_model(self),
+            object_id=self.pk,
+            status__in=[
+                JobStatusChoices.STATUS_PENDING,
+                JobStatusChoices.STATUS_SCHEDULED,
+            ],
+        ).delete()
+
+    def enqueue_sync_job(self, adhoc=False, user=None) -> Job | None:
+        def set_syncing_status():
+            self.status = DataSourceStatusChoices.QUEUED
+            IPFabricSync.objects.filter(pk=self.pk).update(status=self.status)
+
+        def sync_snapshots():
+            Job.enqueue(
+                import_string("ipfabric_netbox.jobs.sync_ipfabricsource"),
+                name=f"{self.name} Snapshot Sync (Pre Ingestion)",
+                instance=self.snapshot_data.source,
+                user=self.user,
+            )
 
         # Enqueue a sync job
         if not user:
             user = self.user
 
-        if not adhoc and self.scheduled:
-            job = Job.enqueue(
-                import_string("ipfabric_netbox.jobs.sync_ipfabric"),
-                name=f"{self.name} - (scheduled)",
-                instance=self,
-                user=self.user,
-                schedule_at=self.scheduled,
-                interval=self.interval,
-            )
-        elif adhoc:
+        if not adhoc:
+            if self.scheduled:
+                # We want to schedule a recurring Job
+                # We need to replace the old scheduled Job to make sure it has current context
+                self.delete_scheduled_jobs()
+                set_syncing_status()
+                sync_snapshots()
+                job = Job.enqueue(
+                    import_string("ipfabric_netbox.jobs.sync_ipfabric"),
+                    name=f"{self.name} - (scheduled)",
+                    instance=self,
+                    user=self.user,
+                    schedule_at=self.scheduled,
+                    interval=self.interval,
+                )
+            else:
+                # There should be no scheduled Job anymore, clean it up
+                self.delete_scheduled_jobs()
+                job = None
+        else:
+            # Start adhoc job immediately
+            set_syncing_status()
+            sync_snapshots()
             job = Job.enqueue(
                 import_string("ipfabric_netbox.jobs.sync_ipfabric"),
                 instance=self,

{ipfabric_netbox-4.3.0b7 → ipfabric_netbox-4.3.0b8}/ipfabric_netbox/tests/test_views.py

@@ -719,7 +719,7 @@ class IPFabricSyncTestCase(
 
     @override_settings(EXEMPT_VIEW_PERMISSIONS=["*"])
     def test_get_htmx_request(self):
-        instance = self._get_queryset().first()
+        instance = self._get_queryset().last()
         # Try GET with HTMX
         response = self.client.get(
             instance.get_absolute_url(), **{"HTTP_HX-Request": "true"}

{ipfabric_netbox-4.3.0b7 → ipfabric_netbox-4.3.0b8}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ipfabric_netbox"
-version = "4.3.0b7"
+version = "4.3.0b8"
 description = "NetBox plugin to sync IP Fabric data into NetBox"
 authors = ["Solution Architecture <solution.architecture@ipfabric.io>"]
 license = "MIT"