nautobot 2.0.5__py3-none-any.whl → 2.1.0b1__py3-none-any.whl

nautobot/extras/models/models.py

@@ -1,13 +1,15 @@
-import json
 from collections import OrderedDict
+import json
 
 from db_file_storage.model_utils import delete_file, delete_file_if_needed
 from db_file_storage.storage import DatabaseFileStorage
 from django.conf import settings
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
-from django.core.serializers.json import DjangoJSONEncoder
+from django.core.files.storage import get_storage_class
 from django.core.exceptions import ValidationError
+from django.core.serializers.json import DjangoJSONEncoder
+from django.core.validators import MinValueValidator
 from django.db import models
 from django.http import HttpResponse
 from graphene_django.settings import graphene_settings
@@ -20,7 +22,8 @@ from rest_framework.utils.encoders import JSONEncoder
 
 from nautobot.core.models import BaseManager, BaseModel
 from nautobot.core.models.fields import ForeignKeyWithAutoRelatedName
-from nautobot.core.models.generics import OrganizationalModel
+from nautobot.core.models.generics import OrganizationalModel, PrimaryModel
+from nautobot.core.models.validators import EnhancedURLValidator
 from nautobot.core.utils.data import deepmerge, render_jinja2
 from nautobot.extras.choices import (
     ButtonClassChoices,
@@ -450,6 +453,50 @@ class ExportTemplate(BaseModel, ChangeLoggedModel, RelationshipModel, NotesMixin
             self.file_extension = self.file_extension[1:]
 
 
+#
+# External integrations
+#
+
+
+class ExternalIntegration(PrimaryModel):
+    """Model for tracking integrations with external applications."""
+
+    name = models.CharField(max_length=255, unique=True)
+    remote_url = models.CharField(
+        max_length=500,
+        verbose_name="Remote URL",
+        validators=[EnhancedURLValidator()],
+    )
+    secrets_group = models.ForeignKey(
+        null=True,
+        blank=True,
+        to="extras.SecretsGroup",
+        on_delete=models.PROTECT,
+        help_text="Credentials used for authenticating with the remote system",
+    )
+    verify_ssl = models.BooleanField(
+        default=True,
+        verbose_name="Verify SSL",
+        help_text="Verify SSL certificates when connecting to the remote system",
+    )
+    timeout = models.IntegerField(
+        default=30,
+        validators=[MinValueValidator(0)],
+        help_text="Number of seconds to wait for a response",
+    )
+    extra_config = models.JSONField(
+        blank=True,
+        null=True,
+        help_text="Optional user-defined JSON data for this integration",
+    )
+
+    def __str__(self):
+        return f"{self.name} ({self.remote_url})"
+
+    class Meta:
+        ordering = ["name"]
+
+
 #
 # File attachments
 #
@@ -476,29 +523,55 @@ class FileAttachment(BaseModel):
 
 
 def database_storage():
-    """Returns storage backend used by `FileProxy.file` to store files in the database."""
+    """
+    Returns an instance of DatabaseFileStorage() unconditionally.
+
+    This is kept around to support legacy migrations; it shouldn't generally be used outside that.
+    Use `_job_storage()` instead.
+    """
     return DatabaseFileStorage()
 
 
+def _job_storage():
+    return get_storage_class(settings.JOB_FILE_IO_STORAGE)()
+
+
+def _upload_to(instance, filename):
+    """
+    Returns the upload path for attaching the given filename to the given FileProxy instance.
+
+    Because django-db-file-storage has specific requirements for this path to configure the FileAttachment model,
+    this needs to inspect which storage backend is in use in order to make the right determination.
+    """
+    if get_storage_class(settings.JOB_FILE_IO_STORAGE) == DatabaseFileStorage:
+        # must be a string of the form
+        # "<app_label>.<ModelName>/<data field name>/<filename field name>/<mimetype field name>/filename"
+        return f"extras.FileAttachment/bytes/filename/mimetype/{filename}"
+    else:
+        return f"files/{instance.pk}-{filename}"
+
+
 class FileProxy(BaseModel):
-    """An object to store a file in the database.
+    """A database object to reference and index a file, such as a Job input file or a Job output file.
+
+    The `file` field can be used like a file handle.
 
-    The `file` field can be used like a file handle. The file contents are stored and retrieved from
-    `FileAttachment` objects.
+    The file contents are stored and retrieved from `FileAttachment` objects,
+    if `settings.JOB_FILE_IO_STORAGE` is set to use DatabaseFileStorage,
+    otherwise they're written to whatever other file storage backend is in use.
 
-    The associated `FileAttachment` is removed when `delete()` is called. For this reason, one
-    should never use bulk delete operations on `FileProxy` objects, unless `FileAttachment` objects
-    are also bulk-deleted, because a model's `delete()` method is not called during bulk operations.
+    When using DatabaseFileStorage, the associated `FileAttachment` is removed when `delete()` is called.
+    For this reason, one should never use bulk delete operations on `FileProxy` objects,
+    unless `FileAttachment` objects are also bulk-deleted,
+    because a model's `delete()` method is not called during bulk operations.
     In most cases, it is better to iterate over a queryset of `FileProxy` objects and call
     `delete()` on each one individually.
     """
 
     name = models.CharField(max_length=255)
-    file = models.FileField(
-        upload_to="extras.FileAttachment/bytes/filename/mimetype",
-        storage=database_storage,  # Use only this backend
-    )
+    file = models.FileField(upload_to=_upload_to, storage=_job_storage)
     uploaded_at = models.DateTimeField(auto_now_add=True)
+    job_result = models.ForeignKey(to=JobResult, null=True, blank=True, on_delete=models.CASCADE, related_name="files")
 
     def __str__(self):
         return self.name
@@ -514,12 +587,19 @@ class FileProxy(BaseModel):
     natural_key_field_names = ["name", "uploaded_at"]
 
     def save(self, *args, **kwargs):
-        delete_file_if_needed(self, "file")
+        if get_storage_class(settings.JOB_FILE_IO_STORAGE) == DatabaseFileStorage:
+            delete_file_if_needed(self, "file")
+        else:
+            # TODO check whether there's an existing file with a different filename and delete it if so?
+            pass
         super().save(*args, **kwargs)
 
     def delete(self, *args, **kwargs):
+        if get_storage_class(settings.JOB_FILE_IO_STORAGE) != DatabaseFileStorage:
+            self.file.delete()
         super().delete(*args, **kwargs)
-        delete_file(self, "file")
+        if get_storage_class(settings.JOB_FILE_IO_STORAGE) == DatabaseFileStorage:
+            delete_file(self, "file")
 
 
 #

nautobot/extras/navigation.py

@@ -5,7 +5,6 @@ from nautobot.core.apps import (
     NavMenuAddButton,
     NavMenuGroup,
     NavMenuItem,
-    NavMenuImportButton,
     NavMenuTab,
 )
 
@@ -33,12 +32,6 @@ menu_items = (
                                     "extras.add_tag",
                                 ],
                             ),
-                            NavMenuImportButton(
-                                link="extras:tag_import",
-                                permissions=[
-                                    "extras.add_tag",
-                                ],
-                            ),
                         ),
                     ),
                 ),
@@ -61,12 +54,6 @@ menu_items = (
                                     "extras.add_status",
                                 ],
                             ),
-                            NavMenuImportButton(
-                                link="extras:status_import",
-                                permissions=[
-                                    "extras.add_status",
-                                ],
-                            ),
                         ),
                     ),
                 ),
@@ -89,12 +76,6 @@ menu_items = (
                                     "extras.add_role",
                                 ],
                             ),
-                            NavMenuImportButton(
-                                link="extras:role_import",
-                                permissions=[
-                                    "extras.add_role",
-                                ],
-                            ),
                         ),
                     ),
                 ),
@@ -136,10 +117,7 @@ menu_items = (
                         name="Secrets",
                         weight=100,
                         permissions=["extras.view_secret"],
-                        buttons=(
-                            NavMenuAddButton(link="extras:secret_add", permissions=["extras.add_secret"]),
-                            NavMenuImportButton(link="extras:secret_import", permissions=["extras.add_secret"]),
-                        ),
+                        buttons=(NavMenuAddButton(link="extras:secret_add", permissions=["extras.add_secret"]),),
                     ),
                     NavMenuItem(
                         link="extras:secretsgroup_list",
@@ -272,12 +250,6 @@ menu_items = (
                                     "extras.add_gitrepository",
                                 ],
                             ),
-                            NavMenuImportButton(
-                                link="extras:gitrepository_import",
-                                permissions=[
-                                    "extras.add_gitrepository",
-                                ],
-                            ),
                         ),
                     ),
                 ),
@@ -380,6 +352,22 @@ menu_items = (
                             ),
                         ),
                     ),
+                    NavMenuItem(
+                        link="extras:externalintegration_list",
+                        name="External Integrations",
+                        weight=300,
+                        permissions=[
+                            "extras.view_externalintegration",
+                        ],
+                        buttons=(
+                            NavMenuAddButton(
+                                link="extras:externalintegration_add",
+                                permissions=[
+                                    "extras.add_externalintegration",
+                                ],
+                            ),
+                        ),
+                    ),
                     NavMenuItem(
                         link="extras:webhook_list",
                         name="Webhooks",

nautobot/extras/signals.py

@@ -5,9 +5,13 @@ import shutil
 import logging
 from datetime import timedelta
 
+from db_file_storage.model_utils import delete_file
+from db_file_storage.storage import DatabaseFileStorage
+from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
 from django.core.cache import cache
 from django.core.exceptions import ValidationError
+from django.core.files.storage import get_storage_class
 from django.db import transaction
 from django.db.models.signals import m2m_changed, pre_delete, post_save, pre_save, post_delete
 from django.dispatch import receiver
@@ -346,6 +350,17 @@ post_save.connect(dynamic_group_update_cached_members, sender=DynamicGroupMember
 #
 
 
+@receiver(pre_delete, sender=JobResult)
+def job_result_delete_associated_files(instance, **kwargs):
+    """For each related FileProxy, make sure its file gets deleted correctly from disk or database."""
+    if get_storage_class(settings.JOB_FILE_IO_STORAGE) == DatabaseFileStorage:
+        for file_proxy in instance.files.all():
+            delete_file(file_proxy, "file")
+    else:
+        for file_proxy in instance.files.all():
+            file_proxy.file.delete()
+
+
 def refresh_job_models(sender, *, apps, **kwargs):
     """
     Callback for the nautobot_database_ready signal; updates Jobs in the database based on Job source file availability.

nautobot/extras/tables.py

@@ -26,6 +26,7 @@ from .models import (
     DynamicGroup,
     DynamicGroupMembership,
     ExportTemplate,
+    ExternalIntegration,
     GitRepository,
     GraphQLQuery,
     Job as JobModel,
@@ -407,6 +408,32 @@ class ExportTemplateTable(BaseTable):
         )
 
 
+class ExternalIntegrationTable(BaseTable):
+    pk = ToggleColumn()
+    name = tables.Column(linkify=True)
+    remote_url = tables.Column(linkify=False)
+    secrets_group = tables.Column(linkify=True)
+
+    class Meta(BaseTable.Meta):
+        model = ExternalIntegration
+        fields = (
+            "pk",
+            "name",
+            "remote_url",
+            "secrets_group",
+            "verify_ssl",
+            "timeout",
+        )
+        default_columns = (
+            "pk",
+            "name",
+            "remote_url",
+            "secrets_group",
+            "verify_ssl",
+            "timeout",
+        )
+
+
 class GitRepositoryTable(BaseTable):
     pk = ToggleColumn()
     name = tables.LinkColumn()

nautobot/extras/templates/extras/externalintegration_retrieve.html

@@ -0,0 +1,37 @@
+{% extends 'generic/object_retrieve.html' %}
+{% load helpers %}
+
+{% block content_left_page %}
+        <div class="panel panel-default">
+            <div class="panel-heading">
+                <strong>External Integration</strong>
+            </div>
+
+            <table class="table table-hover panel-body attr-table">
+                <tr>
+                    <td>Name</td>
+                    <td>{{ object.name }}</td>
+                </tr>
+                <tr>
+                    <td>Remote URL</td>
+                    <td>{{ object.remote_url }}</td>
+                </tr>
+                <tr>
+                    <td>Verify SSL</td>
+                    <td>{{ object.verify_ssl | render_boolean }}</td>
+                </tr>
+                <tr>
+                    <td>Secrets Group</td>
+                    <td>{{ object.secrets_group | hyperlinked_object }}</td>
+                </tr>
+                <tr>
+                    <td>Timeout</td>
+                    <td>{{ object.timeout }}</td>
+                </tr>
+                <tr>
+                    <td>Extra Config</td>
+                    <td><pre>{{ object.extra_config | render_json }}</pre></td>
+                </tr>
+            </table>
+        </div>
+{% endblock content_left_page %}

nautobot/extras/templates/extras/inc/jobresult.html

@@ -50,6 +50,30 @@
             {% endif %}
             </td>
         </tr>
+        {% if result.files.exists %}
+            <tr>
+                <td>File Output(s)</td>
+                <td>
+                    <ul>
+                        {% for file_proxy in result.files.all %}
+                            {% if file_proxy.file %}
+                                <li>
+                                    <a
+                                    {% if settings.JOB_FILE_IO_STORAGE == "db_file_storage.storage.DatabaseFileStorage" %}
+                                        href="{% url "db_file_storage.download_file" %}?name={{ file_proxy.file }}"
+                                    {% else %}
+                                        href="{{ file_proxy.file.url }}"
+                                    {% endif %}
+                                        download="{{ file_proxy.name }}">
+                                        {{ file_proxy.name }}
+                                    </a>
+                                </li>
+                            {% endif %}
+                        {% endfor %}
+                    </ul>
+                </td>
+            </tr>
+        {% endif %}
     </table>
 </div>
 

nautobot/extras/templates/extras/jobresult.html

@@ -118,6 +118,30 @@
                         {% include 'extras/inc/json_data.html' with data=result.result format="json" %}
                     </div>
                 </div>
+                <div class="panel panel-default">
+                    <div class="panel-heading">
+                        <strong>File Output(s)</strong>
+                    </div>
+                    <div class="panel-body">
+                        <ul>
+                            {% for file_proxy in result.files.all %}
+                                {% if file_proxy.file %}
+                                    <li>
+                                        <a
+                                        {% if settings.JOB_FILE_IO_STORAGE == "db_file_storage.storage.DatabaseFileStorage" %}
+                                            href="{% url "db_file_storage.download_file" %}?name={{ file_proxy.file }}"
+                                        {% else %}
+                                            href="{{ file_proxy.file.url }}"
+                                        {% endif %}
+                                            download="{{ file_proxy.name }}">
+                                            {{ file_proxy.name }}
+                                        </a>
+                                    </li>
+                                {% endif %}
+                            {% endfor %}
+                        </ul>
+                    </div>
+                </div>
                 <div class="panel panel-default">
                     <div class="panel-heading">
                         <strong>Traceback</strong>

nautobot/extras/test_jobs/file_output.py

@@ -0,0 +1,16 @@
+from nautobot.core.celery import register_jobs
+from nautobot.extras.jobs import IntegerVar, Job
+
+
+class FileOutputJob(Job):
+    lines = IntegerVar()
+
+    class Meta:
+        name = "File Output job"
+        description = "Creates a text file as output."
+
+    def run(self, lines):
+        self.create_file("output.txt", "Hello World!\n" * lines)
+
+
+register_jobs(FileOutputJob)

nautobot/extras/tests/test_api.py

@@ -48,6 +48,8 @@ from nautobot.extras.models import (
     DynamicGroup,
     DynamicGroupMembership,
     ExportTemplate,
+    ExternalIntegration,
+    FileProxy,
     GitRepository,
     GraphQLQuery,
     ImageAttachment,
@@ -813,6 +815,96 @@ class ExportTemplateTest(APIViewTestCases.APIViewTestCase):
         )
 
 
+class ExternalIntegrationTest(APIViewTestCases.APIViewTestCase):
+    model = ExternalIntegration
+    create_data = [
+        {
+            "name": "Test External Integration 1",
+            "remote_url": "ssh://example.com/test1/",
+            "verify_ssl": False,
+            "timeout": 5,
+            "extra_config": "{'foo': 'bar'}",
+        },
+        {
+            "name": "Test External Integration 2",
+            "remote_url": "http://example.com/test2/",
+        },
+        {
+            "name": "Test External Integration 3",
+            "remote_url": "https://example.com/test3/",
+            "verify_ssl": True,
+            "timeout": 30,
+            "extra_config": "{'foo': ['bat', 'baz']}",
+        },
+    ]
+    bulk_update_data = {"timeout": 10, "verify_ssl": True, "extra_config": r"{}"}
+
+
+class FileProxyTest(
+    APIViewTestCases.GetObjectViewTestCase,
+    APIViewTestCases.ListObjectsViewTestCase,
+):
+    model = FileProxy
+
+    @classmethod
+    def setUpTestData(cls):
+        job = Job.objects.first()
+        job_results = (
+            JobResult.objects.create(
+                job_model=job,
+                name=job.class_path,
+                date_done=now(),
+                status=JobResultStatusChoices.STATUS_SUCCESS,
+            ),
+            JobResult.objects.create(
+                job_model=job,
+                name=job.class_path,
+                date_done=now(),
+                status=JobResultStatusChoices.STATUS_SUCCESS,
+            ),
+            JobResult.objects.create(
+                job_model=job,
+                name=job.class_path,
+                date_done=now(),
+                status=JobResultStatusChoices.STATUS_SUCCESS,
+            ),
+        )
+        cls.file_proxies = []
+        for i, job_result in enumerate(job_results):
+            file = SimpleUploadedFile(name=f"Output {i}.txt", content=f"Content {i}\n".encode("utf-8"))
+            file_proxy = FileProxy.objects.create(name=file.name, file=file, job_result=job_result)
+            cls.file_proxies.append(file_proxy)
+
+    @override_settings(EXEMPT_VIEW_PERMISSIONS=[])
+    def test_download_file_without_permission(self):
+        """Test `download` action without permission."""
+        url = reverse("extras-api:fileproxy-download", kwargs={"pk": self.file_proxies[0].pk})
+        response = self.client.get(url, **self.header)
+        self.assertHttpStatus(response, status.HTTP_403_FORBIDDEN)
+
+    @override_settings(EXEMPT_VIEW_PERMISSIONS=[])
+    def test_download_file_with_permission(self):
+        """Test `download` action with permission."""
+        obj_perm = ObjectPermission(
+            name="Test permission", constraints={"pk": self.file_proxies[0].pk}, actions=["view"]
+        )
+        obj_perm.validated_save()
+        obj_perm.object_types.add(ContentType.objects.get_for_model(self.model))
+        obj_perm.users.add(self.user)
+
+        # FileProxy permitted by permission
+        url = reverse("extras-api:fileproxy-download", kwargs={"pk": self.file_proxies[0].pk})
+        response = self.client.get(url, **self.header)
+        self.assertHttpStatus(response, status.HTTP_200_OK)
+        content = b"".join(data for data in response)
+        self.assertEqual(content.decode("utf-8"), "Content 0\n")
+
+        # FileProxy not permitted by permission
+        url = reverse("extras-api:fileproxy-download", kwargs={"pk": self.file_proxies[1].pk})
+        response = self.client.get(url, **self.header)
+        self.assertHttpStatus(response, status.HTTP_404_NOT_FOUND)
+
+
 class GitRepositoryTest(APIViewTestCases.APIViewTestCase):
     model = GitRepository
     bulk_update_data = {

nautobot/extras/tests/test_filters.py

@@ -2,6 +2,7 @@ import uuid
 
 from django.contrib.auth import get_user_model
 from django.contrib.contenttypes.models import ContentType
+from django.core.files.uploadedfile import SimpleUploadedFile
 from django.db.models import Q
 from django.test import override_settings
 
@@ -33,6 +34,8 @@ from nautobot.extras.filters import (
     CustomFieldChoiceFilterSet,
     CustomLinkFilterSet,
     ExportTemplateFilterSet,
+    ExternalIntegrationFilterSet,
+    FileProxyFilterSet,
     GitRepositoryFilterSet,
     GraphQLQueryFilterSet,
     ImageAttachmentFilterSet,
@@ -59,6 +62,8 @@ from nautobot.extras.models import (
     CustomFieldChoice,
     CustomLink,
     ExportTemplate,
+    ExternalIntegration,
+    FileProxy,
     GitRepository,
     GraphQLQuery,
     ImageAttachment,
@@ -500,6 +505,63 @@ class ExportTemplateTestCase(FilterTestCases.FilterTestCase):
         self.assertEqual(self.filterset(params, self.queryset).qs.count(), 3)
 
 
+class FileProxyTestCase(FilterTestCases.FilterTestCase):
+    queryset = FileProxy.objects.all()
+    filterset = FileProxyFilterSet
+
+    generic_filter_tests = (
+        ["job", "job_result__job_model__id"],
+        ["job", "job_result__job_model__name"],
+        ["job_result_id"],
+        ["name"],
+        ["uploaded_at"],
+    )
+
+    @classmethod
+    def setUpTestData(cls):
+        jobs = Job.objects.all()[:3]
+        job_results = (JobResult.objects.create(job_model=job) for job in jobs)
+        for i, job_result in enumerate(job_results):
+            FileProxy.objects.create(
+                name=f"File {i}.txt", file=SimpleUploadedFile(name=f"File {i}.txt", content=b""), job_result=job_result
+            )
+
+
+class ExternalIntegrationTestCase(FilterTestCases.FilterTestCase):
+    queryset = ExternalIntegration.objects.all()
+    filterset = ExternalIntegrationFilterSet
+
+    generic_filter_tests = (
+        ["name"],
+        ["remote_url"],
+        ["timeout"],
+        ["secrets_group", "secrets_group__id"],
+        ["secrets_group", "secrets_group__name"],
+    )
+
+    @classmethod
+    def setUpTestData(cls):
+        secrets_groups = (
+            SecretsGroup.objects.create(name="Secrets Group 1"),
+            SecretsGroup.objects.create(name="Secrets Group 2"),
+        )
+        external_integrations = list(ExternalIntegration.objects.all()[:2])
+        external_integrations[0].secrets_group = secrets_groups[0]
+        external_integrations[1].secrets_group = secrets_groups[1]
+        for ei in external_integrations:
+            ei.validated_save()
+
+    def test_verify_ssl(self):
+        params = {"verify_ssl": True}
+        self.assertQuerysetEqualAndNotEmpty(
+            self.filterset(params, self.queryset).qs, self.queryset.filter(verify_ssl=True)
+        )
+        params = {"verify_ssl": False}
+        self.assertQuerysetEqualAndNotEmpty(
+            self.filterset(params, self.queryset).qs, self.queryset.filter(verify_ssl=False)
+        )
+
+
 class GitRepositoryTestCase(FilterTestCases.FilterTestCase):
     queryset = GitRepository.objects.all()
     filterset = GitRepositoryFilterSet
@@ -1170,7 +1232,7 @@ class RelationshipAssociationTestCase(FilterTestCases.FilterTestCase):
             ),
         )
         vlan_status = Status.objects.get_for_model(VLAN).first()
-        vlan_group = VLANGroup.objects.first()
+        vlan_group = VLANGroup.objects.create(name="Test VLANGroup 1")
         cls.vlans = (
             VLAN.objects.create(vid=1, name="VLAN 1", status=vlan_status, vlan_group=vlan_group),
             VLAN.objects.create(vid=2, name="VLAN 2", status=vlan_status, vlan_group=vlan_group),
@@ -1297,7 +1359,7 @@ class RelationshipModelFilterSetTestCase(FilterTestCases.FilterTestCase):
             ),
         )
         vlan_status = Status.objects.get_for_model(VLAN).first()
-        vlan_group = VLANGroup.objects.first()
+        vlan_group = VLANGroup.objects.create(name="Test VLANGroup 1")
         cls.vlans = (
             VLAN.objects.create(vid=1, name="VLAN 1", status=vlan_status, vlan_group=vlan_group),
             VLAN.objects.create(vid=2, name="VLAN 2", status=vlan_status, vlan_group=vlan_group),