nautobot 2.0.0a2__py3-none-any.whl → 2.0.0b1__py3-none-any.whl

nautobot/core/jobs/__init__.py

@@ -0,0 +1,75 @@
+from nautobot.core.celery import app, register_jobs
+from nautobot.extras.choices import LogLevelChoices
+from nautobot.extras.datasources import ensure_git_repository, git_repository_dry_run, refresh_datasource_content
+from nautobot.extras.jobs import Job, ObjectVar
+from nautobot.extras.models import GitRepository
+
+name = "System Jobs"
+
+
+class GitRepositorySync(Job):
+    """
+    System job to clone and/or pull a Git repository, then invoke `refresh_datasource_content()`.
+    """
+
+    repository = ObjectVar(
+        description="Git Repository to pull and refresh",
+        label="Git Repository",
+        model=GitRepository,
+    )
+
+    class Meta:
+        name = "Git Repository: Sync"
+        has_sensitive_variables = False
+
+    def run(self, repository):
+        job_result = self.job_result
+        user = job_result.user
+
+        job_result.log(
+            f'Creating/refreshing local copy of Git repository "{repository.name}"...',
+            logger=self.logger,
+        )
+
+        try:
+            ensure_git_repository(repository, job_result=job_result, logger=self.logger)
+            refresh_datasource_content("extras.gitrepository", repository, user, job_result, delete=False)
+            # Given that the above succeeded, tell all workers (including ourself) to call ensure_git_repository()
+            app.control.broadcast("refresh_git_repository", repository_pk=repository.pk, head=repository.current_head)
+        finally:
+            job_result.log(
+                f"Repository synchronization completed in {job_result.duration}",
+                level_choice=LogLevelChoices.LOG_INFO,
+                logger=self.logger,
+            )
+
+
+class GitRepositoryDryRun(Job):
+    """System Job to perform a dry run on a Git repository."""
+
+    repository = ObjectVar(
+        description="Git Repository to dry-run",
+        label="Git Repository",
+        model=GitRepository,
+    )
+
+    class Meta:
+        name = "Git Repository: Dry-Run"
+        has_sensitive_variables = False
+
+    def run(self, repository):
+        job_result = self.job_result
+        job_result.log(f'Performing a Dry Run on Git repository "{repository.name}"...', logger=self.logger)
+
+        try:
+            git_repository_dry_run(repository, job_result=job_result, logger=self.logger)
+        finally:
+            job_result.log(
+                f"Repository dry run completed in {job_result.duration}",
+                level_choice=LogLevelChoices.LOG_INFO,
+                logger=self.logger,
+            )
+
+
+jobs = [GitRepositorySync, GitRepositoryDryRun]
+register_jobs(*jobs)

nautobot/core/management/commands/build_ui.py

@@ -0,0 +1,255 @@
+import copy
+from importlib import import_module
+import json
+import os
+from pathlib import Path
+import re
+import shlex
+import subprocess
+
+from django.apps import apps
+from django.conf import settings
+from django.contrib.admindocs.views import simplify_regex
+from django.core.management.base import BaseCommand, CommandError
+import jinja2
+
+
+class Command(BaseCommand):
+    help = "Build the user interface for the Nautobot server environment and installed Nautobot Apps."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--npm-install",
+            action="store_true",
+            dest="npm_install",
+            default=False,
+            help="Install UI packages.",
+        )
+        parser.add_argument(
+            "--no-render-apps",
+            action="store_false",
+            dest="render_apps",
+            default=True,
+            help="Do not render Nautobot App imports.",
+        )
+        parser.add_argument(
+            "--no-npm-build",
+            action="store_false",
+            dest="npm_build",
+            default=True,
+            help="Do not compile UI.",
+        )
+
+    def convert_django_url_regex_to_react_route_path(self, regex):
+        """
+        Converts a regular expression object to its equivalent react router path representation.
+
+        Args:
+            regex (re.Pattern): A regular expression object.
+
+        Example:
+            >>> pattern = re.compile('^other-models/(?P<pk>[^/.]+)/notes/$')
+            >>> convert_django_url_regex_to_react_route_path(pattern)
+            '/other-models/:pk/notes/'
+        """
+        pattern = str(regex.pattern)
+        path = simplify_regex(pattern).replace("<", ":").replace(">", "").replace("\\Z", "").replace("\\", "")
+        return path if path.endswith("/") else f"{path}/"
+
+    def get_app_component(self, file_path, route_name):
+        """
+        Obtains the view component for the given route name from the index.js file of the App.
+        This locates the `route_name` in the app configuration `routes_view_components` dict and
+        returns the View Component registered for the `route_name`.
+
+        Parameters:
+            file_path (str): The path to the JavaScript file to read from.
+            route_name (str): The name of the route to extract the view component for.
+
+        Returns:
+            str: The view component associated with the specified route name, or None if not found.
+
+        Example:
+            >>> get_app_component(file_path="/src/example_plugin/example_plugin/ui/index.js", route_name="example-plugin:examplemodel_list")
+            "ExampleModelListView"
+        """
+        try:
+            with open(file_path, "r") as f:
+                js_content = f.read()
+        except FileNotFoundError:
+            return None
+
+        # Construct a regular expression that matches the specified key and extracts the associated view component
+        pattern = rf'routes_view_components:\s*{{[^}}]*"{re.escape(route_name)}"\s*:\s*"([^"]+)"'
+        view_component_match = re.search(pattern, js_content)
+        return view_component_match[1] if view_component_match else None
+
+    def render_routes_imports(self, app_base_path, app_name, app_config):
+        """
+        Renders the imports for the React Router components of the App's URLs.
+
+        This method inspects the `urlpatterns` list of the `urls.py` module of the App and generates
+        a list of dictionaries representing the React Router components and their corresponding URL paths.
+
+        Args:
+            app_base_path (Path): The base path of the Django app.
+            app_name (str): The name of the Django app.
+            app_config (AppConfig): The configuration object of the Django app.
+
+        Returns:
+            List[Dict[str, str]]: A list of dictionaries representing the React Router components and their
+            corresponding URL paths.
+            Each dictionary has the following keys:
+            - "path": The URL path pattern.
+            - "component": The React component associated with the URL path.
+
+        Example:
+            >>> render_routes_imports("/src/example_plugin/", "example_plugin", <AppConfig instance>)
+            [
+                { "path": "example-plugin/", "component": "HomeView"},
+                { "path": "example-plugin/config/", "component": "ConfigView"},
+            ]
+        """
+        data = []
+        try:
+            module = import_module(f"{app_name}.urls")
+            base_url = app_config.base_url or app_config.label
+            for urlpattern in module.urlpatterns:
+                if component := self.get_app_component(
+                    app_base_path / "ui/index.js",
+                    f"{base_url}:{urlpattern.name}",
+                ):
+                    path_regex = urlpattern.pattern.regex
+                    url_path = self.convert_django_url_regex_to_react_route_path(path_regex)
+                    data.append({"path": base_url + url_path, "component": component})
+            return data
+        except (AttributeError, ModuleNotFoundError):
+            # If an app does not include its url.py file or urls.py does not include a urlpatterns, skip.
+            return data
+
+    def render_app_imports(self):
+        """Render `app_imports.js`, update `jsconfig.json` to map to the path for each and render `app_routes.json` to register all app routes."""
+        self.stdout.write(self.style.WARNING(">>> Rendering Nautobot App imports..."))
+
+        ui_dir = settings.NAUTOBOT_UI_DIR
+
+        router_file_path = Path(ui_dir, "src", "router.js")
+        app_routes_file_path = Path(ui_dir, "src", "app_routes.json")
+        jsconfig_file_path = Path(ui_dir, "jsconfig.paths.json")
+        jsconfig_base_file_path = Path(ui_dir, "jsconfig-base.json")
+        app_routes = {}
+
+        with open(jsconfig_base_file_path, "r", encoding="utf-8") as base_config_file:
+            jsconfig = json.load(base_config_file)
+
+        # We're going to modify this list if apps don't have a `ui` directory.
+        enabled_apps = copy.copy(settings.PLUGINS)
+
+        for app_class_path in settings.PLUGINS:
+            app_name = app_class_path.split(".")[-1]
+            app_config = apps.get_app_config(app_name)
+            abs_app_path = Path(app_config.path).resolve()
+            abs_app_ui_path = abs_app_path / "ui"
+            app_path = Path(os.path.relpath(abs_app_path, ui_dir))
+            app_ui_path = app_path / "ui"
+            if app_routes_imports := self.render_routes_imports(abs_app_path, app_name, app_config):
+                app_routes[app_name] = app_routes_imports
+
+            # Assert that an App has a UI folder.
+            if not abs_app_ui_path.exists():
+                self.stdout.write(self.style.ERROR(f"- App {app_name!r} does not publish a UI; Skipping..."))
+                enabled_apps.remove(app_class_path)
+                continue
+            self.stdout.write(self.style.SUCCESS(f"- App {app_name!r} imported"))
+
+            jsconfig["compilerOptions"]["paths"][f"@{app_name}/*"] = [f"{app_ui_path}/*"]
+
+        with open(jsconfig_file_path, "w", encoding="utf-8") as generated_config_file:
+            json.dump(jsconfig, generated_config_file, indent=4)
+
+        with open(app_routes_file_path, "w", encoding="utf-8") as app_routes_file:
+            json.dump(app_routes, app_routes_file, indent=4)
+
+        app_imports_final_file_path = Path(ui_dir, "src", "app_imports.js")
+        environment = jinja2.sandbox.SandboxedEnvironment(loader=jinja2.FileSystemLoader(ui_dir))
+        template = environment.get_template("app_imports.js.j2")
+        content = template.render(apps=enabled_apps)
+
+        with open(app_imports_final_file_path, "w", encoding="utf-8") as generated_import_file:
+            generated_import_file.write(content)
+
+        # Touch the router to attempt to trigger a server reload.
+        Path(router_file_path).touch()
+
+    def run_command(self, command, message, cwd=settings.NAUTOBOT_UI_DIR):
+        """
+        Run a `command`, displaying `message` and exit. This splits it for you and runs it.
+
+        Args:
+            command (str): The command to execute
+            message (str): Message to display when command is executed
+        """
+        self.stdout.write(self.style.WARNING(message))
+        self.stdout.write(f"Running '{command}' in '{cwd}'...")
+
+        try:
+            result = subprocess.run(
+                shlex.split(command),
+                check=False,
+                cwd=cwd,
+                env={**os.environ.copy()},
+                encoding="utf-8",
+                capture_output=True,
+            )
+        except FileNotFoundError as err:
+            raise CommandError(f"'{command}' failed with error: {err}")
+
+        if result.returncode:
+            self.stdout.write(self.style.NOTICE(result.stdout))
+            self.stderr.write(self.style.ERROR(result.stderr))
+            raise CommandError(f"'{command}' failed with exit code {result.returncode}")
+
+    def handle(self, *args, **options):
+        verbosity = options["verbosity"]
+
+        # NAUTOBOT_DEBUG is also set here so that `console.log` isn't suppressed when we want
+        # verbosity.
+        if verbosity > 1:
+            os.environ["NAUTOBOT_DEBUG"] = "1"
+
+        verbosity_map = {
+            0: "silent",
+            1: "notice",
+            2: "info",
+            3: "silly",  # MAXIMUM OVERDEBUG
+        }
+        loglevel = f"--loglevel {verbosity_map[verbosity]}"
+
+        if not os.path.exists(Path(settings.NAUTOBOT_UI_DIR)):
+            os.makedirs(Path(settings.NAUTOBOT_UI_DIR))
+
+        self.run_command(
+            f"cp -r {Path(settings.BASE_DIR, 'ui')}/. {Path(settings.NAUTOBOT_UI_DIR)}",
+            ">>> Copying UI source files...",
+        )
+
+        # Generate `app_imports.js`
+        if options["render_apps"]:
+            self.render_app_imports()
+
+        # Run `npm install` and keep it silent by default.
+        if options["npm_install"]:
+            args = f"install {loglevel} --no-progress"
+            self.run_command(f"npm {args}", ">>> Installing Nautobot UI dependencies...")
+
+        # Run `npm build` and keep it silent by default.
+        if options["npm_build"]:
+            args = f"run build {loglevel}"
+            self.run_command(f"npm {args}", ">>> Compiling Nautobot UI packages...")
+            self.run_command(
+                f"cp {Path(settings.NAUTOBOT_UI_DIR, 'build' ,'asset-manifest.json')} "
+                f"{Path(settings.NAUTOBOT_UI_DIR, 'build' , 'static', 'asset-manifest.json')}",
+                ">>> Copying built manifest...",
+            )
+
+        self.stdout.write(self.style.SUCCESS(">>> Nautobot UI build complete! 🎉"))

nautobot/core/management/commands/celery.py

@@ -13,7 +13,6 @@ class Command(BaseCommand):
     """
 
     def run_from_argv(self, argv):
-
         # The "celery" command uses Click, which directly relies upon
         # `sys.argv`. So we must explicitly remove "celery" from `sys.argv` so
         # that we can directly invoke Click ourselves, letting it work with the

nautobot/core/management/commands/generate_test_data.py

@@ -37,7 +37,6 @@ class Command(BaseCommand):
         )
 
     def _generate_factory_data(self, seed):
-
         try:
             import factory.random
 
@@ -62,6 +61,7 @@ class Command(BaseCommand):
             )
             from nautobot.extras.utils import TaggableClassesQuery
             from nautobot.ipam.factory import (
+                NamespaceFactory,
                 PrefixFactory,
                 RIRFactory,
                 RouteTargetFactory,
@@ -101,6 +101,8 @@ class Command(BaseCommand):
         LocationFactory.create_batch(7, has_parent=True)
         LocationFactory.create_batch(40)
         LocationFactory.create_batch(10, has_parent=False)
+        self.stdout.write("Creating Namespaces...")
+        NamespaceFactory.create_batch(1)
         self.stdout.write("Creating RIRs...")
         RIRFactory.create_batch(9)  # only 9 unique RIR names are hard-coded presently
         self.stdout.write("Creating RouteTargets...")
@@ -115,27 +117,29 @@ class Command(BaseCommand):
         self.stdout.write("Creating Prefixes and IP Addresses...")
         PrefixFactory.create_batch(30)
         self.stdout.write("Creating Manufacturers...")
-        ManufacturerFactory.create_batch(10)  # First 10 hard-coded Manufacturers
+        ManufacturerFactory.create_batch(8)  # First 8 hard-coded Manufacturers
         self.stdout.write("Creating Platforms (with manufacturers)...")
         PlatformFactory.create_batch(20, has_manufacturer=True)
         self.stdout.write("Creating Platforms (without manufacturers)...")
         PlatformFactory.create_batch(5, has_manufacturer=False)
-        self.stdout.write("Creating Manufacturers without platforms...")
-        ManufacturerFactory.create_batch(4)  # Remaining 4 hard-coded Manufacturers
+        self.stdout.write("Creating Manufacturers without Platforms...")
+        ManufacturerFactory.create_batch(4)  # 4 more hard-coded Manufacturers
         self.stdout.write("Creating DeviceTypes...")
-        DeviceTypeFactory.create_batch(20)
+        DeviceTypeFactory.create_batch(30)
+        self.stdout.write("Creating Manufacturers without DeviceTypes or Platforms...")
+        ManufacturerFactory.create_batch(2)  # Last 2 hard-coded Manufacturers
         self.stdout.write("Creating DeviceRedundancyGroups...")
-        DeviceRedundancyGroupFactory.create_batch(10)
+        DeviceRedundancyGroupFactory.create_batch(20)
         self.stdout.write("Creating CircuitTypes...")
-        CircuitTypeFactory.create_batch(10)
+        CircuitTypeFactory.create_batch(20)
         self.stdout.write("Creating Providers...")
-        ProviderFactory.create_batch(10)
+        ProviderFactory.create_batch(20)
         self.stdout.write("Creating ProviderNetworks...")
-        ProviderNetworkFactory.create_batch(10)
+        ProviderNetworkFactory.create_batch(20)
         self.stdout.write("Creating Circuits...")
-        CircuitFactory.create_batch(20)
+        CircuitFactory.create_batch(40)
         self.stdout.write("Creating Providers without Circuits...")
-        ProviderFactory.create_batch(10)
+        ProviderFactory.create_batch(20)
         self.stdout.write("Creating CircuitTerminations...")
         CircuitTerminationFactory.create_batch(2, has_location=True, term_side="A")
         CircuitTerminationFactory.create_batch(2, has_location=True, term_side="Z")
@@ -181,15 +185,16 @@ Type 'yes' to continue, or 'no' to cancel: """
             call_command("flush", "--no-input")
 
         if options["cache_test_fixtures"] and os.path.exists(options["fixture_file"]):
+            self.stdout.write(self.style.WARNING(f"Loading factory data from file {options['fixture_file']}"))
             call_command("loaddata", options["fixture_file"])
         else:
             self._generate_factory_data(options["seed"])
 
             if options["cache_test_fixtures"]:
+                self.stdout.write(self.style.WARNING(f"Saving factory data to file {options['fixture_file']}"))
+
                 call_command(
                     "dumpdata",
-                    "--natural-foreign",
-                    "--natural-primary",
                     indent=2,
                     format="json",
                     exclude=["contenttypes", "auth.permission", "extras.job", "extras.customfield"],

nautobot/core/management/commands/post_upgrade.py

@@ -11,10 +11,10 @@ This will run the following management commands with default settings, in order:
 
 - migrate
 - trace_paths
+- build_ui --npm-install
 - collectstatic
 - remove_stale_contenttypes
 - clearsessions
-- invalidate all
 """
 
 
@@ -28,6 +28,13 @@ class Command(BaseCommand):
         return parser
 
     def add_arguments(self, parser):
+        parser.add_argument(
+            "--no-build-ui",
+            action="store_false",
+            dest="build_ui",
+            default=True,
+            help="Do not automatically build the user interface.",
+        )
         parser.add_argument(
             "--no-clearsessions",
             action="store_false",
@@ -42,13 +49,6 @@ class Command(BaseCommand):
             default=True,
             help="Do not automatically collect static files into a single location.",
         )
-        parser.add_argument(
-            "--no-invalidate-all",
-            action="store_false",
-            dest="invalidate_all",
-            default=True,
-            help="Do not automatically invalidate cache for entire application.",
-        )
         parser.add_argument(
             "--no-migrate",
             action="store_false",
@@ -74,41 +74,41 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         # Run migrate
         if options.get("migrate"):
-            print("Performing database migrations...")
+            self.stdout.write("Performing database migrations...")
             call_command(
                 "migrate",
                 interactive=False,
                 traceback=options["traceback"],
                 verbosity=options["verbosity"],
             )
-            print()
+            self.stdout.write()
 
         # Run trace_paths
         if options.get("trace_paths"):
-            print("Generating cable paths...")
+            self.stdout.write("Generating cable paths...")
             call_command("trace_paths", no_input=True)
-            print()
+            self.stdout.write()
+
+        # Run build
+        if options.get("build_ui"):
+            self.stdout.write("Building user interface...")
+            call_command("build_ui", npm_install=True)
+            self.stdout.write()
 
         # Run collectstatic
         if options.get("collectstatic"):
-            print("Collecting static files...")
+            self.stdout.write("Collecting static files...")
             call_command("collectstatic", interactive=False)
-            print()
+            self.stdout.write()
 
         # Run remove_stale_contenttypes
         if options.get("remove_stale_contenttypes"):
-            print("Removing stale content types...")
+            self.stdout.write("Removing stale content types...")
             call_command("remove_stale_contenttypes", interactive=False)
-            print()
+            self.stdout.write()
 
         # Run clearsessions
         if options.get("clearsessions"):
-            print("Removing expired sessions...")
+            self.stdout.write("Removing expired sessions...")
             call_command("clearsessions")
-            print()
-
-        # Run invalidate all
-        if options.get("invalidate_all"):
-            print("Invalidating cache...")
-            call_command("invalidate", "all")
-            print()
+            self.stdout.write()

nautobot/core/management/commands/validate_models.py

@@ -47,7 +47,6 @@ class Command(BaseCommand):
         return models
 
     def handle(self, *args, **options):
-
         models = self._get_models(args)
 
         if options["verbosity"]:

nautobot/core/middleware.py

@@ -115,7 +115,6 @@ class ExceptionHandlingMiddleware:
         return self.get_response(request)
 
     def process_exception(self, request, exception):
-
         # Don't catch exceptions when in debug mode
         if settings.DEBUG:
             return None

nautobot/core/models/__init__.py

@@ -1,11 +1,13 @@
 import uuid
 
 from django.db import models
+from django.urls import NoReverseMatch, reverse
 from django.utils.functional import classproperty
 
 from nautobot.core.models.managers import BaseManager
 from nautobot.core.models.querysets import RestrictedQuerySet
 from nautobot.core.models.utils import construct_natural_key_slug
+from nautobot.core.utils.lookup import get_route_for_model
 
 
 class BaseModel(models.Model):
@@ -37,6 +39,29 @@ class BaseModel(models.Model):
         """
         return not self._state.adding
 
+    def get_absolute_url(self, api=False):
+        """
+        Return the canonical URL for this object in either the UI or the REST API.
+        """
+
+        # Iterate the pk-like fields and try to get a URL, or return None.
+        fields = ["pk", "slug"]  # TODO: Eventually all PKs
+        actions = ["retrieve", "detail", ""]  # TODO: Eventually all retrieve
+
+        for field in fields:
+            if not hasattr(self, field):
+                continue
+
+            for action in actions:
+                route = get_route_for_model(self, action, api=api)
+
+                try:
+                    return reverse(route, kwargs={field: getattr(self, field)})
+                except NoReverseMatch:
+                    continue
+
+        return AttributeError(f"Cannot find a URL for {self} ({self._meta.app_label}.{self._meta.model_name})")
+
     class Meta:
         abstract = True
 
@@ -48,7 +73,7 @@ class BaseModel(models.Model):
         which in effect enforces model validation prior to saving the instance, without having
         to manually make these calls seperately. This is a slight departure from Django norms,
         but is intended to offer an optional, simplified interface for performing this common
-        workflow. The intended use is for user defined Jobs and scripts run via the `nautobot-server nbshell`
+        workflow. The intended use is for user defined Jobs run via the `nautobot-server nbshell`
         command.
         """
         self.full_clean()

nautobot/core/models/fields.py

@@ -8,9 +8,11 @@ from django.db import models
 from django.utils.text import slugify
 from django_extensions.db.fields import AutoSlugField as _AutoSlugField
 from netaddr import AddrFormatError, EUI, mac_unix_expanded
+from taggit.managers import TaggableManager
 
 from nautobot.core.forms import fields, widgets
 from nautobot.core.models import ordering
+from nautobot.core.models.managers import TagsManager
 
 
 class mac_unix_expanded_uppercase(mac_unix_expanded):
@@ -184,10 +186,6 @@ class ForeignKeyLimitedByContentTypes(ForeignKeyWithAutoRelatedName):
         - the content-type to which the field is attached (e.g. `dcim.device`)
     """
 
-    def __init__(self, **kwargs):
-        kwargs.setdefault("null", True)
-        super().__init__(**kwargs)
-
     def get_limit_choices_to(self):
         return {"content_types": ContentType.objects.get_for_model(self.model)}
 
@@ -323,7 +321,7 @@ class JSONArrayField(models.JSONField):
                 # Assume we're deserializing
                 vals = json.loads(value)
                 value = [self.base_field.to_python(val) for val in vals]
-            except json.JSONDecodeError as e:
+            except (TypeError, json.JSONDecodeError) as e:
                 raise exceptions.ValidationError(e)
         return value
 
@@ -370,3 +368,24 @@ class JSONArrayField(models.JSONField):
                 **kwargs,
             }
         )
+
+
+class TagsField(TaggableManager):
+    """Override FormField method on taggit.managers.TaggableManager to match the Nautobot UI."""
+
+    def __init__(self, *args, **kwargs):
+        from nautobot.extras.models.tags import TaggedItem
+
+        kwargs.setdefault("through", TaggedItem)
+        kwargs.setdefault("manager", TagsManager)
+        kwargs.setdefault("ordering", ["name"])
+        super().__init__(*args, **kwargs)
+
+    def formfield(self, form_class=fields.DynamicModelMultipleChoiceField, **kwargs):
+        from nautobot.extras.models.tags import Tag
+
+        queryset = Tag.objects.get_for_model(self.model)
+        kwargs.setdefault("queryset", queryset)
+        kwargs.setdefault("required", False)
+        kwargs.setdefault("query_params", {"content_types": self.model._meta.label_lower})
+        return super().formfield(form_class=form_class, **kwargs)