nautobot 2.0.0a3__py3-none-any.whl → 2.0.0b1__py3-none-any.whl

nautobot/core/api/metadata.py

@@ -1,21 +1,180 @@
+from typing import Any, Dict, List
+
 from django.core.exceptions import PermissionDenied
 from django.http import Http404
-from django.utils.encoding import force_str
+import drf_react_template.schema_form_encoder as schema
 from rest_framework import exceptions
+from rest_framework import fields as drf_fields
+from rest_framework import serializers as drf_serializers
 from rest_framework.metadata import SimpleMetadata
 from rest_framework.request import clone_request
 
-from nautobot.core.api import ContentTypeField
 
+# FIXME(jathan): I hate this pattern that these fields are hard-coded here. But for the moment, this
+# works reliably.
+BOTTOM_FIELDS = ["computed_fields", "custom_fields", "relationships"]
 
-class NautobotMetadata(SimpleMetadata):
-    def determine_actions(self, request, view):
+
+class NautobotProcessingMixin(schema.ProcessingMixin):
+    """Processing mixin to account for custom field types and behaviors for Nautobot."""
+
+    def _get_type_map_value(self, field: schema.SerializerType):
+        """Overload default to add "required" as a default mapping."""
+        # This adds "required" as a default type mapping compared to drf_react_template core.
+        result = {
+            "type": field.style.get("schema:type"),
+            "enum": field.style.get("schema:enum"),
+            "format": field.style.get("schema:format"),
+            "widget": field.style.get("ui:widget"),
+            "required": field.style.get("schema:required"),
+            "readOnly": field.style.get("schema:readOnly"),
+        }
+        result_default = self.TYPE_MAP.get(type(field).__name__, {})
+        for k in result_default:
+            if result[k] is None:
+                result[k] = result_default[k]
+        return result
+
+    def order_fields(self, fields):
+        """Explicitly order the "big ugly" fields to the bottom."""
+        # FIXME(jathan): Correct the behavior introduced in #3500 by switching to `__all__` to
+        # assert these get added at the end.
+        for field_name in BOTTOM_FIELDS:
+            if field_name in fields:
+                fields.remove(field_name)
+                fields.append(field_name)
+        return fields
+
+
+class NautobotSchemaProcessor(NautobotProcessingMixin, schema.SchemaProcessor):
+    """SchemaProcessor to account for custom field types and behaviors for Nautobot."""
+
+    def _get_field_properties(self, field: schema.SerializerType, name: str) -> Dict[str, Any]:
+        """
+        This method is used to generate the proper schema based on serializer field mappings or
+        per-field attribute markup.
+
+        This has been overloaded with an `elif` to account for `ManyRelatedField`.
+        """
+        result = {}
+        type_map_obj = self._get_type_map_value(field)
+        result["type"] = type_map_obj["type"]
+        result["title"] = self._get_title(field, name)
+
+        if isinstance(field, drf_serializers.ListField):
+            if field.allow_empty:
+                result["required"] = not getattr(field, "allow_empty", True)
+            result["items"] = self._get_field_properties(field.child, "")
+            result["uniqueItems"] = True
+        elif isinstance(field, drf_serializers.ManyRelatedField):
+            if field.allow_empty:
+                result["required"] = type_map_obj.get("required", [])
+            result["items"] = self._get_field_properties(field.child_relation, "")
+            result["uniqueItems"] = True
+        else:
+            if field.allow_null:
+                result["type"] = [result["type"], "null"]
+            enum = type_map_obj.get("enum")
+            if enum:
+                if enum == "choices":
+                    choices = field.choices
+                    result["enum"] = list(choices.keys())
+                    result["enumNames"] = list(choices.values())
+                if isinstance(enum, (list, tuple)):
+                    if isinstance(enum, (list, tuple)):
+                        result["enum"] = [item[0] for item in enum]
+                        result["enumNames"] = [item[1] for item in enum]
+                    else:
+                        result["enum"] = enum
+                        result["enumNames"] = list(enum)
+
+            # Process "format"
+            format_ = type_map_obj["format"]
+            if format_:
+                result["format"] = format_
+
+            # Process "readOnly"
+            read_only = type_map_obj["readOnly"]
+            if read_only:
+                result["readOnly"] = read_only
+
+            try:
+                result["default"] = field.get_default()
+            except drf_fields.SkipField:
+                pass
+
+        result = self._set_validation_properties(field, result)
+
+        return result
+
+
+class NautobotUiSchemaProcessor(NautobotProcessingMixin, schema.UiSchemaProcessor):
+    """UiSchemaProcessor to account for custom field types and behaviors for Nautobot."""
+
+    def _field_order(self) -> List[str]:
+        """
+        Overload the base which just returns `Meta.fields` and doesn't play nicely with "__all__".
+
+        This instead calls `get_fields()` and returns the keys.
+        """
+        if self._is_list_serializer(self.serializer):
+            fields = self.serializer.child.get_fields()
+        else:
+            fields = self.serializer.get_fields()
+
+        field_names = self.order_fields(list(fields))
+
+        return field_names
+
+    def _get_ui_field_properties(self, field: schema.SerializerType, name: str) -> Dict[str, Any]:
         """
-        Replace the stock determine_actions() method to assess object permissions only
-        when viewing a specific object. This is necessary to support OPTIONS requests
-        with bulk update in place (see #5470).
+        We had to overload this here to make it so that array types with children validate properly
+        and to also use `NautobotUiSchemaProcessor` over the default.
         """
-        actions = {}
+        data_index = self._generate_data_index(name)
+        result = {}
+        is_list = False
+        if self._is_field_serializer(field):
+            return NautobotUiSchemaProcessor(field, self.renderer_context, prefix=data_index).get_ui_schema()
+        elif isinstance(field, drf_serializers.ListField):
+            is_list = True
+            child = field.child
+            is_int = isinstance(child, drf_serializers.IntegerField)
+            widget = self._get_type_map_value(field=child).get("widget")
+            if not widget and isinstance(child, drf_serializers.ChoiceField):
+                widget = "checkbox"
+        else:
+            widget = self._get_type_map_value(field=field).get("widget")
+        help_text = field.help_text
+        if widget:
+            if is_list and is_int:
+                if "items" not in result:
+                    result["items"] = {}
+                result["items"]["ui:widget"] = widget
+            else:
+                result["ui:widget"] = widget
+        if help_text:
+            result["ui:help"] = help_text
+        result.update(self._get_style_dict(field))
+        result = self._set_validation_properties(field, result)
+        return result
+
+
+class NautobotColumnProcessor(NautobotProcessingMixin, schema.ColumnProcessor):
+    """ColumnProcessor to account for custom field types and behaviors for Nautobot."""
+
+
+class NautobotMetadata(SimpleMetadata):
+    """
+    Metadata class that emits JSON schema. It contains `schema` and `uiSchema` keys where:
+
+    - schema: The object JSON schema
+    - uiSchema: The object UI schema which describes the form layout in the UI
+    """
+
+    def determine_actions(self, request, view):
+        """Generate the actions and return the names of the allowed methods."""
+        actions = []
         for method in {"PUT", "POST"} & set(view.allowed_methods):
             view.request = clone_request(request, method)
             try:
@@ -28,32 +187,80 @@ class NautobotMetadata(SimpleMetadata):
             except (exceptions.APIException, PermissionDenied, Http404):
                 pass
             else:
-                # If user has appropriate permissions for the view, include
-                # appropriate metadata about the fields that should be supplied.
-                serializer = view.get_serializer()
-                actions[method] = self.get_serializer_info(serializer)
+                actions.append(method)
             finally:
                 view.request = request
-
         return actions
 
-    def get_field_info(self, field):
-        """
-        Fixup field information:
+    def get_list_display_fields(self, serializer):
+        """Try to get the list display fields or default to an empty list."""
+        serializer_meta = getattr(serializer, "Meta", None)
+        return list(getattr(serializer_meta, "list_display_fields", []))
 
-        - Set choices for ContentTypeField.
-        - Replace DRF choices `display_name` to `display` to match new pattern.
-        """
-        field_info = super().get_field_info(field)
-        for choice in field_info.get("choices", []):
-            choice["display"] = choice.pop("display_name")
-        if hasattr(field, "queryset") and not field_info.get("read_only") and isinstance(field, ContentTypeField):
-            field_info["choices"] = [
+    def determine_view_options(self, request, serializer):
+        """Determine view options that will be used for non-form display metadata."""
+        view_options = {}
+        list_display = []
+        fields = []
+
+        processor = NautobotColumnProcessor(serializer, request.parser_context)
+        field_map = dict(serializer.fields)
+        all_fields = list(field_map)
+
+        # Explicitly order the "big ugly" fields to the bottom.
+        processor.order_fields(all_fields)
+
+        list_display_fields = self.get_list_display_fields(serializer)
+
+        # Process the list_display fields first.
+        for field_name in list_display_fields:
+            try:
+                field = field_map[field_name]
+            except KeyError:
+                continue  # Ignore unknown fields.
+            column_data = processor._get_column_properties(field, field_name)
+            list_display.append(column_data)
+            fields.append(column_data)
+
+        # Process the rest of the fields second.
+        for field_name in all_fields:
+            # Don't process list display fields twice.
+            if field_name in list_display_fields:
+                continue
+            try:
+                field = field_map[field_name]
+            except KeyError:
+                continue  # Ignore unknown fields.
+            column_data = processor._get_column_properties(field, field_name)
+            fields.append(column_data)
+
+        view_options["list_display_fields"] = list_display
+        view_options["fields"] = fields
+
+        return view_options
+
+    def determine_metadata(self, request, view):
+        """This is the metadata that gets returned on an `OPTIONS` request."""
+        metadata = super().determine_metadata(request, view)
+
+        # If there's a serializer, do the needful to bind the schema/uiSchema.
+        if hasattr(view, "get_serializer"):
+            serializer = view.get_serializer()
+            # TODO(jathan): Bit of a WIP here. Will likely refactor. There might be cases where we
+            # want to explicitly override the UI field ordering, but that's not yet accounted for
+            # here. For now the assertion is always put the `list_display_fields` first, and then
+            # include the rest in whatever order.
+            # See: https://rjsf-team.github.io/react-jsonschema-form/docs/usage/objects#specifying-property-order
+            ui_schema = NautobotUiSchemaProcessor(serializer, request.parser_context).get_ui_schema()
+            ui_schema["ui:order"] = self.get_list_display_fields(serializer) + ["*"]
+            metadata.update(
                 {
-                    "value": choice_value,
-                    "display": force_str(choice_name, strings_only=True),
+                    "schema": NautobotSchemaProcessor(serializer, request.parser_context).get_schema(),
+                    # "uiSchema": NautobotUiSchemaProcessor(serializer, request.parser_context).get_ui_schema(),
+                    "uiSchema": ui_schema,
                 }
-                for choice_value, choice_name in field.choices.items()
-            ]
-            field_info["choices"].sort(key=lambda item: item["display"])
-        return field_info
+            )
+
+            metadata["view_options"] = self.determine_view_options(request, serializer)
+
+        return metadata

nautobot/core/api/mixins.py

@@ -1,3 +1,20 @@
+import logging
+import uuid
+
+from django.core.exceptions import (
+    FieldError,
+    MultipleObjectsReturned,
+    ObjectDoesNotExist,
+)
+from django.db.models import AutoField
+from rest_framework.exceptions import ValidationError
+
+from nautobot.core.api.utils import dict_to_filter_params
+from nautobot.core.utils.data import is_url
+
+logger = logging.getLogger(__name__)
+
+
 class LimitQuerysetChoicesSerializerMixin:
     """Mixin field that restricts queryset choices to those accessible
     for the queryset model that implemented it."""
@@ -12,3 +29,80 @@ class LimitQuerysetChoicesSerializerMixin:
         except AttributeError:
             model = self.parent.parent.Meta.model
         return queryset.get_for_model(model)
+
+
+class WritableSerializerMixin:
+    """
+    WritableSerializerMixin provides the to_internal_value() function.
+    The function provides the ability to write API requests that identify unique objects based on
+    combinations of fields other than the primary key.
+    e.g:
+    "parent": { "location_type__parent": {"name": "Campus"}, "parent__name": "Campus-29" }
+    vs
+    "parent": "10dff139-7333-46b0-bef6-f6a5a7b5497c"
+    """
+
+    def remove_non_filter_fields(self, filter_params):
+        """
+        Make output from a WritableSerializer "round-trip" capable by automatically stripping from the
+        data any serializer fields that do not correspond to a specific model field
+        """
+        if hasattr(self, "fields"):
+            for field_name, field_instance in self.fields.items():
+                if field_name in filter_params and field_instance.source == "*":
+                    logger.debug("Discarding non-filter field %s", field_name)
+                    del filter_params[field_name]
+        return filter_params
+
+    def get_queryset_filter_params(self, data, queryset):
+        """
+        Data could be a dictionary and an int (for the User model) or a str that represents the primary key.
+        If it is a dictionary, we return it after remove non-filter fields.
+        If it is a primary key, we return a dictionary object formatted like this {"pk": pk}
+        """
+        if isinstance(data, dict):
+            params = dict_to_filter_params(data)
+            return self.remove_non_filter_fields(params)
+
+        # Account for the fact that HyperlinkedIdentityFields might pass in URLs.
+        if is_url(data):
+            # Strip the trailing slash and split on slashes, taking the last value as the PK.
+            data = data.rstrip("/").split("/")[-1]
+
+        try:
+            # The int case here is taking into account for the User model we inherit from django
+            pk = int(data) if isinstance(queryset.model._meta.pk, AutoField) else uuid.UUID(str(data))
+        except (TypeError, ValueError) as e:
+            raise ValidationError(
+                "Related objects must be referenced by ID or by dictionary of attributes. Received an "
+                f"unrecognized value: {data}"
+            ) from e
+        return {"pk": pk}
+
+    def get_object(self, data, queryset):
+        """
+        Retrieve an unique object based on a dictionary of data attributes and raise errors accordingly if the object is not found.
+        """
+        filter_params = self.get_queryset_filter_params(data=data, queryset=queryset)
+        try:
+            return queryset.get(**filter_params)
+        except ObjectDoesNotExist as e:
+            raise ValidationError(f"Related object not found using the provided attributes: {filter_params}") from e
+        except MultipleObjectsReturned as e:
+            raise ValidationError(f"Multiple objects match the provided attributes: {filter_params}") from e
+        except FieldError as e:
+            raise ValidationError(e) from e
+
+    def to_internal_value(self, data):
+        """
+        Return an object or a list of objects based on a dictionary of data attributes or an UUID.
+        """
+        if data is None:
+            return None
+        if hasattr(self, "queryset"):
+            queryset = self.queryset
+        else:
+            queryset = self.Meta.model.objects
+        if isinstance(data, list):
+            return [self.get_object(data=entry, queryset=queryset) for entry in data]
+        return self.get_object(data=data, queryset=queryset)

nautobot/core/api/pagination.py

@@ -11,6 +11,10 @@ class OptionalLimitOffsetPagination(LimitOffsetPagination):
     """
 
     def paginate_queryset(self, queryset, request, view=None):
+        # No pagination when rendering to CSV
+        if "text/csv" in request.accepted_media_type:
+            return None
+
         self.count = self.get_count(queryset)
         self.limit = self.get_limit(request)
         self.offset = self.get_offset(request)

nautobot/core/api/parsers.py

@@ -0,0 +1,154 @@
+import csv
+from io import StringIO
+import json
+import logging
+
+from django.conf import settings
+
+from rest_framework import serializers
+from rest_framework.exceptions import ParseError
+from rest_framework.parsers import BaseParser
+
+from nautobot.core.models.utils import deconstruct_natural_key_slug
+
+
+logger = logging.getLogger(__name__)
+
+
+class NautobotCSVParser(BaseParser):
+    """Counterpart to NautobotCSVRenderer - import CSV data."""
+
+    media_type = "text/csv"
+
+    def parse(self, stream, media_type=None, parser_context=None):
+        parser_context = parser_context or {}
+        encoding = parser_context.get("encoding", "UTF-8")
+        try:
+            if "serializer_class" in parser_context:
+                # UI bulk-import case
+                serializer_class = parser_context["serializer_class"]
+            else:
+                # REST API case
+                serializer_class = parser_context["view"].get_serializer_class()
+        except (KeyError, AttributeError):
+            raise ParseError("No serializer_class was provided by the parser_context")
+        if serializer_class is None:
+            raise ParseError("Serializer class for this parser_context is None, unable to proceed")
+
+        serializer = serializer_class(context={"request": parser_context.get("request", None), "depth": 0})
+
+        try:
+            text = stream.read().decode(encoding)
+            reader = csv.DictReader(StringIO(text))
+
+            data = []
+            for counter, row in enumerate(reader, start=1):
+                data.append(self.row_elements_to_data(counter, row, serializer=serializer))
+
+            if "pk" in parser_context.get("kwargs", {}):
+                # Single-object update, not bulk update - strip it so that we get the expected input and return format
+                data = data[0]
+            # Note that we can't distinguish between single-create and bulk-create with a list of one object,
+            # as both would have the same CSV representation. Therefore create via CSV **always** acts as bulk-create,
+            # and the response will always be a list of created objects, never a single object
+
+            if settings.DEBUG:
+                logger.debug("CSV loaded into data:\n%s", json.dumps(data, indent=2))
+            return data
+        except ParseError:
+            raise
+        except Exception as exc:
+            raise ParseError(str(exc)) from exc
+
+    def row_elements_to_data(self, counter, row, serializer):
+        """
+        Parse a single row of CSV data (represented as a dict) into a dict suitable for consumption by the serializer.
+
+        TODO: it would be more elegant if our serializer fields knew how to deserialize the CSV data themselves;
+        could we then literally have the parser just return list(reader) and not need this function at all?
+        """
+        data = {}
+        for column, key in enumerate(row.keys(), start=1):
+            if not key:
+                raise ParseError(f"Row {counter}: Column {column}: missing/empty header for this column")
+
+            value = row[key]
+            if key.startswith("cf_"):
+                # Custom field
+                if value == "":
+                    value = None
+                data.setdefault("custom_fields", {})[key[3:]] = value
+                continue
+
+            serializer_field = serializer.fields.get(key, None)
+            if serializer_field is None:
+                # The REST API normally just ignores any columns the serializer doesn't understand
+                logger.debug('Skipping unknown column "%s"', key)
+                continue
+
+            if serializer_field.read_only and key != "id":
+                # Deserializing read-only fields is tricky, especially for things like SerializerMethodFields that
+                # can potentially render as anything. We don't strictly need such fields (except "id" for bulk PATCH),
+                # so let's just skip it.
+                continue
+
+            if isinstance(serializer_field, serializers.ManyRelatedField):
+                # A list of related objects, represented as a list of natural-key-slugs
+                if value:
+                    related_model = serializer_field.child_relation.get_queryset().model
+                    value = [self.get_natural_key_dict(slug, related_model) for slug in value.split(",")]
+                else:
+                    value = []
+            elif isinstance(serializer_field, serializers.RelatedField):
+                # A single related object, represented by its natural-key-slug
+                if value:
+                    related_model = serializer_field.get_queryset().model
+                    value = self.get_natural_key_dict(value, related_model)
+                else:
+                    value = None
+            elif isinstance(serializer_field, (serializers.ListField, serializers.MultipleChoiceField)):
+                if value:
+                    value = value.split(",")
+                else:
+                    value = []
+            elif isinstance(serializer_field, (serializers.DictField, serializers.JSONField)):
+                # We currently only store lists or dicts in JSONFields, never bare ints/strings.
+                # On the CSV write side, we only render dicts to JSON
+                if "{" in value or "[" in value:
+                    value = json.loads(value)
+                elif value:
+                    value = value.split(",")
+                    try:
+                        # We have some cases where it's a list of integers, such as in RackReservation.units
+                        value = [int(v) for v in value]
+                    except ValueError:
+                        # Guess not!
+                        pass
+
+            # CSV doesn't provide a ready distinction between blank and null, so in this case we have to pick one.
+            # This does mean that for a nullable AND blankable field, there's no way for CSV to set it to blank string.
+            # See corresponding logic in NautobotCSVRenderer.
+            if value == "" and serializer_field.allow_null:
+                value = None
+
+            data[key] = value
+
+        return data
+
+    def get_natural_key_dict(self, natural_key_slug, model):
+        """
+        Get the data dictionary corresponding to the given natural key list or string for the given model.
+        """
+        if not natural_key_slug:
+            return None
+        if model._meta.label_lower == "contenttypes.contenttype":
+            # Our ContentTypeField just uses the "app_label.model" string to look up ContentTypes, rather than the
+            # actual ([app_label, model]) natural key for ContentType.
+            return natural_key_slug
+        if model._meta.label_lower == "auth.group":
+            # auth.Group is a base Django model and so doesn't implement our natural_key_args_to_kwargs() method.
+            return {"name": deconstruct_natural_key_slug(natural_key_slug)}
+        if hasattr(model, "natural_key_args_to_kwargs"):
+            return model.natural_key_args_to_kwargs(deconstruct_natural_key_slug(natural_key_slug))
+        logger.error("%s doesn't implement natural_key_args_to_kwargs()", model.__name__)
+        return {"pk": natural_key_slug}