catalystwan 0.40.2.dev1__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

catalystwan/core/models/serialize.py

@@ -0,0 +1,137 @@
+from dataclasses import Field, fields, is_dataclass
+from typing import Any, Dict, List, Optional, Protocol, Union, cast
+
+from catalystwan.core.encoder import serialize_model_value
+from catalystwan.core.types import MODEL_TYPES, AliasPath, DataclassInstance, is_variable
+
+
+class ValueWrapperCallable(Protocol):
+    def __call__(self, field: Field, field_value: Any, to_json: bool): ...
+
+
+class ModelSerializer:
+    def __init__(self, model: DataclassInstance):
+        self.model = model
+        try:
+            self.model_type: MODEL_TYPES = model._catalystwan_model_type  # type: ignore[attr-defined]
+        except AttributeError:
+            self.model_type = "base"
+        # Values need to be wrapped in a specific form, depending on the model type
+        self.VALUE_WRAPPERS: Dict[MODEL_TYPES, ValueWrapperCallable] = {
+            "base": self.__value_wrapper_base,
+            "feature_template": self.__value_wrapper_ft,
+            "parcel": self.__value_wrapper_parcel,
+        }
+
+    def serialize(
+        self, by_alias: bool = True, to_json: bool = False, exclude_none: bool = True
+    ) -> Dict[str, Any]:
+        return_dict: Dict[str, Any] = {}
+        for field in fields(self.model):
+            return_value = getattr(self.model, field.name)
+            # skip None values, if needed
+            if exclude_none and return_value is None:
+                continue
+
+            return_value = self.__value_serializer(
+                field, return_value, by_alias, to_json, exclude_none
+            )
+            if by_alias:
+                alias: Optional[Union[str, AliasPath]] = field.metadata.get("alias")
+                field_name = alias or field.name
+                key_path = field_name if isinstance(field_name, AliasPath) else [field_name]
+                self.__insert_key(return_dict, key_path, return_value)
+            else:
+                return_dict[field.name] = return_value
+        return return_dict
+
+    def __value_serializer(
+        self,
+        field: Field,
+        field_value: Any,
+        by_alias: bool,
+        to_json: bool,
+        exclude_none: bool,
+    ) -> Any:
+        return_value: Any
+        if is_dataclass(field_value):
+            return_value = serialize(
+                model=cast(DataclassInstance, field_value),
+                by_alias=by_alias,
+                to_json=to_json,
+                exclude_none=exclude_none,
+            )
+        elif isinstance(field_value, list):
+            if len(field_value) == 0 or not is_dataclass(field_value[0]):
+                return_value = serialize_model_value(field_value, to_json)
+            else:
+                return_value = [
+                    serialize(
+                        model=v,
+                        by_alias=by_alias,
+                        to_json=to_json,
+                        exclude_none=exclude_none,
+                    )
+                    for v in field_value
+                ]
+        else:
+            return_value = serialize_model_value(field_value, to_json)
+
+        return self.__value_wrapper(field=field, field_value=return_value, to_json=to_json)
+
+    @property
+    def __value_wrapper(self) -> ValueWrapperCallable:
+        return self.VALUE_WRAPPERS[self.model_type]
+
+    def __value_wrapper_base(self, field: Field, field_value: Any, to_json: bool):
+        return field_value
+
+    def __value_wrapper_ft(self, field: Field, field_value: Any, to_json: bool) -> Any:
+        if not field.metadata.get("wrap", True) or not to_json:
+            return field_value
+        if is_variable(field_value):
+            return {
+                "vipType": "variable",
+                "vipValue": field_value,
+                "vipObjectType": field.metadata.get("object_type"),
+            }
+        else:
+            return {
+                "vipType": "constant",
+                "vipValue": field_value,
+                "vipObjectType": field.metadata.get("object_type"),
+            }
+
+    def __value_wrapper_parcel(self, field: Field, field_value: Any, to_json: bool) -> Any:
+        if (
+            not field.metadata.get("wrap", True)
+            or (isinstance(field_value, (list, dict)) or is_dataclass(field_value))
+            or not to_json
+        ):
+            return field_value
+        if is_variable(field_value):
+            return {"option_type": "variable", "value": field_value}
+        elif field_value == field.default:
+            return {"option_type": "default", "value": field_value}
+        else:
+            return {"option_type": "global", "value": field_value}
+
+    def __insert_key(self, d: Dict[str, Any], key_path: List[str], value: Any):
+        nested = d
+        for key in key_path[:-1]:
+            nested = nested.setdefault(key, {})
+        nested[key_path[-1]] = value
+
+
+def serialize(
+    model: DataclassInstance,
+    by_alias: bool = True,
+    to_json: bool = False,
+    exclude_none: bool = True,
+) -> Dict[str, Any]:
+    if not is_dataclass(model):
+        raise Exception
+
+    return ModelSerializer(model=model).serialize(
+        by_alias=by_alias, to_json=to_json, exclude_none=exclude_none
+    )

catalystwan/core/request_adapter.py

@@ -0,0 +1,246 @@
+from __future__ import annotations
+
+import logging
+from copy import copy
+from dataclasses import dataclass, field, fields, is_dataclass
+from string import Formatter
+from typing import Any, Dict, List, Optional, Type, TypeVar, Union, cast
+
+from catalystwan.abc import RequestAdapterInterface, ResponseInterface, SessionInterface
+from catalystwan.abc.types import HTTP_METHOD, JSON
+from catalystwan.core.exceptions import (
+    CatalystwanModelValidationError,
+    CatalystwanResponseTypeError,
+)
+from catalystwan.core.models.deserialize import deserialize
+from catalystwan.core.models.serialize import serialize
+from catalystwan.core.types import DataclassInstance
+from typing_extensions import get_args, get_origin
+
+DataclassType = TypeVar("DataclassType", bound=DataclassInstance)
+ReturnType = TypeVar("ReturnType")
+Payload = TypeVar("Payload")
+
+
+@dataclass
+class JsonContent:
+    # data extracted from a specific key
+    extracted_data: JSON
+    # raw version of the data
+    raw_data: JSON
+
+
+# Used for data extraction from a response, when it takes a specific form.
+# data_key is the key to pull data from, if required_keys are present
+@dataclass
+class ResponseDataPath:
+    data_key: str
+    required_keys: List[str] = field(default_factory=list)
+
+    def is_data_available(self, data: Dict[str, Any]) -> bool:
+        keys = self.required_keys + [self.data_key]
+        return all(key in data for key in keys)
+
+    def get_data(self, data: Dict[str, Any]) -> JSON:
+        return data[self.data_key]
+
+
+class RequestAdapter(RequestAdapterInterface):
+    known_data_paths: List[ResponseDataPath] = [
+        ResponseDataPath(data_key="data", required_keys=["header"]),
+        ResponseDataPath(data_key="devices", required_keys=["header"]),
+    ]
+
+    def __init__(self, session: SessionInterface, logger: Optional[logging.Logger] = None):
+        self.session = session
+        self.logger = logger or logging.getLogger(__name__)
+
+    def request(
+        self,
+        method: HTTP_METHOD,
+        url: str,
+        payload: Union[Payload, JSON] = None,
+        params: Optional[dict] = None,
+        return_type: Optional[Type[ReturnType]] = None,
+        headers: Optional[dict] = None,
+        **kwargs,
+    ) -> Union[ReturnType, JSON, bytes, None]:
+        if params:
+            path_keys = [
+                field_name
+                for _, field_name, _, _ in Formatter().parse(url)
+                if field_name is not None
+            ]
+            url = url.format_map({path_key: params.pop(path_key) for path_key in path_keys})
+        if is_dataclass(payload):
+            payload = serialize(cast(DataclassInstance, payload), to_json=True)
+        response = self.session.request(
+            method=method,
+            url=url,
+            json=payload,
+            params=params,
+            headers=headers,
+            **kwargs,
+        )
+
+        content = self.__get_content(response)
+        return self.__prepare_return_type(return_type, content)
+
+    def __prepare_return_type(
+        self, return_type: Optional[Type[ReturnType]], content: Union[JsonContent, str, bytes, None]
+    ) -> Union[ReturnType, JSON, bytes, None]:
+        if is_dataclass(return_type):
+            return cast(
+                ReturnType, self.__get_dataclass(return_type=return_type, model_payload=content)
+            )
+        # If we are expecting a list, it should be found within the extracted_data - the response was either a list from
+        # the beginning or we extracted it from an appropriate key
+        elif (
+            get_origin(return_type) is list
+            and type(content) is JsonContent
+            and type(content.extracted_data) is list
+        ):
+            return [
+                self.__parse_list(get_args(return_type)[0], value)
+                for value in content.extracted_data
+            ]
+        else:
+            extracted_data: Union[JSON, bytes, None]
+            if isinstance(content, JsonContent):
+                extracted_data = content.extracted_data
+            else:
+                extracted_data = content
+            if type(extracted_data) is not return_type and return_type is not None:
+                self.logger.warning(
+                    f"Server returned unexpected data. Expected: {return_type}, received {type(extracted_data)}"
+                )
+            return extracted_data
+
+    def __get_content(self, response: ResponseInterface) -> Union[JsonContent, str, bytes, None]:
+        content_type: str = response.headers.get("content-type", "")
+        if not content_type:
+            return None
+        if content_type.startswith("application/json"):
+            return self.__extract_json_data(response.json())
+        if content_type.startswith("text"):
+            return response.text
+        else:
+            return response.content
+
+    def __parse_list(
+        self, arg_type: Type[ReturnType], values: Union[JSON]
+    ) -> Union[ReturnType, JSON]:
+        if is_dataclass(arg_type):
+            model_payload = self.__extract_json_data(values)
+            return cast(
+                ReturnType, self.__get_dataclass(return_type=arg_type, model_payload=model_payload)
+            )
+        elif get_origin(arg_type) is list:
+            if not isinstance(values, list):
+                raise CatalystwanResponseTypeError(
+                    f"Expected type: list. Type received: {type(values)}"
+                )
+            return [self.__parse_list(get_args(arg_type)[0], value) for value in values]
+        else:
+            return values
+
+    def __extract_json_data(self, data: JSON) -> JsonContent:
+        if not isinstance(data, dict):
+            return JsonContent(extracted_data=data, raw_data=data)
+        for known_data_path in self.known_data_paths:
+            if known_data_path.is_data_available(data):
+                return JsonContent(extracted_data=known_data_path.get_data(data), raw_data=data)
+        return JsonContent(extracted_data=data, raw_data=data)
+
+    def __get_dataclass(
+        self, return_type: Type[DataclassType], model_payload: Union[JsonContent, str, bytes, None]
+    ) -> DataclassType:
+        # Get a dataclass using payload from a response.
+        # This method includes a bit of brute-force trickery, to fight inconsistent schemas.
+        # Sometimes the schema requires are to pull data from a specific key, while at other times it doesn't.
+        # This method tries to check which version - extracted or raw - matches better.
+        @dataclass
+        class ModelPayload:
+            name: str
+            data: JSON
+            priority: int
+
+        @dataclass
+        class ModelReturn:
+            model: DataclassType
+            payload: ModelPayload
+
+        if not isinstance(model_payload, JsonContent):
+            raise CatalystwanResponseTypeError(
+                f"Expected data for {return_type} model. Received data of type {type(model_payload)} instead."
+            )
+
+        data_sources = [
+            ModelPayload(data=model_payload.extracted_data, name="extracted_data", priority=1),
+            ModelPayload(data=model_payload.raw_data, name="raw_data", priority=0),
+        ]
+
+        field_names = [f.metadata.get("alias") or f.name for f in fields(return_type)]
+        # Empty dataclass, return as is
+        if not field_names:
+            return return_type()
+
+        valid_models: List[ModelReturn] = []
+        for data_source in data_sources:
+            try:
+                if isinstance(data_source.data, dict):
+                    valid_models.append(
+                        ModelReturn(
+                            model=deserialize(return_type, **data_source.data), payload=data_source
+                        )
+                    )
+                else:
+                    self.logger.debug(
+                        f"Failed to create model {return_type} using {data_source.name} source. Reason: data of type {type(data_source.data)} is invalid"
+                    )
+            except CatalystwanModelValidationError:
+                self.logger.debug(
+                    f"Failed to create model {return_type} using {data_source.name} source."
+                )
+
+        if not valid_models:
+            raise CatalystwanResponseTypeError(
+                f"Failed to create model {return_type} from given data."
+            )
+        if len(valid_models) == 1:
+            return valid_models[0].model
+
+        # return model that matches best with the input
+        valid_models.sort(
+            key=lambda x: (
+                self.__count_matching_keys(x.model, cast(dict, x.payload.data)),
+                x.payload.priority,
+            ),
+            reverse=True,
+        )
+        return valid_models[0].model
+
+    def __count_matching_keys(self, model: DataclassType, model_payload: dict):
+        matched_keys = 0
+        for key, value in model_payload.items():
+            try:
+                model_value = getattr(model, key)
+                matched_keys += 1
+                if is_dataclass(model_value) and isinstance(value, dict):
+                    matched_keys += self.__count_matching_keys(
+                        cast(DataclassType, model_value), value
+                    )
+                elif (
+                    isinstance(model_value, list)
+                    and all([is_dataclass(element) for element in model_value])
+                    and isinstance(value, list)
+                ):
+                    for model_v, input_v in zip(model_value, value):
+                        matched_keys += self.__count_matching_keys(model_v, input_v)
+            except AttributeError:
+                continue
+
+        return matched_keys
+
+    def __copy__(self) -> RequestAdapter:
+        return RequestAdapter(session=copy(self.session), logger=self.logger)

catalystwan/{request_limiter.py → core/request_limiter.py}

@@ -1,4 +1,3 @@
-# Copyright 2024 Cisco Systems, Inc. and its affiliates
 from __future__ import annotations
 
 from contextlib import AbstractContextManager
@@ -6,7 +5,7 @@ from threading import Semaphore
 
 
 class RequestLimiter(AbstractContextManager):
-    def __init__(self, max_requests: int = 49):
+    def __init__(self, max_requests: int = 10):
         self._max_requests: int = max_requests
         self._semaphore: Semaphore = Semaphore(value=self._max_requests)
 

catalystwan/{response.py → core/response.py}

@@ -1,32 +1,28 @@
-# Copyright 2023 Cisco Systems, Inc. and its affiliates
-
 import re
 from datetime import datetime
 from email.utils import parsedate_to_datetime
-from functools import wraps
 from os import environ
 from pprint import pformat
-from typing import Any, Callable, Dict, Optional, Sequence, Type, TypeVar, Union, cast
+from typing import Any, Dict, Optional, TypeVar, Union
 from urllib.parse import urlparse
 
-from pydantic import BaseModel
+from catalystwan.core.exceptions import ManagerErrorInfo
+from catalystwan.core.metadata import with_proc_info_header
 from requests import PreparedRequest, Request, Response
 from requests.cookies import RequestsCookieJar
 from requests.exceptions import JSONDecodeError
 
-from catalystwan import with_proc_info_header
-from catalystwan.abstractions import APIEndpointClientResponse
-from catalystwan.exceptions import ManagerErrorInfo
-from catalystwan.typed_list import DataSequence
-from catalystwan.utils.creation_tools import create_dataclass
-
 T = TypeVar("T")
-PRINTABLE_CONTENT = re.compile(r"(text\/.+)|(application\/(json|html|xhtml|xml|x-www-form-urlencoded))", re.IGNORECASE)
+PRINTABLE_CONTENT = re.compile(
+    r"(text\/.+)|(application\/(json|html|xhtml|xml|x-www-form-urlencoded))",
+    re.IGNORECASE,
+)
 SENSITIVE_URL_PATHS = ["/dataservice/settings/configuration/smartaccountcredentials"]
-APIGW_EXPIRED_TOKEN_MESSAGE = "token has invalid claims: token is expired"
 
 
-def response_debug(response: Optional[Response], request: Union[Request, PreparedRequest, None]) -> str:
+def response_debug(
+    response: Optional[Response], request: Union[Request, PreparedRequest, None]
+) -> str:
     """Returns human readable string containing Request-Response contents (helpful for debugging).
 
     Args:
@@ -87,7 +83,9 @@ def response_debug(response: Optional[Response], request: Union[Request, Prepare
 
 
 @with_proc_info_header
-def response_history_debug(response: Optional[Response], request: Union[Request, PreparedRequest, None]) -> str:
+def response_history_debug(
+    response: Optional[Response], request: Union[Request, PreparedRequest, None]
+) -> str:
     """Returns human readable string containing Request-Response history contents for given response.
 
     Args:
@@ -110,7 +108,11 @@ def auth_response_debug(response: Response, title: str = "Auth") -> str:
     if environ.get("catalystwan_auth_trace") is not None:
         return response_history_debug(response, None)
     return ", ".join(
-        [title] + [f"{r.request.method} {r.request.url} <{r.status_code}>" for r in response.history + [response]]
+        [title]
+        + [
+            f"{r.request.method} {r.request.url} <{r.status_code}>"
+            for r in response.history + [response]
+        ]
     )
 
 
@@ -142,14 +144,18 @@ class JsonPayload:
             self.headers = json.get("headers", None)
 
 
-class ManagerResponse(Response, APIEndpointClientResponse):
+class ManagerResponse(
+    Response
+    # ResponseInterface
+):
     """Extends Response object with methods specific to vManage.
     Object is meant to be created from aready received requests.Response"""
 
     def __init__(self, response: Response):
+        # @todo find better way to copy response
         self.__dict__.update(response.__dict__)
         self.jsessionid_expired = self._detect_expired_jsessionid()
-        self.api_gw_unauthorized = self._detect_expired_apigw_token()
+        self.api_gw_unauthorized = self._detect_apigw_unauthorized()
         try:
             self.payload = JsonPayload(response.json())
         except JSONDecodeError:
@@ -162,7 +168,11 @@ class ManagerResponse(Response, APIEndpointClientResponse):
             # get current server time, when not present use local time
             # local time might be innacurate but "Expires" is usually set to year 1970
             response_date = self.headers.get("date")
-            compare_date = parsedate_to_datetime(response_date) if response_date is not None else datetime.now()
+            compare_date = (
+                parsedate_to_datetime(response_date)
+                if response_date is not None
+                else datetime.now()
+            )
             if parsedate_to_datetime(expires) <= compare_date:
                 return True
         return False
@@ -174,8 +184,11 @@ class ManagerResponse(Response, APIEndpointClientResponse):
         jar.update(parse_cookies_to_dict(cookies_string))
         return jar
 
-    def _detect_expired_apigw_token(self) -> bool:
-        return self.status_code == 401 and self.json().get("message", "") == APIGW_EXPIRED_TOKEN_MESSAGE
+    def _detect_apigw_unauthorized(self) -> bool:
+        """Determines if server sent unauthorized response"""
+        return (
+            self.status_code == 401 and self.json().get("message", "") == "failed to validate user"
+        )
 
     def info(self, history: bool = False) -> str:
         """Returns human readable string containing Request-Response contents
@@ -189,72 +202,12 @@ class ManagerResponse(Response, APIEndpointClientResponse):
             return response_history_debug(self, None)
         return response_debug(self, None)
 
-    def dataseq(self, cls: Type[T], sourcekey: Optional[str] = "data", validate: bool = True) -> DataSequence[T]:
-        """Returns data contents from JSON payload parsed as DataSequence of Dataclass/BaseModel instances
-        Args:
-            cls: Dataclass/BaseModel subtype (eg. Devices)
-            sourcekey: name of the JSON key from response payload to be parsed. If None whole JSON payload will be used
-
-        Returns:
-            DataSequence[T] of given type T which is subclassing from Dataclass/BaseModel,
-            in case JSON payload was containing a single Object - sequence with one element is returned
-        """
-        if self.payload.empty:
-            return DataSequence(cls, [])
-
-        if sourcekey is None:
-            data = self.payload.json
-        else:
-            data = self.payload.json.get(sourcekey)
-
-        if isinstance(data, Sequence):
-            sequence = data
-        else:
-            sequence = [cast(dict, data)]
-
-        if issubclass(cls, BaseModel):
-            if validate:
-                return DataSequence(cls, [cls.model_validate(item) for item in sequence])  # type: ignore
-            return DataSequence(cls, [cls.model_construct(**item) for item in sequence])  # type: ignore
-        return DataSequence(cls, [create_dataclass(cls, item) for item in sequence])
-
-    def dataobj(self, cls: Type[T], sourcekey: Optional[str] = "data", validate: bool = True) -> T:
-        """Returns data contents from JSON payload parsed as Dataclass/BaseModel instance
-        Args:
-            cls: Dataclass/BaseModel subtype (eg. Devices)
-            sourcekey: name of the JSON key from response payload to be parsed. If None whole JSON payload will be used
-
-        Returns:
-            Object of given type T which is subclassing from Dataclass/BaseModel,
-
-        """
-        if sourcekey is None:
-            data = self.payload.json
-        else:
-            data = self.payload.json.get(sourcekey)
-
-        if issubclass(cls, BaseModel):
-            if validate:
-                return cls.model_validate(data)  # type: ignore[return-value]
-            return cls.model_construct(**data)  # type: ignore[return-value]
-        return create_dataclass(cls, data)
-
     def get_error_info(self) -> ManagerErrorInfo:
         """Returns error information from JSON payload"""
-
-        if self.payload.error is None:
-            return ManagerErrorInfo(
-                message=None,
-                details=None,
-                code=None,
-            )
-
-        return ManagerErrorInfo(**self.payload.error)
-
-
-def with_vmanage_response(method: Callable[[Any], Response]) -> Callable[[Any], ManagerResponse]:
-    @wraps(method)
-    def wrapper(*args, **kwargs) -> ManagerResponse:
-        return ManagerResponse(method(*args, **kwargs))
-
-    return wrapper
+        error = {} if self.payload.error is None else self.payload.error
+        return ManagerErrorInfo(
+            message=error.get("message"),
+            details=error.get("details"),
+            code=error.get("code"),
+            type=error.get("type"),
+        )