pycarlo 0.12.24__py3-none-any.whl

pycarlo/common/__init__.py

@@ -0,0 +1,31 @@
+import logging
+
+from pycarlo.common.files import BytesFileReader, JsonFileReader, TextFileReader
+from pycarlo.common.mcon import MCONParser, ParsedMCON
+from pycarlo.common.settings import MCD_VERBOSE_ERRORS
+
+
+def get_logger(name: str) -> logging.Logger:
+    """
+    Returns a logger with the specified name.
+
+    :param name: Name of the logger.
+    """
+
+    logger = logging.getLogger(name)
+    handler = logging.StreamHandler()
+    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    logger.setLevel(logging.DEBUG if MCD_VERBOSE_ERRORS else logging.CRITICAL)
+    return logger
+
+
+__all__ = [
+    "BytesFileReader",
+    "JsonFileReader",
+    "MCONParser",
+    "ParsedMCON",
+    "TextFileReader",
+    "get_logger",
+]

pycarlo/common/errors.py

@@ -0,0 +1,31 @@
+from typing import Dict, Mapping, Optional, Union
+
+
+class GqlError(Exception):
+    def __init__(
+        self,
+        body: Optional[Union[Dict, str]] = None,
+        headers: Optional[Mapping] = None,
+        message: str = "",
+        status_code: int = 0,
+        summary: str = "",
+        retryable: Optional[bool] = None,
+    ):
+        self.body = body
+        self.headers = headers
+        self.message = message
+        self.status_code = status_code
+        self.summary = summary
+        self.retryable = status_code >= 500 if retryable is None else retryable
+        super(GqlError, self).__init__()
+
+    def __str__(self) -> str:
+        return self.summary
+
+
+class InvalidSessionError(Exception):
+    pass
+
+
+class InvalidConfigFileError(Exception):
+    pass

pycarlo/common/files.py

@@ -0,0 +1,78 @@
+import json
+from pathlib import Path
+from typing import Callable, Dict, Generic, TypeVar, Union
+
+# file reader return type
+T = TypeVar("T")
+
+
+class FileReader(Generic[T]):
+    """
+    Utility for reading a local file. Return type is determined by the given `decoder` function.
+    """
+
+    def __init__(self, path: Union[Path, str], decoder: Callable[[bytes], T]):
+        """
+        :param path: local file path
+        :param decoder: function that translates file content as bytes into an object of type T
+        """
+        self._path = to_path(path)
+        self._decoder = decoder
+
+    def read(self) -> T:
+        """
+        Read local file.
+
+        :return: contents of file, represented as type T
+        """
+        with open(self._path, "rb") as file:
+            content = file.read()
+            return self._decoder(content)
+
+
+class BytesFileReader(FileReader[bytes]):
+    """
+    Utility for reading a local file as bytes.
+    """
+
+    def __init__(self, path: Union[Path, str]):
+        """
+        :param path: local file path
+        """
+        super().__init__(path=path, decoder=lambda b: b)
+
+
+class JsonFileReader(FileReader[Dict]):
+    """
+    Utility for reading a local JSON file as a dictionary.
+    """
+
+    def __init__(self, path: Union[Path, str]):
+        """
+        :param path: local file path
+        """
+        super().__init__(path=path, decoder=lambda b: json.loads(b))
+
+
+class TextFileReader(FileReader[str]):
+    """
+    Utility for reading a local file as a string.
+    """
+
+    def __init__(self, path: Union[Path, str], encoding: str = "utf-8"):
+        """
+        :param path: local file path
+        :param encoding: character encoding to use when translating file content to a string
+        """
+        super().__init__(path=path, decoder=lambda b: b.decode(encoding))
+
+
+def to_path(path: Union[Path, str]) -> Path:
+    """
+    Simple function to normalize a file path passed as either a string or pathlib.Path to a
+    pathlib.Path instance.
+
+    :param path: local file path
+    :return: local file path represented as an instance of pathlib.Path
+    """
+    return path if isinstance(path, Path) else Path(path)

pycarlo/common/http.py

@@ -0,0 +1,36 @@
+import json
+from typing import Dict, Union
+
+import requests
+
+from pycarlo.common.retries import ExponentialBackoffJitter, retry_with_backoff
+from pycarlo.common.settings import DEFAULT_RETRY_INITIAL_WAIT_TIME, DEFAULT_RETRY_MAX_WAIT_TIME
+
+
+@retry_with_backoff(
+    backoff=ExponentialBackoffJitter(DEFAULT_RETRY_INITIAL_WAIT_TIME, DEFAULT_RETRY_MAX_WAIT_TIME),
+    exceptions=(requests.exceptions.ConnectionError, requests.exceptions.Timeout),
+)
+def upload(
+    url: str,
+    content: Union[bytes, str, Dict],
+    method: str = "post",
+    encoding: str = "utf-8",
+):
+    """
+    Upload a file to a given URL.
+
+    :param url: URL to use for upload
+    :param content: file content
+    :param method: HTTP method (e.g. 'post' or 'put')
+    :param encoding: character encoding to use (unless raw bytes are provided)
+    """
+    if isinstance(content, str):
+        data = content.encode(encoding)
+    elif isinstance(content, dict):
+        data = json.dumps(content).encode(encoding)
+    else:
+        data = content
+
+    response = requests.request(method=method, url=url, data=data)
+    response.raise_for_status()

pycarlo/common/mcon.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass
+from typing import Optional
+from uuid import UUID
+
+
+@dataclass
+class ParsedMCON:
+    account_id: UUID
+    resource_id: UUID
+    object_type: str
+    object_id: str
+
+
+class MCONParser:
+    @staticmethod
+    def parse_mcon(mcon: Optional[str]) -> Optional[ParsedMCON]:
+        if not mcon or not mcon.startswith("MCON++"):
+            return None
+
+        mcon_parts = mcon.split("++", 4)
+        return ParsedMCON(
+            account_id=UUID(mcon_parts[1]),
+            resource_id=UUID(mcon_parts[2]),
+            object_type=mcon_parts[3],
+            object_id=mcon_parts[4],
+        )

pycarlo/common/retries.py

@@ -0,0 +1,129 @@
+import random
+import time
+from abc import ABC, abstractmethod
+from functools import wraps
+from typing import Any, Callable, Generator, Optional, Tuple, Type, Union
+
+
+class Backoff(ABC):
+    """
+    Backoff is an abstract class that dictates a retry strategy.
+    It contains an abstract method `backoff` that returns a calculated delay based on the provided
+    attempt.
+    """
+
+    def __init__(self, start: float, maximum: float):
+        """
+        Defines a new backoff retry strategy.
+
+        :param start: the scaling factor for any calculated delays.  Must be >= 0.
+        :param maximum: defines a cap on the calculated delays to prevent prohibitively long waits
+                        that could time out.
+        """
+        if start < 0:
+            raise ValueError("start must be >= 0")
+        self.start = start
+        self.maximum = maximum
+
+    @abstractmethod
+    def backoff(self, attempt: int) -> float:
+        pass
+
+    def delays(self) -> Generator[float, None, None]:
+        """
+        Generates a duration of time to delay for each successive call based on the configured
+        backoff strategy.
+
+        :return: a generator that yields the next delay duration.
+        """
+        retries = 0
+        max_retries = 100  # Safety limit to prevent infinite loops
+
+        while retries < max_retries:
+            duration = self.backoff(retries)
+            # duration might be 0 when start == 0.
+            # In that case, retry once immediately, and then on maximum.
+            if duration >= self.maximum or duration <= 0:
+                break
+            yield duration
+            retries += 1
+
+        yield self.maximum
+
+
+def retry_with_backoff(
+    backoff: "Backoff",
+    exceptions: Union[Type[Exception], Tuple[Type[Exception], ...]],
+    should_retry: Optional[Callable[[Exception], bool]] = None,
+) -> Callable[[Callable], Callable]:
+    """
+    A decorator to retry a function based on the :param:`backoff` provided
+    if any of the provided :param:`exceptions` are raised and the :param:`should_retry`
+    condition is met.
+
+    :param backoff: the retry strategy to employ.
+    :param exceptions: the exceptions that should trigger the retry. Can be further customized by
+                       defining a custom attribute `retryable` on the exception class. The retries
+                       are abandoned if retryable returns False.
+    :param should_retry: Optional callable to further determine whether to retry on an exception.
+                         Takes an exception and returns a boolean. If `None`, all given exceptions
+                          are retried.
+    :return: The same result the decorated function returns.
+    """
+
+    def _retry(func: Callable) -> Callable:
+        @wraps(func)
+        def _impl(*args: Any, **kwargs: Any) -> Any:
+            delays = backoff.delays()
+            while True:
+                try:
+                    return func(*args, **kwargs)
+                except exceptions as exception:
+                    # If the exception is marked as NOT retryable, stop now.
+                    retryable = getattr(exception, "retryable", True)
+                    if not retryable:
+                        raise exception
+                    # If a custom should_retry function is provided, call it to determine
+                    # if we should retry or not. Otherwise, default to the retryable value.
+                    current_should_retry = should_retry or (lambda _: retryable)
+                    if not current_should_retry(exception):
+                        raise exception
+                    try:
+                        delay = next(delays)
+                    except StopIteration:
+                        raise exception
+                    time.sleep(delay)
+
+        return _impl
+
+    return _retry
+
+
+class ExponentialBackoff(Backoff):
+    """
+    A backoff strategy with an exponentially increasing delay in between attempts.
+    """
+
+    def exponential(self, attempt: int) -> float:
+        # Prevent overflow by using float arithmetic and limiting attempt size
+        if attempt > 1000:  # Safety limit to prevent overflow
+            return self.maximum
+        return min(self.maximum, pow(2.0, float(attempt)) * self.start)
+
+    def backoff(self, attempt: int) -> float:
+        return self.exponential(attempt)
+
+
+class ExponentialBackoffJitter(ExponentialBackoff):
+    """
+    An exponential backoff strategy with an added jitter that randomly spreads out the delays
+    uniformly while ensuring monotonically increasing delays.
+    """
+
+    def backoff(self, attempt: int) -> float:
+        base_delay = self.exponential(max(0, attempt - 1))
+        added_delay = self.exponential(max(0, attempt)) - base_delay
+        # Add jitter between 50% and 100% of the base delay to ensure monotonically increasing
+        # delays while still providing randomization
+        jitter_factor = random.uniform(0.5, 1.0)
+        return base_delay + (added_delay * jitter_factor)

pycarlo/common/settings.py

@@ -0,0 +1,89 @@
+import os
+from enum import Enum
+from pathlib import Path
+
+"""Environmental configuration"""
+
+# Enable error logging.
+MCD_VERBOSE_ERRORS = os.getenv("MCD_VERBOSE_ERRORS", False) in (True, "true", "True")
+
+# MCD API endpoint.
+MCD_API_ENDPOINT = os.getenv("MCD_API_ENDPOINT")
+
+# Override MCD Default Profile when reading from the config-file in a session.
+MCD_DEFAULT_PROFILE = os.getenv("MCD_DEFAULT_PROFILE")
+
+# Override MCD API ID when creating a session.
+MCD_DEFAULT_API_ID = os.getenv("MCD_DEFAULT_API_ID")
+
+# Override MCD API Token when creating a session.
+MCD_DEFAULT_API_TOKEN = os.getenv("MCD_DEFAULT_API_TOKEN")
+
+# MCD ID header (for use in local development and testing)
+MCD_USER_ID_HEADER = os.getenv("MCD_USER_ID_HEADER")
+
+# dbt cloud API token
+DBT_CLOUD_API_TOKEN = os.getenv("DBT_CLOUD_API_TOKEN")
+
+# dbt cloud account ID
+DBT_CLOUD_ACCOUNT_ID = os.getenv("DBT_CLOUD_ACCOUNT_ID")
+
+"""Internal Use"""
+
+# Default API endpoint when not provided through env variable nor profile
+DEFAULT_MCD_API_ENDPOINT = "https://api.getmontecarlo.com/graphql"
+
+# Default Gateway endpoint used when no endpoint is provided through env var or profile
+DEFAULT_MCD_IGW_ENDPOINT = "https://integrations.getmontecarlo.com"
+
+# Name of the current package.
+DEFAULT_PACKAGE_NAME = "pycarlo"
+
+# Default config keys for the MC config file. Created via the CLI.
+DEFAULT_MCD_API_ID_CONFIG_KEY = "mcd_id"
+DEFAULT_MCD_API_TOKEN_CONFIG_KEY = "mcd_token"
+DEFAULT_MCD_API_ENDPOINT_CONFIG_KEY = "mcd_api_endpoint"
+
+# Default headers for the MC API.
+DEFAULT_MCD_API_ID_HEADER = f"x-{DEFAULT_MCD_API_ID_CONFIG_KEY.replace('_', '-')}"
+DEFAULT_MCD_API_TOKEN_HEADER = f"x-{DEFAULT_MCD_API_TOKEN_CONFIG_KEY.replace('_', '-')}"
+DEFAULT_MCD_USER_ID_HEADER = "user-id"
+
+# Default headers to trace and help identify requests. For debugging.
+DEFAULT_MCD_SESSION_ID = "x-mcd-session-id"  # Generally the session name.
+DEFAULT_MCD_TRACE_ID = "x-mcd-trace-id"
+
+# File name for profile configuration.
+PROFILE_FILE_NAME = "profiles.ini"
+
+# Default profile to be used.
+DEFAULT_PROFILE_NAME = "default"
+
+# Default path where any configuration files are written.
+DEFAULT_CONFIG_PATH = os.path.join(str(Path.home()), ".mcd")
+
+# Default initial wait time for retries in seconds.
+DEFAULT_RETRY_INITIAL_WAIT_TIME = 0.25
+
+# Default maximum wait time for retries in seconds.
+DEFAULT_RETRY_MAX_WAIT_TIME = 10.0
+
+# Default initial wait time for idempotent request retries in seconds.
+DEFAULT_IDEMPOTENT_RETRY_INITIAL_WAIT_TIME = 4.0
+
+# Default maximum wait time for idempotent request retries in seconds.
+DEFAULT_IDEMPOTENT_RETRY_MAX_WAIT_TIME = 4 * pow(
+    2, 4
+)  # retry 4 times, max wait 64 seconds, total wait 124
+
+# Default timeout for requests sent to Integration Gateway
+DEFAULT_IGW_TIMEOUT_SECS = 10
+
+# Additional request headers
+HEADER_MCD_TELEMETRY_REASON = "x-mcd-telemetry-reason"  # why the request was made
+HEADER_MCD_TELEMETRY_SERVICE = "x-mcd-telemetry-service"  # what service made the request
+
+
+class RequestReason(Enum):
+    USER = "user"  # request made directly by a user
+    SERVICE = "service"  # request made by a service, on behalf of a user or automation

pycarlo/common/utils.py

@@ -0,0 +1,51 @@
+from collections.abc import Mapping
+from functools import wraps
+from typing import Any, Callable, List, Optional
+
+from box import Box
+
+
+def chunks(lst: List, n: int):
+    """Yield successive n-sized chunks from lst."""
+    for i in range(0, len(lst), n):
+        yield lst[i : i + n]
+
+
+def boxify(
+    use_snakes: bool = False,
+    default_box_attr: Any = object(),
+    default_box: bool = False,
+) -> Callable:
+    """
+    Convenience decorator to convert a returned dictionary into a Box object for ease of use.
+
+    :param use_snakes: Convert camelCase into snake_case.
+    :param default_box_attr: Set default attribute to return.
+    :param default_box: Behave like a recursive default dict.
+    :return: Box object if original return was a Mapping (dictionary).
+    """
+
+    def _boxify(func: Callable):
+        @wraps(func)
+        def _impl(self: Any, *args: Any, **kwargs: Any) -> Optional[Box]:
+            dict_ = func(self, *args, **kwargs)
+            if dict_ and isinstance(dict_, Mapping):
+                return Box(
+                    dict_,
+                    camel_killer_box=use_snakes,
+                    default_box_attr=default_box_attr,
+                    default_box=default_box,
+                )
+            return dict_
+
+        return _impl
+
+    return _boxify
+
+
+def truncate_string(unicode_str: str, max_size: int) -> str:
+    str_bytes = unicode_str.encode("utf-8")
+    if len(str_bytes) < max_size:
+        return unicode_str  # keep the original string
+    else:
+        return str_bytes[:max_size].decode("utf-8", errors="ignore")

pycarlo/core/__init__.py

@@ -0,0 +1,10 @@
+from pycarlo.core.client import Client
+from pycarlo.core.operations import Mutation, Query
+from pycarlo.core.session import Session
+
+__all__ = [
+    "Client",
+    "Mutation",
+    "Query",
+    "Session",
+]