cumulusci-plus 5.0.0__py3-none-any.whl

cumulusci/utils/yaml/cumulusci_yml.py

@@ -0,0 +1,401 @@
+"""
+Pydantic models for validating cumulusci.yml
+
+Note: If you change the model here, you should run `make schema`
+to update the JSON Schema version in cumulusci.jsonschema.json
+"""
+
+from logging import getLogger
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Sequence, Union
+
+from pydantic import Field, root_validator, validator
+from pydantic.types import DirectoryPath
+from typing_extensions import Literal, TypedDict
+
+from cumulusci.core.enums import StrEnum
+from cumulusci.utils.fileutils import DataInput, load_from_source
+from cumulusci.utils.yaml.model_parser import CCIDictModel, HashableBaseModel
+from cumulusci.utils.yaml.safer_loader import load_yaml_data
+
+default_logger = getLogger(__name__)
+
+
+#  type aliases
+PythonClassPath = str
+URL = str
+
+
+# additionalProperties here works around an
+# incompatibility with VSCode's Red Hat YAML validator
+# Can probably remove it at some future date if the
+# bug/incompatibilty is fixed elsewhere in the stack
+VSCodeFriendlyDict = Field({}, additionalProperties=True)
+
+
+class PreflightCheck(CCIDictModel):
+    when: str = None
+    action: str = None
+    message: str = None
+
+
+class Step(CCIDictModel):
+    task: str = None
+    flow: str = None
+    ignore_failure: bool = False
+    when: str = None  # is this allowed?
+    options: Dict[str, Any] = VSCodeFriendlyDict
+    ui_options: Dict[str, Any] = VSCodeFriendlyDict
+    checks: List[PreflightCheck] = []
+    description: str = None
+
+    @root_validator()
+    def _check(cls, values):
+        has_task = values.get("task") and values["task"] != "None"
+        has_flow = values.get("flow") and values["flow"] != "None"
+        assert not (
+            has_task and has_flow
+        ), "Steps must have either task or flow but not both"
+        return values
+
+
+class Task(CCIDictModel):
+    class_path: str = None
+    description: str = None
+    group: str = None
+    # additionalProperties here works around an
+    # incompatibility with VSCode's Red Hat YAML validator
+    options: Dict[str, Any] = VSCodeFriendlyDict
+    ui_options: Dict[str, Any] = VSCodeFriendlyDict
+    name: str = None  # get rid of this???
+
+
+class Flow(CCIDictModel):
+    description: str = None
+    steps: Dict[str, Step] = None
+    group: str = None
+
+
+class Package(CCIDictModel):
+    name: Optional[str] = None
+    name_managed: Optional[str] = None
+    namespace: Optional[str] = None
+    install_class: str = None
+    uninstall_class: str = None
+    api_version: str = None
+    metadata_package_id: str = None
+
+
+class Test(CCIDictModel):
+    name_match: str
+
+
+class ReleaseNotesParser(CCIDictModel):
+    class_path: PythonClassPath
+    title: str
+
+
+class ReleaseNotes(CCIDictModel):
+    parsers: Union[
+        Dict[str, Dict[int, ReleaseNotesParser]], Dict[int, ReleaseNotesParser]
+    ]
+
+
+class Git(CCIDictModel):
+    repo_url: str = None
+    default_branch: str = None
+    prefix_feature: str = None
+    prefix_beta: str = None
+    prefix_release: str = None
+    push_prefix_sandbox: str = None
+    push_prefix_production: str = None
+    release_notes: ReleaseNotes = None
+    two_gp_context: str = Field(None, alias="2gp_context")
+    unlocked_context: Optional[str] = None
+
+
+class Plan(CCIDictModel):  # MetaDeploy plans
+    title: str = None
+    description: str = None
+    tier: Literal["primary", "secondary", "additional"] = "primary"
+    slug: str = None
+    is_listed: bool = True
+    steps: Dict[str, Step] = None
+    checks: List[PreflightCheck] = []
+    error_message: str = None
+    post_install_message: str = None
+    preflight_message: str = None
+    allowed_org_providers: List[Literal["devhub", "user"]] = ["user"]
+
+
+class DependencyResolutions(CCIDictModel):
+    production: str = None
+    preproduction: str = None
+    resolution_strategies: Dict[str, List[str]] = None
+
+
+class Project(CCIDictModel):
+    name: Optional[str] = None
+    package: Optional[Package] = None
+    test: Optional[Test] = None
+    git: Optional[Git] = None
+    dependencies: Optional[List[Dict[str, str]]] = None
+    dependency_resolutions: Optional[DependencyResolutions] = None
+    dependency_pins: Optional[List[Dict[str, str]]]
+    source_format: Literal["sfdx", "mdapi"] = "mdapi"
+    custom: Optional[Dict] = None
+
+
+class ScratchOrg(CCIDictModel):
+    config_file: Path = None
+    days: int = None
+    namespaced: str = None
+    setup_flow: str = None
+    noancestors: bool = None
+    release: Literal["preview", "previous"] = None
+
+
+class Orgs(CCIDictModel):
+    scratch: Dict[str, ScratchOrg] = None
+
+
+class ServiceAttribute(CCIDictModel):
+    description: str = None
+    required: bool = None
+    default_factory: PythonClassPath = None
+    default: str = None
+    sensitive: bool = False
+
+
+class Service(CCIDictModel):
+    description: str = None
+    class_path: Optional[str]
+    attributes: Dict[str, ServiceAttribute] = None
+    validator: PythonClassPath = None
+
+
+class Plugin(CCIDictModel):
+    name: Optional[str]
+    version: Optional[str]
+    author: Optional[str]
+    description: Optional[str]
+    config: Dict[str, Any] = {}
+
+    @root_validator()
+    def _check_name_version(cls, values):
+        has_name = values.get("name") is not None
+        has_version = values.get("version") is not None
+
+        # If one is defined, both should be defined
+        if (has_name and not has_version) or (has_version and not has_name):
+            raise ValueError("Plugin Name and Version must be defined.")
+
+        return values
+
+
+class CumulusCIConfig(CCIDictModel):
+    keychain: PythonClassPath
+
+
+class VCSSourceRelease(StrEnum):
+    LATEST = "latest"
+    PREVIOUS = "previous"
+    LATEST_BETA = "latest_beta"
+
+
+class VCSSourceModel(HashableBaseModel):
+    vcs: str
+    url: str
+    resolution_strategy: Optional[str]
+    commit: Optional[str]
+    ref: Optional[str]
+    branch: Optional[str]
+    tag: Optional[str]
+    release: Optional[VCSSourceRelease]
+    description: Optional[str]
+    allow_remote_code: Optional[bool] = False
+
+    @root_validator
+    def validate(cls, values):
+        exclusive_keys = [
+            "resolution_strategy",
+            "commit",
+            "ref",
+            "branch",
+            "tag",
+            "release",
+        ]
+        key_count = len([x for x in exclusive_keys if x in values and values[x]])
+        if key_count > 1:
+            raise ValueError(
+                'Sources must use only one of "resolution_strategy", "commit", "ref", "branch", "tag", or "release".'
+            )
+        elif key_count == 0:
+            values["resolution_strategy"] = "production"
+
+        return values
+
+
+GitHubSourceRelease = VCSSourceRelease
+
+
+class GitHubSourceModel(VCSSourceModel):
+    """For backward compatibility."""
+
+    github: str
+    release: Optional[GitHubSourceRelease]
+    vcs: Optional[str]
+    url: Optional[str]
+
+    def __init__(self, **kwargs):
+        # For backward compatibility, we need to set the vcs and url attributes
+        # if they are not already set.
+        super().__init__(**kwargs)
+        self.vcs = "github"
+        self.url = kwargs.get("github", None)
+
+
+class LocalFolderSourceModel(HashableBaseModel):
+    path: DirectoryPath
+    allow_remote_code: Optional[bool] = False
+
+
+class CumulusCLIConfig(CCIDictModel):
+    show_stacktraces: bool = False
+    plain_output: bool = None
+
+
+class CumulusCIRoot(CCIDictModel):
+    tasks: Dict[str, Task] = {}
+    flows: Dict[str, Flow] = {}
+    project: Project = {}
+    orgs: Orgs = {}
+    services: Dict[str, Service] = {}
+    cumulusci: CumulusCIConfig = None
+    plans: Dict[str, Plan] = {}
+    minimum_cumulusci_version: str = None
+    sources: Dict[
+        str, Union[LocalFolderSourceModel, VCSSourceModel, GitHubSourceModel]
+    ] = {}
+    cli: CumulusCLIConfig = None
+    plugins: Dict[str, Plugin] = {}
+
+    @validator("plans")
+    def validate_plan_tiers(cls, plans):
+        existing_tiers = [plan.tier for plan in plans.values()]
+        has_duplicate_tiers = any(
+            existing_tiers.count(tier) > 1 for tier in ("primary", "secondary")
+        )
+        if has_duplicate_tiers:
+            raise ValueError("Only one plan can be defined as 'primary' or 'secondary'")
+        return plans
+
+
+class CumulusCIFile(CCIDictModel):
+    __root__: Union[CumulusCIRoot, None]
+
+
+def parse_from_yaml(source) -> dict:
+    "Parse from a path, url, path-like or file-like"
+    return CumulusCIFile.parse_from_yaml(source) or {}
+
+
+def validate_data(
+    data: Union[dict, list],
+    context: str = None,
+    on_error: callable = None,
+):
+    """Validate data which has already been loaded into a dictionary or list.
+
+    context is a string that will be used to give context to error messages.
+    on_error will be called for any validation errors with a dictionary in Pydantic error format
+
+    https://pydantic-docs.helpmanual.io/usage/models/#error-handling
+    """
+    return CumulusCIFile.validate_data(data, context=context, on_error=on_error)
+
+
+class ErrorDict(TypedDict):
+    "The structure of a Pydantic error dictionary. Google TypedDict if its new to you."
+    loc: Sequence[Union[str, int]]
+    msg: str
+    type: str
+
+
+def _log_yaml_errors(logger, errors: List[ErrorDict]):
+    "Format and log a Pydantic-style error dictionary"
+    # global has_shown_yaml_error_message
+    plural = "" if len(errors) <= 1 else "s"
+    logger.warning(f"CumulusCI Configuration Warning{plural}:")
+    for error in errors:
+        loc = " -> ".join(repr(x) for x in error["loc"] if x != "__root__")
+        logger.warning("  %s\n    %s", loc, error["msg"])
+
+
+def cci_safe_load(
+    source: DataInput, context: str = None, on_error: callable = None, logger=None
+) -> dict:
+    """Load a CumulusCI.yml file and issue warnings for unknown structures."""
+    errors = []
+    assert not (
+        on_error and logger
+    ), "Please specify either on_error or logger but not both"
+    on_error = on_error or errors.append
+
+    logger = logger or default_logger
+
+    with load_from_source(source) as (data_stream, filename):
+        data = load_yaml_data(data_stream, filename)
+        context = context or filename
+
+        try:
+            validate_data(data, context=context, on_error=on_error)
+            if errors:
+                _log_yaml_errors(logger, errors)
+        except Exception as e:
+            # should never be executed
+            print(f"Error validating cumulusci.yml {e}")
+            if on_error:
+                on_error(
+                    {
+                        "loc": (context,),
+                        "msg": f"Error validating cumulusci.yml {e}",
+                        "type": "exception",
+                    }
+                )
+            pass
+        return data or {}
+
+
+def _validate_files(globs):
+    "Validate YML files from Dev CLI for smoke testing"
+
+    from glob import glob
+
+    errors = []
+    for g in globs:
+        print(g)
+        filenames = glob(g, recursive=True)
+        for filename in filenames:
+            print("Validating", filename)
+
+            cci_safe_load(filename, filename, on_error=errors.append)
+    return errors
+
+
+def _validate_url(url):  # pragma: no cover
+    "Validate YML URL from Dev CLI for smoke testing"
+    errors = []
+    cci_safe_load(url, url, on_error=errors.append)
+    return errors
+
+
+# validate YML files as a CLI for smoke testing
+if __name__ == "__main__":  # pragma: no cover
+    import sys
+    from pprint import pprint
+
+    if sys.argv[1].startswith("http"):
+        pprint(_validate_url(sys.argv[1]))
+    else:
+        pprint(_validate_files(sys.argv[1:]))

cumulusci/utils/yaml/model_parser.py

@@ -0,0 +1,156 @@
+from pathlib import Path
+from typing import IO, Sequence, Union
+
+from pydantic import BaseModel, ValidationError
+from pydantic.error_wrappers import ErrorWrapper
+
+from cumulusci.utils.yaml.safer_loader import load_from_source, load_yaml_data
+
+
+class CCIModel(BaseModel):
+    # Base class for CumulusCI's Pydantic models
+
+    _magic_fields = ["fields"]
+
+    @classmethod
+    def parse_from_yaml(cls, source: Union[str, Path, IO]):
+        "Parse from a path, url, path-like or file-like"
+        with load_from_source(source) as (f, path):
+            data = load_yaml_data(f)
+            obj = cls.parse_obj(data, path)
+            return getattr(obj, "__root__", obj)
+
+    @classmethod
+    def parse_obj(cls, data: Union[dict, list], path: str = None):
+        "Parse a structured dict or list into Model objects"
+        try:
+            return super().parse_obj(data)
+        except ValidationError as e:
+            _add_filenames(e, path)
+            raise e
+
+    @classmethod
+    def validate_data(
+        cls, data: Union[dict, list], context: str = None, on_error: callable = None
+    ):
+        """Validate data which has already been loaded into a dictionary or list.
+
+        context is a string that will be used to give context to error messages.
+        on_error will be called for any validation errors with a dictionary in Pydantic error format
+
+        https://pydantic-docs.helpmanual.io/usage/models/#error-handling
+        """
+        try:
+            cls.parse_obj(data, context)
+            return True
+        except ValidationError as e:
+            if on_error:
+                for error in e.errors():
+                    on_error(error)
+
+            return False
+
+    def _alias_for_field(self, name):
+        "Find the name that we renamed a field to, to avoid Pydantic name clash"
+        for field in self.__fields__.values():
+            if field.alias == name:
+                return field.name
+
+    @property
+    def fields(self):
+        "Override deprecated Pydantic behaviour"
+        fields_alias = self._alias_for_field("fields")
+        if fields_alias:
+            return getattr(self, fields_alias)
+        else:
+            raise AttributeError
+
+    # @property.setter does not work because baseclass.__setattr__
+    # takes precedence (shrug emoji)
+    def __setattr__(self, name, value):
+        "Override deprecated Pydantic behaviour for .fields"
+        if name in self._magic_fields:
+            name = self._alias_for_field(name)
+        return super().__setattr__(name, value)
+
+    def copy(self, *args, **kwargs):
+        """Copy with a default behaviour similar to Python's.
+
+        If you supply arguments, you get Pydantic copy behaviour.
+        https://pydantic-docs.helpmanual.io/usage/exporting_models/#modelcopy
+        """
+        if not args and not kwargs:
+            return self.__class__(**self.__dict__)
+        else:
+            return super().copy(self, *args, **kwargs)
+
+    class Config:
+        """Pydantic Config
+
+        If you replace this config class in a sub-class, make sure you replace
+        the parameter below too.
+
+        https://pydantic-docs.helpmanual.io/usage/model_config/
+        """
+
+        extra = "forbid"
+
+
+class CCIDictModel(CCIModel):
+    # A base class that acts as both a model and a dict. For transitioning from
+    # one to the other.
+
+    def __getitem__(self, name):
+        """Pretend to a do my_dict[name]"""
+        try:
+            return getattr(self, name)
+        except AttributeError:
+            raise IndexError(name)
+
+    def __setitem__(self, name, value):
+        """Pretend to a do my_dict[name] = X"""
+        setattr(self, name, value)
+
+    def __contains__(self, name):
+        "Support 'x in my_dict' syntax."
+        if name not in self._magic_fields:
+            return name in self.__dict__
+        else:
+            return self._alias_for_field(name) in self.__dict__
+
+    def get(self, name, default=None):
+        "Emulate dict.get()."
+        if name in self._magic_fields:
+            name = self._alias_for_field(name)
+
+        return self.__dict__.get(name, default)
+
+    def __delitem__(self, name):
+        if name in self._magic_fields:
+            name = self._alias_for_field(name)
+        del self.__dict__[name]
+
+
+def _add_filenames(e: ValidationError, filename):
+    def _recursively_add_filenames(val):
+        processed = False
+        if isinstance(val, Sequence):
+            for e in val:
+                _recursively_add_filenames(e)
+            processed = True
+        elif isinstance(val, ErrorWrapper):
+            val._loc = (filename, val._loc)
+
+            processed = True
+        assert processed, f"Should have processed by now {val}, {repr(val)}"
+
+    _recursively_add_filenames(e.raw_errors)
+
+
+class HashableBaseModel(CCIModel):
+    # Base Pydantic model class that has a functional `hash()` method.
+    # Requires that model can be converted to JSON.
+
+    # See https://github.com/samuelcolvin/pydantic/issues/1303
+    def __hash__(self):
+        return hash((type(self),) + tuple(self.json()))

cumulusci/utils/yaml/safer_loader.py

@@ -0,0 +1,74 @@
+import re
+import typing as T
+from io import StringIO
+from logging import getLogger
+from pathlib import Path
+
+import yaml
+from yaml.error import MarkedYAMLError
+
+from cumulusci.core.exceptions import YAMLParseException
+from cumulusci.utils.fileutils import FSResource, load_from_source
+
+NBSP = "\u00A0"
+
+pattern = re.compile(r"^\s*[\u00A0]+\s*", re.MULTILINE)
+
+logger = getLogger(__name__)
+
+
+def _replace_nbsp(origdata, filename):
+    """Replace nbsp characters in leading whitespace in a YAML file."""
+    counter = 0
+
+    def _replacer_func(matchobj):
+        nonlocal counter
+        counter += 1
+        string = matchobj.group(0)
+        rc = string.replace(NBSP, " ")
+        return rc
+
+    data = pattern.sub(_replacer_func, origdata)
+
+    if counter:
+        plural = "s were" if counter > 1 else " was"
+        logger.warning(
+            f"Note: {counter} lines with non-breaking space character{plural} detected in {filename}.\n"
+            "Perhaps you cut and pasted from a Web page?\n"
+            "Future versions of CumulusCI may disallow these characters.\n"
+        )
+    return data
+
+
+def load_yaml_data(
+    source: T.Union[str, T.IO[T.Text], Path, FSResource], context: str = None
+):
+    """Load a file, convert NBSP->space and parse it in YAML.
+
+    Raises YAMLParseException with a nicely formatted error message
+    if an error occurs while parsing.
+
+    If you use this method directly (or, heaven forbid, yaml.safe_load)
+    consider making a CCIModel subclass instead.
+    """
+    with load_from_source(source) as (f_config, filename):
+        if filename == "<stream>":
+            filename = "a yaml file"
+        context = context or filename
+        data = _replace_nbsp(f_config.read(), context)
+        try:
+            rc = yaml.safe_load(StringIO(data))
+        except MarkedYAMLError as e:
+            line_num = e.problem_mark.line + 1
+            column_num = e.problem_mark.column
+            message = (
+                f"An error occurred parsing yaml file at line {line_num}, column {column_num}.\n"
+                + f"{filename}:{line_num}:{column_num}\n"
+                + f"Error message: {e.problem}."
+            )
+            raise YAMLParseException(message)
+        except Exception as e:
+            message = f"An error occurred parsing {filename}.\nError message: {e}"
+            raise YAMLParseException(message)
+
+        return rc

cumulusci/utils/yaml/tests/bad_cci.yml

@@ -0,0 +1,5 @@
+foo:
+
+bar:
+
+baz:

cumulusci/utils/yaml/tests/cassettes/TestCumulusciYml.test_validate_url__with_errors.yaml

@@ -0,0 +1,20 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Request-Headers:
+      - Elided
+    method: GET
+    uri: https://raw.githubusercontent.com/SFDO-Tooling/CumulusCI/8b8d1eb9a1593503bf625030fa702b6d4651cb55/cumulusci/tasks/bulkdata/tests/snowfakery/simple_snowfakery.load.yml
+  response:
+    body:
+      string: "- sf_object: Account\n  api: bulk\n"
+    headers:
+      Response-Headers:
+        Content-Type:
+        - text/plain; charset=utf-8
+        Others: Elided
+    status:
+      code: 200
+      message: OK
+version: 1