cumulusci-plus 5.0.0__py3-none-any.whl

cumulusci/utils/http/tests/test_multi_request.py

@@ -0,0 +1,255 @@
+import pytest
+import responses
+
+from cumulusci.tests.util import FakeUnreliableRequestHandler
+from cumulusci.utils.http.multi_request import CompositeParallelSalesforce
+
+COMPOSITE_RESPONSE = {
+    "compositeResponse": [
+        {
+            "body": {
+                "totalSize": 1,
+                "done": True,
+                "records": [
+                    {
+                        "attributes": {
+                            "type": "Account",
+                            "url": "/services/data/v49.0/sobjects/Account/0013B00000ddh3PQAQ",
+                        },
+                        "Id": "0013B00000ddh3PQAQ",
+                    }
+                ],
+            },
+            "httpHeaders": {},
+            "httpStatusCode": 200,
+            "referenceId": "one",
+        },
+        {
+            "body": {
+                "totalSize": 1,
+                "done": True,
+                "records": [
+                    {
+                        "attributes": {
+                            "type": "Account",
+                            "url": "/services/data/v49.0/sobjects/Account/0013B00000ddh7qQAA",
+                        },
+                        "Name": "Elizabeth Foster",
+                    }
+                ],
+            },
+            "httpHeaders": {},
+            "httpStatusCode": 200,
+            "referenceId": "two",
+        },
+    ]
+}
+
+
+class TestCompositeParallelSalesforce:
+    @pytest.mark.vcr()
+    def test_composite_parallel_salesforce(
+        self, sf, run_code_without_recording, delete_data_from_org
+    ):
+        run_code_without_recording(lambda: delete_data_from_org("Entitlement,Account"))
+        sf.Account.create(
+            {"Name": "Smith Corp."},
+        )
+        requests = [
+            {
+                "method": "GET",
+                "url": "/services/data/v50.0/query?q=SELECT Id FROM Account LIMIT 1",
+            },
+            {
+                "method": "GET",
+                "url": "/services/data/v50.0/query?q=SELECT Name FROM Account LIMIT 1",
+            },
+        ] * 2
+        with CompositeParallelSalesforce(sf, 5, max_workers=1) as cpsf:
+            results, errors = cpsf.do_composite_requests(requests)
+
+        assert len(list(results)) == len(requests)
+        for result in results:
+            assert result["body"]
+            assert result["body"]["done"]
+            assert result["body"]["records"][0]["attributes"]
+
+    @pytest.mark.vcr()
+    def test_not_opened(self, sf):
+        cpsf = CompositeParallelSalesforce(sf, 5, max_workers=1)
+        with pytest.raises(AssertionError):
+            cpsf.do_composite_requests([])
+
+    @pytest.mark.vcr()
+    def test_empty(self, sf):
+        with CompositeParallelSalesforce(sf, 5, max_workers=1) as cpsf:
+            results, errors = cpsf.do_composite_requests([])
+            assert list(results) == []
+
+    @pytest.mark.vcr()
+    def test_http_headers(self, sf, vcr):
+        requests = [
+            {
+                "method": "GET",
+                "url": "/services/data/v50.0/sobjects",
+                "httpHeaders": {"If-Modified-Since": "Thu, 03 Sep 2020 21:35:07 GMT"},
+            },
+        ] * 3
+        # don't re-record this one because you'll need to fiddle
+        # with the dates
+        with vcr.use_cassette(
+            "ManualEditTestCompositeParallelSalesforce.test_http_headers.yaml"
+        ):
+            with CompositeParallelSalesforce(sf, 4, max_workers=1) as cpsf:
+                results, errors = cpsf.do_composite_requests(requests)
+        assert results[0]["httpStatusCode"] == 304
+
+    @pytest.mark.vcr()
+    def test_reference_ids(self, sf, run_code_without_recording, delete_data_from_org):
+        run_code_without_recording(lambda: delete_data_from_org("Account"))
+        requests = [
+            {
+                "method": "GET",
+                "url": "/services/data/v50.0/query?q=SELECT Id FROM Account LIMIT 1",
+                "referenceId": "one",
+            },
+            {
+                "method": "GET",
+                "url": "/services/data/v50.0/query?q=SELECT Name FROM Account LIMIT 1",
+                "referenceId": "two",
+            },
+            {
+                "method": "GET",
+                "url": "/services/data/v50.0/query?q=SELECT Name FROM Account LIMIT 1",
+                "referenceId": "three",
+            },
+        ]
+        with CompositeParallelSalesforce(sf, 2, max_workers=1) as cpsf:
+            results, errors = cpsf.do_composite_requests(requests)
+        assert len(list(results)) == len(requests)
+        assert set(result["referenceId"] for result in results) == set(
+            request["referenceId"] for request in requests
+        )
+
+    @pytest.mark.vcr()
+    def test_errors(self, sf):
+        requests = [{"method": "GET", "url": "/services/data/v50.0/sobjects/Foo"}]
+        with CompositeParallelSalesforce(sf, 4, max_workers=1) as cpsf:
+            results, errors = cpsf.do_composite_requests(requests)
+        assert results[0]["httpStatusCode"] == 404, results[0]["httpStatusCode"]
+
+    @responses.activate
+    def test_multirequest_timeout(self, sf):
+        requests = [
+            {
+                "method": "GET",
+                "url": "/services/data/v50.0/query?q=SELECT Id FROM Account LIMIT 1",
+                "referenceId": "one",
+            },
+            {
+                "method": "GET",
+                "url": "/services/data/v50.0/query?q=SELECT Name FROM Account LIMIT 1",
+                "referenceId": "two",
+            },
+            {
+                "method": "GET",
+                "url": "/services/data/v50.0/query?q=SELECT Name FROM Account LIMIT 1",
+                "referenceId": "three",
+            },
+        ] * 2
+
+        composite_handler = FakeUnreliableRequestHandler(COMPOSITE_RESPONSE)
+        responses.add_callback(
+            responses.POST,
+            f"{sf.base_url}composite",
+            callback=composite_handler.request_callback,
+            content_type="application/json",
+        )
+
+        single_request_handler = FakeUnreliableRequestHandler(COMPOSITE_RESPONSE)
+        responses.add_callback(
+            responses.GET,
+            f"{sf.base_url}query?q=SELECT%20Id%20FROM%20Account%20LIMIT%201",
+            callback=single_request_handler.request_callback,
+            content_type="application/json",
+        )
+
+        with CompositeParallelSalesforce(sf, 2, max_workers=1) as cpsf:
+            results, errors = cpsf.do_composite_requests(requests)
+        assert len(errors) == 1  # one of the single requests fails after retry
+        assert len(results) == 5  # 4 succeed first time, one fails on retry
+
+        assert single_request_handler.counter == 2
+        assert composite_handler.counter == 3
+
+    @responses.activate
+    def test_multirequest_unknown_exception(self, sf):
+        requests = [
+            {
+                "method": "GET",
+                "url": "/services/data/v50.0/query?q=SELECT Id FROM Account LIMIT 1",
+                "referenceId": "one",
+            },
+            {
+                "method": "GET",
+                "url": "/services/data/v50.0/query?q=SELECT Name FROM Account LIMIT 1",
+                "referenceId": "two",
+            },
+            {
+                "method": "GET",
+                "url": "/services/data/v50.0/query?q=SELECT Name FROM Account LIMIT 1",
+                "referenceId": "three",
+            },
+        ] * 2
+
+        with CompositeParallelSalesforce(sf, 2, max_workers=1) as cpsf:
+            results, errors = cpsf.do_composite_requests(requests)
+        assert len(errors) == 3  # all should be rejected by Responses.
+        assert len(results) == 0
+
+    @responses.activate
+    def test_multirequest_timeout_POSTs(self, sf):
+        requests = [
+            {
+                "method": "GET",
+                "url": "/services/data/v50.0/query?q=SELECT Id FROM Account LIMIT 1",
+                "referenceId": "one",
+            },
+            {
+                "method": "GET",
+                "url": "/services/data/v50.0/query?q=SELECT Name FROM Account LIMIT 1",
+                "referenceId": "two",
+            },
+            {
+                "method": "POST",
+                "url": "/services/data/v50.0/query?q=SELECT Name FROM Account LIMIT 1",
+                "referenceId": "three",
+            },
+        ] * 2
+
+        composite_handler = FakeUnreliableRequestHandler(COMPOSITE_RESPONSE)
+        responses.add_callback(
+            responses.POST,
+            f"{sf.base_url}composite",
+            callback=composite_handler.request_callback,
+            content_type="application/json",
+        )
+
+        single_request_handler = FakeUnreliableRequestHandler(COMPOSITE_RESPONSE)
+        responses.add_callback(
+            responses.GET,
+            f"{sf.base_url}query?q=SELECT%20Id%20FROM%20Account%20LIMIT%201",
+            callback=single_request_handler.request_callback,
+            content_type="application/json",
+        )
+
+        with CompositeParallelSalesforce(sf, 2, max_workers=1) as cpsf:
+            results, errors = cpsf.do_composite_requests(requests)
+
+        # Three chunks of 2. SUCCEED, FAIL, SUCCEED
+        assert composite_handler.counter == 3
+
+        # The GET in the failing chunk should be retried and succeed
+        # The POST should not.
+        assert len(errors) == 1, str(errors)
+        assert single_request_handler.counter == 1

cumulusci/utils/iterators.py

@@ -0,0 +1,21 @@
+import itertools
+import typing as T
+from itertools import filterfalse, tee
+
+
+def iterate_in_chunks(n: int, iterable: T.Iterable) -> T.Iterable[T.Tuple]:
+    """Split an iterable into chunks of size 'n'."""
+    it = iter(iterable)
+    while True:
+        chunk = tuple(itertools.islice(it, n))
+        if not chunk:
+            return
+        yield chunk
+
+
+# From https://docs.python.org/3/library/itertools.html
+def partition(pred, iterable):
+    "Use a predicate to partition entries into false entries and true entries"
+    # partition(is_odd, range(10)) --> 0 2 4 6 8   and  1 3 5 7 9
+    t1, t2 = tee(iterable)
+    return filterfalse(pred, t1), filter(pred, t2)

cumulusci/utils/logging.py

@@ -0,0 +1,128 @@
+import contextlib
+import io
+import logging
+import os
+import pdb
+import re
+import sys
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+
+
+@contextlib.contextmanager
+def tee_stdout_stderr(args, logger, tempfile):
+    """Tee stdout and stderr so that they're also routed to
+    a log file. Add the current command arguments
+    as the first item in the log."""
+    real_stdout_write = sys.stdout.write
+    real_stderr_write = sys.stderr.write
+
+    # Add current command args as first line in logfile
+    logger.debug(" ".join(args) + "\n")
+
+    def stdout_write(s):
+        output = strip_ansi_sequences(s)
+        logger.debug(output)
+        real_stdout_write(s)
+
+    def stderr_write(s):
+        output = strip_ansi_sequences(s)
+        logger.debug(output)
+        real_stderr_write(s)
+
+    sys.stdout.write = stdout_write
+    sys.stderr.write = stderr_write
+    try:
+        yield
+    finally:
+        # reset write functions
+        sys.stdout.write = real_stdout_write
+        sys.stderr.write = real_stderr_write
+
+        # close temporary logfile
+        logger.handlers[0].close()
+
+        # log contents of tempfile to rotating log files
+        with open(tempfile, "r", encoding="utf-8", errors="backslashreplace") as f:
+            contents = f.read()
+
+        logger = get_gist_logger()
+        logger.debug(contents)
+        # delete temporary log file
+        os.remove(tempfile)
+
+
+def get_gist_logger():
+    """Determines the appropriate filepath for logfile
+    and name for the logger. Returns a logger with
+    RotatingFileHandler attached."""
+    logfile_dir = Path.home() / ".cumulusci" / "logs"
+    logfile_dir.mkdir(parents=True, exist_ok=True)
+    logfile_path = logfile_dir / "cci.log"
+
+    return get_rot_file_logger("stdout/stderr", logfile_path)
+
+
+def get_rot_file_logger(name, path):
+    """Returns a logger with a rotating file handler"""
+    logger = logging.getLogger(name)
+
+    handler = RotatingFileHandler(path, backupCount=5, encoding="utf-8")
+    handler.doRollover()  # rollover existing log files
+    handler.terminator = ""  # click.echo already adds a newline
+    logger.addHandler(handler)
+    logger.setLevel(logging.DEBUG)
+    return logger
+
+
+def strip_ansi_sequences(input):
+    """Strip ANSI sequences from what's in buffer"""
+    ansi_escape = re.compile(r"(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]")
+    return ansi_escape.sub("", input)
+
+
+class StreamToLog(io.TextIOBase):
+    """An IO stream which relays writes to a logger.
+
+    Writes without a newline at the end are buffered
+    so we can send a complete line to the logger.
+    """
+
+    def __init__(self, logger, level=logging.INFO):
+        self.logger = logger
+        self.level = level
+        self.buffer = []
+
+    def write(self, s):
+        self.buffer.append(s)
+        if s.endswith("\n"):
+            self.logger.log(self.level, "".join(self.buffer).strip())
+            self.buffer = []
+
+
+# Monkey-patch pdb to make sure it uses unredirected stdout by default
+orig_init = pdb.Pdb.__init__
+
+
+def pdb_init(self, *args, **kw):
+    orig_init(self, *args, **kw)
+    if self.stdout is sys.stdout:
+        self.stdout = sys.__stdout__
+        self.use_rawinput = 0
+
+
+pdb.Pdb.__init__ = pdb_init
+
+
+@contextlib.contextmanager
+def redirect_output_to_logger(logger):
+    """Context manager which redirects stdout and stderr to a logger"""
+    orig_stdout = sys.stdout
+    orig_stderr = sys.stderr
+    sys.stdout = StreamToLog(logger, logging.INFO)
+    sys.stderr = StreamToLog(logger, logging.WARNING)
+    try:
+        yield
+    finally:
+        sys.stdout = orig_stdout
+        sys.stderr = orig_stderr

cumulusci/utils/metaprogramming.py

@@ -0,0 +1,10 @@
+class classproperty(object):
+    """Similar to @property, but allows class-level properties.
+
+    That is, a property whose getter is like a classmethod."""
+
+    def __init__(self, fget):
+        self.fget = fget
+
+    def __get__(self, owner_self, owner_cls):
+        return self.fget(owner_cls)

cumulusci/utils/options.py

@@ -0,0 +1,138 @@
+import json
+from inspect import signature
+from typing import Any, Dict, List
+
+from pydantic import DirectoryPath, Field, FilePath, create_model
+
+from cumulusci.core.exceptions import TaskOptionsError
+from cumulusci.utils.yaml.model_parser import CCIDictModel
+
+READONLYDICT_ERROR_MSG = (
+    "The 'options' dictionary is read-only. Please use 'parsed_options' instead."
+)
+
+
+def _describe_field(field):
+    "Convert a Pydantic field into a CCI task_option dict"
+    rc = {
+        "description": field.field_info.description,
+        "required": field.required,
+    }
+    if field.field_info.default != ...:
+        rc["default"] = field.field_info.default
+    return rc
+
+
+class ReadOnlyOptions(dict):
+    """To enforce self.options to be read-only"""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def __setitem__(self, key, value):
+        raise TaskOptionsError(READONLYDICT_ERROR_MSG)
+
+    def __delitem__(self, key):
+        raise TaskOptionsError(READONLYDICT_ERROR_MSG)
+
+    def pop(self, key, default=None):
+        raise TaskOptionsError(READONLYDICT_ERROR_MSG)
+
+
+class CCIOptions(CCIDictModel):
+    "Base class for all options in tasks"
+
+    @classmethod
+    def as_task_options(cls):
+        return {
+            fieldname: _describe_field(field)
+            for fieldname, field in cls.__fields__.items()
+        }
+
+
+class CCIOptionType:
+    """Base class for custom option types.
+
+    Subclasses must implement 'from_str' and it must have a type-hinted return value like this:
+
+    class ListOfStringsOption(CCIOptionType):
+        @classmethod
+        def from_str(cls, v) -> List[str]:
+            return [s.strip() for s in v.split(",")]
+    """
+
+    name = None
+
+    @classmethod
+    def __get_validators__(cls):
+        "https://pydantic-docs.helpmanual.io/usage/types/#classes-with-__get_validators__"
+        yield cls.validate
+
+    @classmethod
+    def validate(cls, v):
+        """Validate and convert a value.
+
+        If its a string, parse it, else, just validate it.
+        """
+        if isinstance(v, str):
+            if v.startswith("{") or v.startswith("["):
+                v = json.loads(v)
+            else:
+                v = cls.from_str(v)
+
+        # Pydantic can't just parse/validate arbitrary data unless
+        # it has a model. So we create a dummy model for
+        # it to have a parsing/validating context
+        target_type = signature(cls.from_str).return_annotation
+
+        Dummy = create_model(cls.name or cls.__name__, __root__=(target_type, ...))
+
+        return Dummy.parse_obj(v).__root__
+
+
+class ListOfStringsOption(CCIOptionType):
+    """Parses a list of strings from a comma-delimited string"""
+
+    @classmethod
+    def from_str(cls, v) -> List[str]:
+        return [s.strip() for s in v.split(",")]
+
+
+class MappingOption(CCIOptionType):
+    """Parses a Mapping of Str->Any from a string in format a:b,c:d"""
+
+    @classmethod
+    def from_str(cls, v) -> Dict[str, Any]:
+        return parse_list_of_pairs_dict_arg(v)
+
+
+def parse_list_of_pairs_dict_arg(arg):
+    """Process an arg in the format "aa:bb,cc:dd" """
+    if isinstance(arg, dict):
+        return arg
+    elif isinstance(arg, str):
+        rc = {}
+        for key_value in arg.split(","):
+            subparts = key_value.split(":", maxsplit=1)
+            if len(subparts) == 2:
+                key, value = subparts
+                if key in rc:
+                    raise TypeError(f"Var specified twice: {key}")
+                rc[key] = value
+            else:
+                raise TypeError(f"Var is not a name/value pair: {key_value}")
+        return rc
+    else:
+        raise TypeError(f"Arg is not a dict or string ({type(arg)}): {arg}")
+
+
+__all__ = [
+    "Field",
+    "FilePath",
+    "DirectoryPath",
+    "parse_list_of_pairs_dict_arg",
+    "CCIOptions",
+    "CCIOptionType",
+    "ListOfStringsOption",
+    "MappingOption",
+]

cumulusci/utils/parallel/queries_in_parallel/run_queries_in_parallel.py

@@ -0,0 +1,29 @@
+from concurrent.futures import ThreadPoolExecutor
+from typing import Callable, Dict
+
+
+class RunParallelQueries:
+    @staticmethod
+    def _run_queries_in_parallel(
+        queries: Dict[str, str], run_query: Callable[[str], dict], num_threads: int = 4
+    ) -> Dict[str, list]:
+        """Accepts a set of queries structured as {'query_name': 'query'}
+        and a run_query function that runs a particular query. Runs queries in parallel and returns the queries"""
+        results_dict = {}
+
+        with ThreadPoolExecutor(max_workers=num_threads) as executor:
+            futures = {
+                query_name: executor.submit(run_query, query)
+                for query_name, query in queries.items()
+            }
+
+        for query_name, future in futures.items():
+            try:
+                query_result = future.result()
+                results_dict[query_name] = query_result["records"]
+            except Exception as e:
+                raise Exception(f"Error executing query '{query_name}': {type(e)}: {e}")
+            else:
+                queries.pop(query_name, None)
+
+        return results_dict

cumulusci/utils/parallel/queries_in_parallel/tests/test_run_queries_in_parallel.py

@@ -0,0 +1,50 @@
+from concurrent.futures import Future, ThreadPoolExecutor
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from cumulusci.utils.parallel.queries_in_parallel.run_queries_in_parallel import (
+    RunParallelQueries,
+)
+
+
+def run_query(query):
+    return {"test": query}
+
+
+def test_run_queries_in_parallel():
+    queries = {
+        "query1": "SELECT * FROM Table1",
+        "query2": "SELECT * FROM Table2",
+    }
+
+    expected_results = {
+        "query1": [{"field1": "value1"}, {"field1": "value2"}],
+        "query2": [{"field2": "value3"}, {"field2": "value4"}],
+    }
+
+    mock_future1 = MagicMock()
+    mock_future1.result.return_value = {"records": expected_results["query1"]}
+    mock_future2 = MagicMock()
+    mock_future2.result.return_value = {"records": expected_results["query2"]}
+
+    with patch.object(ThreadPoolExecutor, "submit") as mock_submit:
+        mock_submit.side_effect = [mock_future1, mock_future2]
+
+        results_dict = RunParallelQueries._run_queries_in_parallel(queries, run_query)
+
+    assert results_dict == expected_results
+
+
+def test_run_queries_in_parallel_with_exception():
+    queries = {
+        "Query1": "SELECT Id FROM Table1",
+    }
+
+    with patch.object(Future, "result", side_effect=Exception("Test exception")):
+        with pytest.raises(Exception) as excinfo:
+            RunParallelQueries._run_queries_in_parallel(queries, run_query=run_query)
+        assert (
+            "Error executing query 'Query1': <class 'Exception'>: Test exception"
+            == str(excinfo.value)
+        )