cumulusci-plus 5.0.0__py3-none-any.whl

cumulusci/utils/parallel/task_worker_queues/tests/test_parallel_worker.py

@@ -0,0 +1,353 @@
+from contextlib import contextmanager
+from logging import getLogger
+from multiprocessing import Lock
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from unittest import mock
+
+import pytest
+
+from cumulusci.core import exceptions as exc
+from cumulusci.core.config import BaseProjectConfig, OrgConfig, UniversalConfig
+from cumulusci.tasks.util import Sleep
+from cumulusci.utils.parallel.task_worker_queues.parallel_worker import (
+    ParallelWorker,
+    SubprocessKeychain,
+    TaskWorker,
+    WorkerConfig,
+)
+from cumulusci.utils.parallel.task_worker_queues.parallel_worker_queue import (
+    WorkerQueue,
+    WorkerQueueConfig,
+)
+
+logger = getLogger(__name__)
+dummy_project_config = BaseProjectConfig(
+    UniversalConfig(),
+    config={"project": {"package": "packageA"}},
+)
+dummy_org_config = OrgConfig({}, "dummy_config")
+
+
+class DelaySpawner:
+    def __init__(self, target, args, daemon):
+        logger.info(f"Creating spawner {target} {args}")
+        self.target = target
+        self.args = args
+        self._is_alive = False
+
+    def start(self):
+        logger.info(f"Starting spawner {self}")
+        self._is_alive = True
+
+    def _finish(self):
+        self.target(*self.args)
+        self._is_alive = False
+
+    def is_alive(self):
+        logger.info(f"Checking alive: {self}: {self._is_alive}")
+        return self._is_alive
+
+    def terminate(self):
+        self._is_alive = False
+
+
+class TestWorkerQueue:
+    @contextmanager
+    def configure_worker_queue(self, parent_dir, **kwargs):
+        config = WorkerQueueConfig(
+            project_config=dummy_project_config,
+            org_config=dummy_org_config,
+            connected_app=None,
+            redirect_logging=True,
+            spawn_class=DelaySpawner,
+            parent_dir=Path(parent_dir),
+            **kwargs,
+        )
+        q = WorkerQueue(config, filesystem_lock=Lock())
+
+        yield q
+
+    def test_worker_queue(self, tmpdir):
+        with self.configure_worker_queue(
+            parent_dir=tmpdir,
+            name="start",
+            task_class=Sleep,
+            make_task_options=lambda *args, **kwargs: {"seconds": 0},
+            queue_size=3,
+            num_workers=2,
+        ) as q:
+            assert not q.full
+            assert q.empty
+            assert q.num_free_workers == 2
+            assert q.queued_job_dirs == []
+            q.push(name="a")
+            assert not q.empty
+            assert not q.full
+            assert q.num_free_workers == 1
+            assert len(q.queued_job_dirs) == 0
+            q.tick()
+            assert not q.full
+            assert q.num_free_workers == 1
+            assert len(q.queued_job_dirs) == 0
+            q.workers[0].process._finish()
+            q.tick()
+            assert not q.full
+            assert q.num_free_workers == 2
+            assert len(q.queued_job_dirs) == 0
+            q.tick()
+            assert not q.full
+            assert q.num_free_workers == 2
+            assert len(q.queued_job_dirs) == 0
+            q.push(name="bb")
+            q.tick()
+            assert not q.full
+            assert q.num_free_workers == 1
+            assert len(q.queued_job_dirs) == 0
+            q.push(name="cc")
+            q.tick()
+            assert not q.full
+            assert q.num_free_workers == 0
+            assert len(q.queued_job_dirs) == 0
+            q.push(name="dd")
+            q.tick()
+            assert not q.full
+            assert q.num_free_workers == 0
+            assert len(q.queued_job_dirs) == 1
+            q.push(name="ee")
+            q.tick()
+            assert not q.full
+            assert q.num_free_workers == 0
+            assert len(q.queued_job_dirs) == 2
+            q.push(name="ff")
+            q.tick()
+            assert q.full
+            assert q.num_free_workers == 0
+            assert len(q.queued_job_dirs) == 3
+
+            with pytest.raises(ValueError):
+                q.push(name="hh")
+
+            assert q.full
+            assert q.num_free_workers == 0
+            assert len(q.queued_job_dirs) == 3
+
+            for worker in q.workers:
+                worker.process._finish()
+
+            q.tick()
+
+            assert not q.full
+            assert q.num_free_workers == 0
+            assert len(q.queued_job_dirs) == 1
+
+            for worker in q.workers:
+                worker.process._finish()
+
+            q.tick()
+
+            assert not q.full
+            assert q.num_free_workers == 1
+            assert len(q.queued_job_dirs) == 0
+
+            for worker in q.workers:
+                worker.process._finish()
+
+            q.tick()
+
+            assert not q.full
+            assert q.num_free_workers == 2
+            assert len(q.queued_job_dirs) == 0
+
+            q.push(name="ii")
+            q.push(name="jj")
+            q.push(name="kk")
+            q.push(name="ll")
+            q.push(name="mm")
+            with pytest.raises(ValueError):
+                q.push(name="nn")
+
+            q.tick()
+
+            assert q.full
+            assert q.num_free_workers == 0
+            assert len(q.queued_job_dirs) == 3
+
+    def test_worker_queues_together(self, tmpdir):
+        with self.configure_worker_queue(
+            parent_dir=tmpdir,
+            name="start",
+            task_class=Sleep,
+            make_task_options=lambda *args, **kwargs: {"seconds": 0},
+            queue_size=3,
+            num_workers=2,
+        ) as q1, self.configure_worker_queue(
+            parent_dir=tmpdir,
+            name="next",
+            task_class=Sleep,
+            make_task_options=lambda *args, **kwargs: {"seconds": 0},
+            queue_size=3,
+            num_workers=2,
+        ) as q2:
+            q1.feeds_data_to(q2)
+            q1.push(name="a")
+            assert not q1.full
+            assert q1.num_free_workers == 1
+            assert len(q1.queued_job_dirs) == 0
+            q1.tick()
+            q1.tick()
+            assert not q1.full
+            assert q1.num_free_workers == 1
+            assert len(q1.queued_job_dirs) == 0
+
+            for worker in q1.workers:
+                worker.process._finish()
+            q1.tick()
+            q2.tick()
+            assert not q2.full
+            assert q1.num_free_workers == 2
+            assert q2.num_free_workers == 1
+            assert len(q2.queued_job_dirs) == 0
+
+            q1.push(name="b")
+            q1.push(name="c")
+            q1.push(name="d")
+            q1.push(name="e")
+
+            for worker in q1.workers:
+                worker.process._finish()
+            q1.tick()
+            q2.tick()
+
+    def test_worker_queues_together__outbox_cannot_be_removed(self, tmpdir):
+        with self.configure_worker_queue(
+            parent_dir=tmpdir,
+            name="start",
+            task_class=Sleep,
+            make_task_options=lambda *args, **kwargs: {"seconds": 0},
+            queue_size=3,
+            num_workers=2,
+        ) as q1, self.configure_worker_queue(
+            parent_dir=tmpdir,
+            name="next",
+            task_class=Sleep,
+            make_task_options=lambda *args, **kwargs: {"seconds": 0},
+            queue_size=3,
+            num_workers=2,
+        ) as q2:
+            q1.outbox_dir.rmdir()
+            with mock.patch(
+                "cumulusci.utils.parallel.task_worker_queues.parallel_worker_queue.logger.info"
+            ) as logger_info:
+                q1.feeds_data_to(q2)
+                assert "Cannot remove outbox dir" in logger_info.mock_calls[0][1][0]
+
+    def test_worker_queue_from_path(self, tmpdir):
+        parentdirs = [
+            "start_inprogress",
+            "next_inprogress",
+        ]
+
+        def make_task_options(path: Path, *args, **kwargs):
+            assert path.name == "foo"
+            assert path.parent.name == parentdirs.pop(0), path.parent.name
+            return {"seconds": 0}
+
+        with self.configure_worker_queue(
+            parent_dir=tmpdir,
+            name="start",
+            task_class=Sleep,
+            make_task_options=make_task_options,
+            queue_size=3,
+            num_workers=2,
+        ) as q1, self.configure_worker_queue(
+            parent_dir=tmpdir,
+            name="next",
+            task_class=Sleep,
+            make_task_options=make_task_options,
+            queue_size=3,
+            num_workers=2,
+        ) as q2:
+            q1.feeds_data_to(q2)
+            foo = tmpdir / "foo"
+            foo.mkdir()
+            assert not q1.full
+            assert q1.num_free_workers == 2
+            assert q1.queued_job_dirs == []
+            q1.push(Path(foo))
+            assert not q1.full
+            assert q1.num_free_workers == 1
+            assert len(q1.queued_job_dirs) == 0
+            q1.tick()
+
+            for worker in q1.workers:
+                worker.process._finish()
+
+            q1.tick()
+            q2.tick()
+            assert q2.inprogress_jobs == ["foo"]
+            q2.tick()
+
+            for worker in q2.workers:
+                worker.process._finish()
+            q2.tick()
+            assert q2.outbox_jobs == ["foo"]
+
+
+class TestParallelWorker:
+    def test_terminate_parallel_worker(self):
+        with TemporaryDirectory() as failures_dir, TemporaryDirectory() as outbox_dir, TemporaryDirectory() as working_dir:
+            config = WorkerConfig(
+                project_config=dummy_project_config,
+                org_config=dummy_org_config,
+                connected_app=None,
+                redirect_logging=True,
+                task_class=Sleep,
+                task_options={"seconds": 0},
+                failures_dir=failures_dir,
+                outbox_dir=outbox_dir,
+                working_dir=working_dir,
+            )
+            worker = ParallelWorker(DelaySpawner, config, None, Lock())
+            worker.start()
+            worker.terminate()
+            assert "Alive: False" in repr(worker)
+
+
+class TestTaskWorker:
+    def test_worker__cannot_move_to_outdir(self):
+        with TemporaryDirectory() as failures_dir, TemporaryDirectory() as outbox_dir, TemporaryDirectory() as working_dir:
+            config = WorkerConfig(
+                project_config=dummy_project_config,
+                org_config=dummy_org_config,
+                connected_app=None,
+                redirect_logging=True,
+                task_class=Sleep,
+                task_options={"seconds": 0},
+                failures_dir=failures_dir,
+                outbox_dir=outbox_dir,
+                working_dir=working_dir,
+            )
+            p = TaskWorker(config.as_dict(), None, Lock())
+            with mock.patch("shutil.move", side_effect=AssertionError):
+                with pytest.raises(AssertionError):
+                    p.run()
+            assert Path(working_dir, "exception.txt").exists()
+
+
+# Frankly these tests are primarily for coverage-counting purposes.
+# Meaningful tests of keychain stuff are by definition integration
+# tests, and we only use connected apps for persistent orgs, which
+# makes this even more messy.
+
+# Also we usually mock refresh_oauth_token which is what would
+# invoke this. In other words, using integration tests to cover this
+# function is far easier than using unit test.
+class TestSubprocessKeychain:
+    def test_subprocess_keychain(self):
+        skc = SubprocessKeychain("Blah")
+
+        assert skc.get_service("connected_app") == "Blah"
+        skc.set_org()
+        with pytest.raises(exc.ServiceNotConfigured):
+            skc.get_service("xyzzy")

cumulusci/utils/salesforce/count_sobjects.py

@@ -0,0 +1,46 @@
+import typing as T
+
+from simple_salesforce import Salesforce
+
+from cumulusci.utils.http.multi_request import CompositeParallelSalesforce
+from cumulusci.utils.iterators import partition
+
+
+class ObjectCount(T.NamedTuple):
+    counts: T.Dict[str, int]
+    transport_errors: T.Sequence[dict]  # e.g. couldn't reach server
+    salesforce_errors: T.Sequence[dict]  # e.g. 404, 401
+
+
+def count_sobjects(sf: Salesforce, objs: T.Sequence[str]) -> ObjectCount:
+    """Quickly count SObjects using SOQL and Parallelization"""
+    with CompositeParallelSalesforce(sf, max_workers=8, chunk_size=5) as cpsf:
+        responses, transport_errors = cpsf.do_composite_requests(
+            (
+                {
+                    "method": "GET",
+                    "url": f"/services/data/v{sf.sf_version}/query/?q=select count() from {obj}",
+                    "referenceId": f"ref{obj}",
+                }
+                for obj in objs
+            )
+        )
+        salesforce_errors, successes = partition(
+            lambda response: response["httpStatusCode"] == 200, responses
+        )
+
+        transport_errors = tuple(error._asdict() for error in transport_errors)
+
+    successes = list(successes)
+
+    def removeprefix(mainstr: str, prefix: str):
+        # Until Python 3.9 is minimum supported
+        a, b = mainstr[0 : len(prefix)], mainstr[len(prefix) :]
+        assert a == prefix
+        return b
+
+    ret = {
+        removeprefix(response["referenceId"], "ref"): response["body"]["totalSize"]
+        for response in successes
+    }
+    return ObjectCount(ret, transport_errors, tuple(salesforce_errors))

cumulusci/utils/salesforce/soql.py

@@ -0,0 +1,17 @@
+from typing import Optional
+
+
+def format_subscriber_package_version_where_clause(
+    spv_id: str, install_key: Optional[str] = None
+) -> str:
+    """Get the where clause for a SubscriberPackageVersion query
+
+    Does not include the WHERE.
+    Includes the installation key if provided.
+    """
+    where_clause = f"Id='{spv_id}'"
+
+    if install_key:
+        where_clause += f" AND InstallationKey ='{install_key}'"
+
+    return where_clause

cumulusci/utils/salesforce/tests/cassettes/ManualEdit_TestCountSObjects.test_count_sobjects__network_errors.yaml

@@ -0,0 +1,23 @@
+version: 1 # Manually edited to induce 500 error
+interactions:
+    - request:
+          method: POST
+          uri: https://orgname.my.salesforce.com/services/data/vxx.0/composite
+          body:
+              '{"compositeRequest": [{"referenceId": "refAccount", "method": "GET", "url":
+              "/services/data/vxx.0/query/?q=select count() from Account"}, {"referenceId":
+              "refXYZZY", "method": "GET", "url": "/services/data/vxx.0/query/?q=select count()
+              from XYZZY"}]}'
+          headers:
+              Request-Headers:
+                  - Elided
+      response:
+          status:
+              code: 500 # Manually edited to induce an error
+              message: OK
+          headers:
+              Content-Type:
+                  - application/json;charset=UTF-8
+              Others: Elided
+          body:
+              string: ""

cumulusci/utils/salesforce/tests/cassettes/TestCountSObjects.test_count_sobjects__errors.yaml

@@ -0,0 +1,33 @@
+version: 1
+interactions:
+    - request:
+          method: POST
+          uri: https://orgname.my.salesforce.com/services/data/vxx.0/composite
+          body:
+              '{"compositeRequest": [{"referenceId": "refAccount", "method": "GET", "url":
+              "/services/data/vxx.0/query/?q=select count() from Account"}, {"referenceId":
+              "refXYZZY", "method": "GET", "url": "/services/data/vxx.0/query/?q=select count()
+              from XYZZY"}]}'
+          headers:
+              Request-Headers:
+                  - Elided
+      response:
+          status:
+              code: 200
+              message: OK
+          headers:
+              Content-Type:
+                  - application/json;charset=UTF-8
+              Others: Elided
+          body:
+              string:
+                  "{\n  \"compositeResponse\" : [ {\n    \"body\" : {\n      \"totalSize\"\
+                  \ : 17,\n      \"done\" : true,\n      \"records\" : [ ]\n    },\n    \"httpHeaders\"\
+                  \ : { },\n    \"httpStatusCode\" : 200,\n    \"referenceId\" : \"refAccount\"\
+                  \n  }, {\n    \"body\" : [ {\n      \"message\" : \"\\nselect count() from\
+                  \ XYZZY\\n                    ^\\nERROR at Row:1:Column:21\\nsObject type\
+                  \ 'XYZZY' is not supported. If you are attempting to use a custom object,\
+                  \ be sure to append the '__c' after the entity name. Please reference your\
+                  \ WSDL or the describe call for the appropriate names.\",\n      \"errorCode\"\
+                  \ : \"INVALID_TYPE\"\n    } ],\n    \"httpHeaders\" : { },\n    \"httpStatusCode\"\
+                  \ : 400,\n    \"referenceId\" : \"refXYZZY\"\n  } ]\n}"

cumulusci/utils/salesforce/tests/cassettes/TestCountSObjects.test_count_sobjects_simple.yaml

@@ -0,0 +1,29 @@
+version: 1
+interactions:
+    - request:
+          method: POST
+          uri: https://orgname.my.salesforce.com/services/data/vxx.0/composite
+          body:
+              '{"compositeRequest": [{"referenceId": "refAccount", "method": "GET", "url":
+              "/services/data/vxx.0/query/?q=select count() from Account"}, {"referenceId":
+              "refOpportunity", "method": "GET", "url": "/services/data/vxx.0/query/?q=select
+              count() from Opportunity"}]}'
+          headers:
+              Request-Headers:
+                  - Elided
+      response:
+          status:
+              code: 200
+              message: OK
+          headers:
+              Content-Type:
+                  - application/json;charset=UTF-8
+              Others: Elided
+          body:
+              string:
+                  "{\n  \"compositeResponse\" : [ {\n    \"body\" : {\n      \"totalSize\"\
+                  \ : 17,\n      \"done\" : true,\n      \"records\" : [ ]\n    },\n    \"httpHeaders\"\
+                  \ : { },\n    \"httpStatusCode\" : 200,\n    \"referenceId\" : \"refAccount\"\
+                  \n  }, {\n    \"body\" : {\n      \"totalSize\" : 0,\n      \"done\" : true,\n\
+                  \      \"records\" : [ ]\n    },\n    \"httpHeaders\" : { },\n    \"httpStatusCode\"\
+                  \ : 200,\n    \"referenceId\" : \"refOpportunity\"\n  } ]\n}"

cumulusci/utils/salesforce/tests/test_count_sobjects.py

@@ -0,0 +1,29 @@
+import pytest
+import vcr
+
+from cumulusci.utils.salesforce.count_sobjects import count_sobjects
+
+
+class TestCountSObjects:
+    @pytest.mark.vcr()
+    def test_count_sobjects_simple(self, sf):
+        results, net_errors, sf_errors = count_sobjects(sf, ["Account", "Opportunity"])
+        assert "Account" in results
+        assert "Opportunity" in results
+        assert not net_errors and not sf_errors
+
+    @pytest.mark.vcr()
+    def test_count_sobjects__errors(self, sf):
+        results, net_errors, sf_errors = count_sobjects(sf, ["Account", "XYZZY"])
+        assert "Account" in results
+        assert "XYZZY" not in results
+        assert not net_errors
+        assert sf_errors and "XYZZY" in str(sf_errors)
+
+    @pytest.mark.vcr()
+    def test_count_sobjects__network_errors(self, sf):
+        network_errors_yaml = "cumulusci/utils/salesforce/tests/cassettes/ManualEdit_TestCountSObjects.test_count_sobjects__network_errors.yaml"
+        with vcr.use_cassette(network_errors_yaml):
+            _, net_errors, sf_errors = count_sobjects(sf, ["Account", "XYZZY"])
+            assert net_errors
+            assert not sf_errors

cumulusci/utils/salesforce/tests/test_soql.py

@@ -0,0 +1,30 @@
+import pytest
+
+from cumulusci.utils.salesforce.soql import (
+    format_subscriber_package_version_where_clause,
+)
+
+
+class TestSoql:
+    spv_id = "04t000000000000"
+
+    @pytest.mark.vcr()
+    def test_format_subscriber_package_version_where_clause_simple(self):
+        where_clause = format_subscriber_package_version_where_clause(self.spv_id, None)
+        assert f"Id='{self.spv_id}'" in where_clause
+        assert " AND InstallationKey =" not in where_clause
+
+    @pytest.mark.vcr()
+    def format_subscriber_package_version_where_clause_install_key_set(self):
+        install_key = "hunter2"
+        where_clause = format_subscriber_package_version_where_clause(
+            self.spv_id, install_key
+        )
+        assert f"Id='{self.spv_id}'" in where_clause
+        assert f" AND InstallationKey ='{install_key}'" in where_clause
+
+    @pytest.mark.vcr()
+    def format_subscriber_package_version_where_clause_install_key_none(self):
+        where_clause = format_subscriber_package_version_where_clause(self.spv_id, None)
+        assert f"Id='{self.spv_id}'" in where_clause
+        assert " AND InstallationKey =" not in where_clause

cumulusci/utils/tests/cassettes/ManualEditTestDescribeOrg.test_minimal_schema.yaml

@@ -0,0 +1,36 @@
+interactions:
+    - include_file: GET_sobjects_Global_describe.yaml
+    - request:
+          body:
+              '{"compositeRequest": [{"referenceId": "refAccount", "method": "GET", "url":
+              "/services/data/vxx.0/sobjects/Account/describe", "httpHeaders": {"If-Modified-Since":
+              "Sat, 1 Jan 2000 00:00:01 GMT"}}, {"referenceId": "refOpportunity", "method":
+              "GET", "url": "/services/data/vxx.0/sobjects/Opportunity/describe", "httpHeaders":
+              {"If-Modified-Since": "Sat, 1 Jan 2000 00:00:01 GMT"}}]}'
+          headers:
+              Request-Headers:
+                  - Elided
+          method: POST
+          uri: https://orgname.my.salesforce.com/services/data/vxx.0/composite
+      response:
+          body:
+              string:
+                  '{"compositeResponse": [{"body": {"label": "Account", "labelPlural": "Accounts", "layoutable":
+                  true, "listviewable": null, "lookupLayoutable": null, "mergeable": true, "mruEnabled":
+                  true, "name": "Account", "fields":[]}, "httpHeaders": {"ETag": "\"2b806538\"",
+                  "Last-Modified": "Tue, 28 Jun 2022 03:46:54 GMT"}, "httpStatusCode": 200,
+                  "referenceId": "refAccount"}, {"body": {"fields": [], "name": "Opportunity",
+                  "namedLayoutInfos": [], "networkScopeFieldName": null, "queryable": true,
+                  "replicateable": true, "retrieveable": true, "searchLayoutable": true,
+                  "searchable": true, "sobjectDescribeOption": "FULL", "triggerable": true,
+                  "undeletable": true, "updateable": true}, "httpHeaders":
+                  {"ETag": "\"2b806538\"", "Last-Modified": "Tue, 28 Jun 2022 03:46:54 GMT"},
+                  "httpStatusCode": 200, "referenceId": "refOpportunity"}]}'
+          headers:
+              Content-Type:
+                  - application/json;charset=UTF-8
+              Others: Elided
+          status:
+              code: 200
+              message: OK
+version: 1