cumulusci-plus 5.0.0__py3-none-any.whl

cumulusci/tests/pytest_plugins/pytest_sf_vcr_serializer.py

@@ -0,0 +1,160 @@
+"""This module compresses VCR files on save using two techniques.
+
+1.  It looks in a small database of "known" request/response pairs to
+    reuse them. If it finds a hit, it creates an include_file reference.
+
+2.  It looks in a small database of "known" string patterns (mostly) XML
+    responses and replaces matching strings with references and variables.
+
+3.  It looks for duplication within a single YAML file and uses YAML
+    references to de-duplicate.
+"""
+
+import typing as T
+from pathlib import Path
+
+import yaml
+
+from .vcr_string_compressor import StringToTemplateCompressor
+
+
+class RequestResponseReplacement(T.NamedTuple):
+    """For the database of 'known' request/response pairs"""
+
+    request: dict
+    response: dict
+    replacement_file: str
+
+
+class CompressionVCRSerializer:
+    """A VCR serializer which compresses VCR files."""
+
+    def __init__(self, path: T.Union[Path, str]):
+        path = Path(path)
+        self.saved_responses, self.include_files = _read_include_files(path)
+        self.compressible_strings = StringToTemplateCompressor.from_directory(
+            path / "vcr_string_templates"
+        )
+        assert self.compressible_strings.patterns
+
+    def deserialize(self, cassette_string: str) -> dict:
+        data = yaml.safe_load(cassette_string)
+        walk_yaml(data, ExpandStringTemplates(self.compressible_strings))
+        return _expand_external_references(data, self.include_files)
+
+    def serialize(self, cassette_dict: dict) -> str:
+        compressed_data = _compress_in_place(
+            cassette_dict, self.saved_responses, self.compressible_strings
+        )
+        return yaml.dump(compressed_data, sort_keys=False)
+
+
+def _read_include_files(path: Path) -> T.Tuple[dict, dict]:
+    """Read canned responses from disk."""
+    saved_responses = {}
+    include_files = {}
+    for file in tuple(path.glob("*.yaml")) + tuple(path.glob("*.yml")):
+        request_response = yaml.safe_load(file.read_text())
+        rrr = RequestResponseReplacement(
+            request_response["request"], request_response["response"], file.name
+        )
+        saved_responses[repr(rrr.request)] = rrr
+        include_files[rrr.replacement_file] = rrr
+    return saved_responses, include_files
+
+
+def _expand_external_references(d: dict, include_files: dict):
+    """Expand external references on deserialization"""
+    interactions = [
+        _expand_reference(interaction, include_files)
+        for interaction in d["interactions"]
+    ]
+    return {"interactions": interactions, "version": d["version"]}
+
+
+def _expand_reference(possible_reference, include_files: dict):
+    """Expand an external reference on deserialization"""
+    if isinstance(possible_reference, dict) and "include_file" in possible_reference:
+        filename = possible_reference["include_file"]
+        replacement_rrr = include_files[filename]
+        return {
+            "request": replacement_rrr.request,
+            "response": replacement_rrr.response,
+        }
+    else:
+        return possible_reference
+
+
+def _replace_interaction_if_possible(
+    interaction: dict, replacements: T.Dict[str, RequestResponseReplacement]
+) -> dict:
+    """Try to replace an interaction with a canned one.
+
+    Otherwise just return the interaction.
+    """
+    replacement = replacements.get(repr(interaction["request"]))
+    if replacement:
+        return {"include_file": replacement.replacement_file}
+    return interaction
+
+
+def _compress_in_place(d: dict, saved_responses: dict, sc: StringToTemplateCompressor):
+    """First replace canned interactions. Then compress repetitive data."""
+    d["interactions"] = [
+        _replace_interaction_if_possible(interaction, saved_responses)
+        for interaction in d["interactions"]
+    ]
+    walk_yaml(d, VCRCompressor(sc))
+    return d
+
+
+def walk_yaml(d: dict, mutator):
+    """Visit each node in a dict/list tree datastructure"""
+    if isinstance(d, (dict, list)):
+        if isinstance(d, dict):
+            items = d.items()
+        else:
+            items = enumerate(d)
+        for key, value in items:
+            walk_yaml(value, mutator)
+            value = mutator(value)
+            d[key] = value
+
+
+class VCRCompressor:
+    """Callable that replaces strings with templates and de-dupes structures"""
+
+    def __init__(self, string_compressor):
+        self.mappings = {}
+        self.string_compressor = string_compressor
+
+    def __call__(self, value):
+        return self.compress(value)
+
+    def compress(self, value):
+        lookup = repr(value)
+        if isinstance(value, str):
+            value = self.string_compressor.string_to_template_if_possible(value)
+            assert value is not None
+        elif lookup in self.mappings:
+            value = self.mappings[lookup]
+        elif isinstance(value, (dict, list)):
+            self.mappings[lookup] = value
+        return value
+
+
+class ExpandStringTemplates:
+    """Callable that replaces templates with strings
+
+    De-duping isn't necessary, because a DAG is as good as a tree for our purposes."""
+
+    def __init__(self, string_compressor):
+        self.string_compressor = string_compressor
+
+    def __call__(self, value):
+        return self.decompress(value)
+
+    def decompress(self, value):
+        if isinstance(value, dict) and "include_template" in value:
+            value = self.string_compressor.template_to_string(value)
+        return value

cumulusci/tests/pytest_plugins/pytest_typeguard.py

@@ -0,0 +1,5 @@
+from typeguard.importhook import install_import_hook
+
+
+def pytest_sessionstart(session):
+    install_import_hook(packages=["cumulusci"])

cumulusci/tests/pytest_plugins/test_vcr_string_compressor.py

@@ -0,0 +1,49 @@
+from cumulusci.tests.pytest_plugins.vcr_string_compressor import (
+    StringToTemplateCompressor,
+)
+
+tpl_dir = "cumulusci/tests/shared_cassettes/vcr_string_templates"
+
+
+example_string = """
+<?xml version="1.0" encoding="UTF-8"?><jobInfo
+   xmlns="http://www.force.com/2009/06/asyncapi/dataload">
+ <id>75023000003RqO6AAK</id>
+ <operation>insert</operation>
+ <object>Account</object>
+ <createdById>00523000002KbrQAAS</createdById>
+ <createdDate>2022-03-29T20:03:56.000Z</createdDate>
+ <systemModstamp>2022-03-29T20:03:56.000Z</systemModstamp>
+ <state>Open</state>
+ <concurrencyMode>Parallel</concurrencyMode>
+ <contentType>CSV</contentType>
+ <numberBatchesQueued>0</numberBatchesQueued>
+ <numberBatchesInProgress>0</numberBatchesInProgress>
+ <numberBatchesCompleted>0</numberBatchesCompleted>
+ <numberBatchesFailed>0</numberBatchesFailed>
+ <numberBatchesTotal>0</numberBatchesTotal>
+ <numberRecordsProcessed>0</numberRecordsProcessed>
+ <numberRetries>0</numberRetries>
+ <apiVersion>vxx.0</apiVersion>
+ <numberRecordsFailed>0</numberRecordsFailed>
+ <totalProcessingTime>0</totalProcessingTime>
+ <apiActiveProcessingTime>0</apiActiveProcessingTime>
+ <apexProcessingTime>0</apexProcessingTime>
+</jobInfo>""".strip()
+
+
+class TestVcrStringToTemplateCompressor:
+    def test_simple_string_compression(self):
+        sc = StringToTemplateCompressor.from_directory(tpl_dir)
+        data = sc.string_to_template_if_possible(example_string)
+        assert data == {
+            "include_template": "jobInfo_insert_xml.tpl",
+            "vars": {
+                "id": "75023000003RqO6AAK",
+                "creator": "00523000002KbrQAAS",
+                "cdate": "2022-03-29T20:03:56.000Z",
+                "smts": "2022-03-29T20:03:56.000Z",
+                "qdbatches": "0",
+                "numbatchtotal": "0",
+            },
+        }

cumulusci/tests/pytest_plugins/vcr_string_compressor.py

@@ -0,0 +1,97 @@
+"""
+A Template like this:
+
+template.tpl:
+
+<?xml ...><foo xmls=...><id>{id}</id><status>{status:Closed}</status></foo>
+
+can be referred to by name like this:
+
+include_file: template.tpl
+vars:
+    id: 001A2431AF3
+
+Note that the status can be elided because it defaults to Closed.
+"""
+
+
+import re
+import typing as T
+from pathlib import Path
+
+MATCHER_REGEXP = re.compile("{(?P<name>[^:}]+)(:(?P<default>.*))?}")
+
+
+class Pattern(T.NamedTuple):
+    """Template pattern that can be referred to in to save space"""
+
+    filename: str  # where to find the template
+    orig_pattern: str  # the contents of the file
+    regexp: re.Pattern  # contents converted to Python compatible regexp
+    replacement_template: str  # contents converted to Python format-string
+    default_values: T.Dict[str, str]  # defaulted values to save space further
+
+    @staticmethod
+    def parse(filename, pat) -> "Pattern":
+        """Parse the contents of a pattern file into a Pattern obj"""
+        replaceables = list(MATCHER_REGEXP.findall(pat))
+        assert replaceables
+        escaped_pat = pat.replace("?", r"\?")  # re.escape is too aggressive :(
+        regexp = re.compile(MATCHER_REGEXP.sub(r"(?P<\1>.*)", escaped_pat))
+        replacement_template = MATCHER_REGEXP.sub(r"{\1}", pat)
+        default_values = {match[0]: match[-1] for match in replaceables if match[-1]}
+        return Pattern(filename, pat, regexp, replacement_template, default_values)
+
+
+class StringToTemplateCompressor:
+    """Code that can translate from strings to templates and back"""
+
+    @classmethod
+    def from_directory(cls, path: T.Union[Path, str]):
+        """Make a compressor from a directory of "tpl" files."""
+        path = Path(path)
+        raw_patterns = {f.name: f.read_text() for f in path.glob("*.tpl")}
+        assert raw_patterns, f"Directory was empty {path}"
+        return cls(raw_patterns)
+
+    def __init__(self, raw_patterns: T.Dict[str, str]):
+        self.patterns_by_filename = {
+            filename: Pattern.parse(filename, pattern)
+            for filename, pattern in raw_patterns.items()
+        }
+        self.patterns = self.patterns_by_filename.values()
+
+    def string_to_template_if_possible(self, string: str):
+        """Return a template dict if a template is matching.
+
+        Else return the original string."""
+        match = _find_match(self.patterns, string)
+        if match:
+            pattern, values = match
+            return {"include_template": pattern.filename, "vars": values}
+        else:
+            return string
+
+    def template_to_string(self, template: dict):
+        """Reverse the templating process"""
+        template_name = template["include_template"]
+        template_vars = template["vars"]
+        template = self.patterns_by_filename[template_name]
+        return template.replacement_template.format(
+            **{**template.default_values, **template_vars}
+        )
+
+
+def _expand(pattern, values):
+    return pattern.replacement_template.format(**{**pattern.default_values, **values})
+
+
+def _find_match(patterns, string):
+    for pattern in patterns:
+        match = pattern.regexp.match(string)
+        if match:
+            return pattern, {
+                k: v
+                for k, v in match.groupdict().items()
+                if v != pattern.default_values.get(k)
+            }