pl-dict-diff 0.0.1__py3-none-any.whl

inspire_dictdiffer/__init__.py

@@ -0,0 +1,430 @@
+# This file is part of Dictdiffer.
+#
+# Copyright (C) 2013 Fatih Erikli.
+# Copyright (C) 2013, 2014, 2015, 2016 CERN.
+# Copyright (C) 2017-2019 ETH Zurich, Swiss Data Science Center, Jiri Kuncar.
+#
+# Dictdiffer is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Dictdiffer is a helper module to diff and patch dictionaries."""
+
+from collections.abc import Iterable, MutableMapping, MutableSequence, MutableSet
+from copy import deepcopy
+
+from .utils import (
+    EPSILON,
+    PathLimit,
+    RemovedObject,
+    are_different,
+    dot_lookup,
+    strip_removed_objects,
+)
+from .version import __version__
+
+(ADD, REMOVE, CHANGE) = ("add", "remove", "change")
+
+__all__ = (
+    "diff",
+    "patch",
+    "swap",
+    "revert",
+    "dot_lookup",
+    "__version__",
+)
+
+DICT_TYPES = (MutableMapping,)
+LIST_TYPES = (MutableSequence,)
+SET_TYPES = (MutableSet,)
+
+try:
+    import numpy
+
+    HAS_NUMPY = True
+    LIST_TYPES += (numpy.ndarray,)
+except ImportError:  # pragma: no cover
+    HAS_NUMPY = False
+
+
+def diff(
+    first,
+    second,
+    node=None,
+    ignore=None,
+    path_limit=None,
+    expand=False,
+    tolerance=EPSILON,
+    absolute_tolerance=None,
+    dot_notation=True,
+):
+    """Compare two dictionary/list/set objects, and returns a diff result.
+
+    Return an iterator with differences between two objects. The diff items
+    represent addition/deletion/change and the item value is a *deep copy*
+    from the corresponding source or destination objects.
+
+    >>> from dictdiffer import diff
+    >>> result = diff({'a': 'b'}, {'a': 'c'})
+    >>> list(result)
+    [('change', 'a', ('b', 'c'))]
+
+    The keys can be skipped from difference calculation when they are included
+    in ``ignore`` argument of type :class:`collections.Container`.
+
+    >>> list(diff({'a': 1, 'b': 2}, {'a': 3, 'b': 4}, ignore=set(['a'])))
+    [('change', 'b', (2, 4))]
+    >>> class IgnoreCase(set):
+    ...     def __contains__(self, key):
+    ...         return set.__contains__(self, str(key).lower())
+    >>> list(diff({'a': 1, 'b': 2}, {'A': 3, 'b': 4}, ignore=IgnoreCase('a')))
+    [('change', 'b', (2, 4))]
+
+    The difference calculation can be limitted to certain path:
+
+    >>> list(diff({}, {'a': {'b': 'c'}}))
+    [('add', '', [('a', {'b': 'c'})])]
+
+    >>> from dictdiffer.utils import PathLimit
+    >>> list(diff({}, {'a': {'b': 'c'}}, path_limit=PathLimit()))
+    [('add', '', [('a', {})]), ('add', 'a', [('b', 'c')])]
+
+    >>> from dictdiffer.utils import PathLimit
+    >>> list(diff({}, {'a': {'b': 'c'}}, path_limit=PathLimit([('a',)])))
+    [('add', '', [('a', {'b': 'c'})])]
+
+    >>> from dictdiffer.utils import PathLimit
+    >>> list(diff({}, {'a': {'b': 'c'}},
+    ...           path_limit=PathLimit([('a', 'b')])))
+    [('add', '', [('a', {})]), ('add', 'a', [('b', 'c')])]
+
+     >>> from dictdiffer.utils import PathLimit
+    >>> list(diff({'a': {'b': 'c'}}, {'a': {'b': 'c'}}, path_limit=PathLimit([('a',)])))
+    []
+
+    The patch can be expanded to small units e.g. when adding multiple values:
+
+    >>> list(diff({'fruits': []}, {'fruits': ['apple', 'mango']}))
+    [('add', 'fruits', [(0, 'apple'), (1, 'mango')])]
+
+    >>> list(diff({'fruits': []}, {'fruits': ['apple', 'mango']}, expand=True))
+    [('add', 'fruits', [(0, 'apple')]), ('add', 'fruits', [(1, 'mango')])]
+
+    >>> list(diff({'a': {'x': 1}}, {'a': {'x': 2}}))
+    [('change', 'a.x', (1, 2))]
+
+    >>> list(diff({'a': {'x': 1}}, {'a': {'x': 2}},
+    ... dot_notation=False))
+    [('change', ['a', 'x'], (1, 2))]
+
+    :param first: The original dictionary, ``list`` or ``set``.
+    :param second: New dictionary, ``list`` or ``set``.
+    :param node: Key for comparison that can be used in :func:`dot_lookup`.
+    :param ignore: Set of keys that should not be checked.
+    :param path_limit: List of path limit tuples or dictdiffer.utils.Pathlimit
+                       object to limit the diff recursion depth.
+                       A diff is still performed beyond the path_limit,
+                       but individual differences will be aggregated up to the path_limit.
+    :param expand: Expand the patches.
+    :param tolerance: Threshold to consider when comparing two float numbers.
+    :param absolute_tolerance: Absolute threshold to consider when comparing
+                               two float numbers.
+    :param dot_notation: Boolean to toggle dot notation on and off.
+
+    .. versionchanged:: 0.3
+       Added *ignore* parameter.
+
+    .. versionchanged:: 0.4
+       Arguments ``first`` and ``second`` can now contain a ``set``.
+
+    .. versionchanged:: 0.5
+       Added *path_limit* parameter.
+       Added *expand* paramter.
+       Added *tolerance* parameter.
+
+    .. versionchanged:: 0.7
+       Diff items are deep copies from its corresponding objects.
+       Argument *ignore* is always converted to a ``set``.
+
+    .. versionchanged:: 0.8
+        Added *dot_notation* parameter.
+    """
+    if path_limit is not None and not isinstance(path_limit, PathLimit):
+        path_limit = PathLimit(path_limit)
+
+    if isinstance(ignore, Iterable):
+
+        def _process_ignore_value(value):
+            if isinstance(value, int):
+                return (value,)
+            elif isinstance(value, list):
+                return tuple(value)
+            elif not dot_notation and isinstance(value, str):
+                return (value,)
+            return value
+
+        ignore = type(ignore)(_process_ignore_value(value) for value in ignore)
+
+    def dotted(node, default_type=list):
+        """Return dotted notation."""
+        if dot_notation and all(
+            map(lambda x: isinstance(x, str) and "." not in x, node)
+        ):
+            return ".".join(node)
+        else:
+            return default_type(node)
+
+    def _diff_recursive(_first, _second, _node=None):
+        _node = _node or []
+
+        dotted_node = dotted(_node)
+
+        differ = False
+
+        if isinstance(_first, DICT_TYPES) and isinstance(_second, DICT_TYPES):
+            # dictionaries are not hashable, we can't use sets
+            def check(key):
+                """Test if key in current node should be ignored."""
+                return ignore is None or (
+                    dotted(_node + [key], default_type=tuple) not in ignore
+                    and tuple(_node + [key]) not in ignore
+                )
+
+            intersection = [k for k in _first if k in _second and check(k)]
+            addition = [k for k in _second if k not in _first and check(k)]
+            deletion = [k for k in _first if k not in _second and check(k)]
+
+            differ = True
+
+        elif isinstance(_first, LIST_TYPES) and isinstance(_second, LIST_TYPES):
+            len_first = len(_first)
+            len_second = len(_second)
+
+            intersection = list(range(0, min(len_first, len_second)))
+            addition = list(range(min(len_first, len_second), len_second))
+            deletion = list(reversed(range(min(len_first, len_second), len_first)))
+
+            differ = True
+
+        elif isinstance(_first, SET_TYPES) and isinstance(_second, SET_TYPES):
+            # Deep copy is not necessary for hashable items.
+            addition = _second - _first
+            if len(addition):
+                yield ADD, dotted_node, [(0, addition)]
+            deletion = _first - _second
+            if len(deletion):
+                yield REMOVE, dotted_node, [(0, deletion)]
+
+            return  # stop here for sets
+
+        if differ:
+            # Compare if object is a dictionary or list.
+            #
+            # NOTE variables: intersection, addition, deletion contain only
+            # hashable types, hence they do not need to be deepcopied.
+            #
+            # Call again the parent function as recursive if dictionary have
+            # child objects.  Yields `add` and `remove` flags.
+            for key in intersection:
+                # if type is not changed,
+                # callees again diff function to compare.
+                # otherwise, the change will be handled as `change` flag.
+                if path_limit and path_limit.path_is_limit(_node + [key]):
+                    if _first[key] == _second[key]:
+                        return
+
+                    yield (
+                        CHANGE,
+                        _node + [key],
+                        (deepcopy(_first[key]), deepcopy(_second[key])),
+                    )
+                else:
+                    recurred = _diff_recursive(
+                        _first[key],
+                        _second[key],
+                        _node=_node + [key],
+                    )
+
+                    for diffed in recurred:
+                        yield diffed
+
+            if addition:
+                if path_limit:
+                    collect = []
+                    collect_recurred = []
+                    for key in addition:
+                        if not isinstance(
+                            _second[key], SET_TYPES + LIST_TYPES + DICT_TYPES
+                        ):
+                            collect.append((key, deepcopy(_second[key])))
+                        elif path_limit.path_is_limit(_node + [key]):
+                            collect.append((key, deepcopy(_second[key])))
+                        else:
+                            collect.append((key, _second[key].__class__()))
+                            recurred = _diff_recursive(
+                                _second[key].__class__(),
+                                _second[key],
+                                _node=_node + [key],
+                            )
+
+                            collect_recurred.append(recurred)
+
+                    if expand:
+                        for key, val in collect:
+                            yield ADD, dotted_node, [(key, val)]
+                    else:
+                        yield ADD, dotted_node, collect
+
+                    for recurred in collect_recurred:
+                        for diffed in recurred:
+                            yield diffed
+                else:
+                    if expand:
+                        for key in addition:
+                            yield ADD, dotted_node, [(key, deepcopy(_second[key]))]
+                    else:
+                        yield (
+                            ADD,
+                            dotted_node,
+                            [
+                                # for additions, return a list that consist with
+                                # two-pair tuples.
+                                (key, deepcopy(_second[key]))
+                                for key in addition
+                            ],
+                        )
+
+            if deletion:
+                if expand:
+                    for key in deletion:
+                        yield REMOVE, dotted_node, [(key, deepcopy(_first[key]))]
+                else:
+                    yield (
+                        REMOVE,
+                        dotted_node,
+                        [
+                            # for deletions, return the list of removed keys
+                            # and values.
+                            (key, deepcopy(_first[key]))
+                            for key in deletion
+                        ],
+                    )
+
+        else:
+            # Compare string and numerical types and yield `change` flag.
+            if are_different(_first, _second, tolerance, absolute_tolerance):
+                yield CHANGE, dotted_node, (deepcopy(_first), deepcopy(_second))
+
+    return _diff_recursive(first, second, node)
+
+
+def patch(diff_result, destination, in_place=False):
+    """Patch the diff result to the destination dictionary.
+
+    :param diff_result: Changes returned by ``diff``.
+    :param destination: Structure to apply the changes to.
+    :param in_place: By default, destination dictionary is deep copied
+                     before applying the patch, and the copy is returned.
+                     Setting ``in_place=True`` means that patch will apply
+                     the changes directly to and return the destination
+                     structure.
+    """
+    if not in_place:
+        destination = deepcopy(destination)
+
+    def add(node, changes):
+        for key, value in changes:
+            dest = dot_lookup(destination, node)
+            if isinstance(dest, LIST_TYPES):
+                dest.insert(key, value)
+            elif isinstance(dest, SET_TYPES):
+                dest |= value
+            else:
+                dest[key] = value
+
+    def change(node, changes):
+        dest = dot_lookup(destination, node, parent=True)
+        if isinstance(node, str):
+            last_node = node.split(".")[-1]
+        else:
+            last_node = node[-1]
+        if isinstance(dest, LIST_TYPES):
+            last_node = int(last_node)
+        _, value = changes
+        dest[last_node] = value
+
+    def remove(node, changes):
+        for key, value in changes:
+            dest = dot_lookup(destination, node)
+            if isinstance(dest, SET_TYPES):
+                dest -= value
+            elif isinstance(dest, LIST_TYPES):
+                dest[key] = RemovedObject()
+            else:
+                del dest[key]
+
+    patchers = {REMOVE: remove, ADD: add, CHANGE: change}
+
+    for action, node, changes in diff_result:
+        patchers[action](node, changes)
+
+    return strip_removed_objects(destination)
+
+
+def swap(diff_result):
+    """Swap the diff result.
+
+    It uses following mapping:
+
+    - remove -> add
+    - add -> remove
+
+    In addition, swap the changed values for `change` flag.
+
+        >>> from dictdiffer import swap
+        >>> swapped = swap([('add', 'a.b.c', [('a', 'b'), ('c', 'd')])])
+        >>> next(swapped)
+        ('remove', 'a.b.c', [('c', 'd'), ('a', 'b')])
+
+        >>> swapped = swap([('change', 'a.b.c', ('a', 'b'))])
+        >>> next(swapped)
+        ('change', 'a.b.c', ('b', 'a'))
+
+    """
+
+    def add(node, changes):
+        return REMOVE, node, list(reversed(changes))
+
+    def remove(node, changes):
+        return ADD, node, changes
+
+    def change(node, changes):
+        first, second = changes
+        return CHANGE, node, (second, first)
+
+    swappers = {REMOVE: remove, ADD: add, CHANGE: change}
+
+    for action, node, change in diff_result:
+        yield swappers[action](node, change)
+
+
+def revert(diff_result, destination, in_place=False):
+    """Call swap function to revert patched dictionary object.
+
+    Usage example:
+
+        >>> from dictdiffer import diff, revert
+        >>> first = {'a': 'b'}
+        >>> second = {'a': 'c'}
+        >>> revert(diff(first, second), second)
+        {'a': 'b'}
+
+    :param diff_result: Changes returned by ``diff``.
+    :param destination: Structure to apply the changes to.
+    :param in_place: By default, destination dictionary is deep
+                     copied before being reverted, and the copy
+                     is returned. Setting ``in_place=True`` means
+                     that revert will apply the changes directly to
+                     and return the destination structure.
+    """
+    return patch(swap(diff_result), destination, in_place)

inspire_dictdiffer/conflict.py

@@ -0,0 +1,80 @@
+# This file is part of Dictdiffer.
+#
+# Copyright (C) 2015 CERN.
+#
+# Dictdiffer is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Sub module to recognize conflicts in dictdiffer patches."""
+
+import itertools
+
+from .utils import get_path, is_super_path
+
+
+class Conflict:
+    """Wrapper class to store and handle two conflicting patches."""
+
+    def __init__(self, patch1, patch2):
+        """Initialize Conflict object.
+
+        :param patch1: First patch tuple
+        :param patch2: Second patch tuple
+        """
+        self.first_patch = patch1
+        self.second_patch = patch2
+        self.take = None
+
+    def take_patch(self):
+        """Return the patch determined by the *take* attribute."""
+        if self.take:
+            return self.first_patch if self.take == "f" else self.second_patch
+        raise Exception("Take attribute not set.")
+
+    def __repr__(self):
+        """Return string representation."""
+        return "Conflict({0}, {1})".format(self.first_patch, self.second_patch)
+
+
+class ConflictFinder:
+    """Responsible for finding conflicting patches."""
+
+    def _is_conflict(self, patch1, patch2):
+        """Decide on a conflict between two patches.
+
+        The conditions are:
+        1. The paths are identical
+        2. On of the paths is the super path of the other
+
+        :param patch1: First patch tuple
+        :param patch2: First patch tuple
+        """
+        path1 = get_path(patch1)
+        path2 = get_path(patch2)
+
+        if path1 == path2:
+            return True
+        elif is_super_path(path1, path2) and patch1[0] == "remove":
+            return True
+        elif is_super_path(path2, path1) and patch2[0] == "remove":
+            return True
+
+        return False
+
+    def find_conflicts(self, first_patches, second_patches):
+        """Find all conflicts between two lists of patches.
+
+        Iterates over the lists of patches, comparing each patch from list
+        one to each patch from list two.
+
+        :param first_patches: List of patch tuples
+        :param second_patches: List of patch tuples
+        """
+        self.conflicts = [
+            Conflict(patch1, patch2)
+            for patch1, patch2 in itertools.product(first_patches, second_patches)
+            if self._is_conflict(patch1, patch2)
+        ]
+
+        return self.conflicts

inspire_dictdiffer/merge.py

@@ -0,0 +1,163 @@
+# This file is part of Dictdiffer.
+#
+# Copyright (C) 2015 CERN.
+# Copyright (C) 2017 ETH Zurich, Swiss Data Science Center, Jiri Kuncar.
+#
+# Dictdiffer is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more
+# details.
+
+"""Sub module to handle the merging of dictdiffer patches."""
+
+from . import diff
+from .conflict import ConflictFinder
+from .resolve import Resolver, UnresolvedConflictsException
+from .unify import Unifier
+from .utils import PathLimit
+
+
+class Merger:
+    """Class wrapping steps of the automated merging process.
+
+    Usage:
+        >>> lca = {}
+        >>> first = {'foo': 'bar'}
+        >>> second = {'bar': 'foo'}
+        >>> path_limits = []
+        >>> actions = {}
+        >>> additional_info = {}
+        >>> m = Merger(lca, first, second, actions,
+        ...            path_limits, additional_info)
+        >>> try:
+        ...     m.run()
+        ... except UnresolvedConflictsException:
+        ...     # fix the conflicts
+        ...     m.continue_run()
+    """
+
+    def __init__(
+        self,
+        lca,
+        first,
+        second,
+        actions,
+        path_limits=[],
+        additional_info=None,
+        ignore=None,
+    ):
+        """Initialize the Merger object.
+
+        :param lca: latest common ancestor of the two diverging data structures
+        :param first: first data structure
+        :param second: second data structure
+        :param path_limits: list of paths, utilized to instantiate a
+                            dictdiffer.utils.PathLimit object
+        :param additional_info: Any object containing additional information
+                                used by the resolution functions
+        :param ignore: Set of keys that should not be merged
+        """
+        self.lca = lca
+        self.first = first
+        self.second = second
+        self.path_limit = PathLimit(path_limits)
+        self.ignore = ignore
+
+        self.actions = actions
+        self.additional_info = additional_info
+
+        self.conflict_finder = ConflictFinder()
+
+        self.resolver = Resolver(self.actions, self.additional_info)
+
+        self.unifier = Unifier()
+
+        self.conflicts = []
+        self.unresolved_conflicts = []
+
+    def run(self):
+        """Run the automated merging process.
+
+        Runs every step necessary for the automated merging process, raising
+        an UnresolvedConflictsException in case that the provided resolution
+        actions can not solve a given conflict.
+
+        After every performed step, the results are stored inside attributes of
+        the merger object.
+        """
+        self.extract_patches()
+        self.find_conflicts()
+        self.resolve_conflicts()
+
+        if self.unresolved_conflicts:
+            raise UnresolvedConflictsException(self.unresolved_conflicts)
+
+        self.unify_patches()
+
+    def continue_run(self, picks):
+        """Continue the merge after an UnresolvedConflictsException.
+
+        :param picks: a list of 'f' or 's' strings, which utilize the Conflicts
+                      class *take* attribute
+        """
+        self.resolver.manual_resolve_conflicts(picks)
+        self.unresolved_conflicts = []
+        self.unify_patches()
+
+    def extract_patches(self):
+        """Extract the patches.
+
+        Extracts the differences between the *lca* and the *first* and
+        *second* data structure and stores them in the attributes
+        *first_patches* and *second_patches*.
+        """
+        self.first_patches = list(
+            diff(
+                self.lca,
+                self.first,
+                path_limit=self.path_limit,
+                ignore=self.ignore,
+                expand=True,
+            )
+        )
+        self.second_patches = list(
+            diff(
+                self.lca,
+                self.second,
+                path_limit=self.path_limit,
+                ignore=self.ignore,
+                expand=True,
+            )
+        )
+
+    def find_conflicts(self):
+        """Find conflicts between the tow lists of patches.
+
+        Finds the conflicts between the two difference lists and stores
+        them in the *conflicts* attribute.
+        """
+        self.conflicts = self.conflict_finder.find_conflicts(
+            self.first_patches, self.second_patches
+        )
+
+    def resolve_conflicts(self):
+        """Resolve the conflicts.
+
+        Runs the automated conflict resolution process.
+        Occurring unresolvable conflicts are stored in *unresolved_conflicts*.
+        """
+        try:
+            self.resolver.resolve_conflicts(
+                self.first_patches, self.second_patches, self.conflicts
+            )
+        except UnresolvedConflictsException as e:
+            self.unresolved_conflicts = e.content
+
+    def unify_patches(self):
+        """Unify the patches after the conflict resolution.
+
+        Unifies the patches after a successful merge and stores them in
+        *unified_patches*.
+        """
+        self.unified_patches = self.unifier.unify(
+            self.first_patches, self.second_patches, self.conflicts
+        )