dcs-sdk 1.6.5__py3-none-any.whl

data_diff/lexicographic_space.py

@@ -0,0 +1,253 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""Contains the implementation of two classes:
+
+- LexicographicSpace - a lexicographic space of arbitrary dimensions.
+- BoundedLexicographicSpace - a lexicographic space, where the lowest point may be non-zero.
+
+A lexicographic space is a space of increasing natural values, ordered by lexicographic order.
+Read more: https://mathworld.wolfram.com/LexicographicOrder.html
+
+These abstractions were written to support compound keys in the hashdiff algorithm.
+In the hashdiff algorithm, we rely on the order of the column keys, to segment the table correctly.
+SQL orders the columns of tables based on lexicographic ordering.
+Since we need an evenly spaced "range" function over the space, which has arbitrary dimensions, we have
+to implement it ourself.
+
+As a further optimization, since we each time operate on segments of the ordered table, we add support
+for working with a restricted space, which will reduce the likelihood of gaps in our "select", when the
+keys are not evenly distributed.
+"""
+
+from random import randint, randrange
+from typing import Tuple
+
+import attrs
+
+from data_diff.utils import safezip
+
+Vector = Tuple[int]
+Interval = Tuple[int]
+
+
+class Overflow(ValueError):
+    pass
+
+
+def neg_interval(interval):
+    return tuple(-i for i in interval)
+
+
+def neg_v(v: Vector):
+    return tuple(-i for i in v)
+
+
+def sub_v(v1: Vector, v2: Vector):
+    return tuple(i1 - i2 for i1, i2 in safezip(v1, v2))
+
+
+def add_v(v1: Vector, v2: Vector):
+    return tuple(i1 + i2 for i1, i2 in safezip(v1, v2))
+
+
+def rand_v_in_range(v1: Vector, v2: Vector):
+    return tuple(irandrange(i1, i2) for i1, i2 in safezip(v1, v2))
+
+
+def irandrange(start, stop):
+    if start == stop:
+        return start
+    return randrange(start, stop)
+
+
+@attrs.define(frozen=True)
+class LexicographicSpace:
+    """Lexicographic space of arbitrary dimensions.
+
+    All elements must be of the same length as the number of dimensions. (no rpadding)
+    """
+
+    def __init__(self, dims: Vector) -> None:
+        super().__init__()
+        self.dims = dims
+
+    def __contains__(self, v: Vector) -> bool:
+        return all(0 <= i < d for i, d in safezip(v, self.dims))
+
+    def add(self, v1: Vector, v2: Vector) -> Vector:
+        # assert v1 in self and v2 in self, (v1, v2)
+
+        carry = 0
+        res = []
+        for i1, i2, d in reversed(list(safezip(v1, v2, self.dims))):
+            n = i1 + i2 + carry
+            carry = n // d
+            assert carry <= 1
+            n %= d
+            res.append(n)
+
+        if carry:
+            raise Overflow("Overflow")
+
+        new_v = tuple(reversed(res))
+        assert new_v in self
+        return new_v
+
+    def sub(self, v1: Vector, v2: Vector):
+        return self.add(v1, neg_v(v2))
+
+    def _divide(self, v: Vector, count: int):
+        n = 0
+        for x, d in zip(v, self.dims[1:] + (1,), strict=True):
+            x += n
+            rem = x % count
+            n = rem * d
+            yield x // count
+
+    def divide(self, v: Vector, count: int) -> Vector:
+        return tuple(self._divide(v, count))
+
+    def range(self, min_value: Vector, max_value: Vector, count: int):
+        assert min_value in self and max_value in self
+        count -= 1
+        size = self.sub(max_value, min_value)
+        interval = self.divide(size, count)
+        n = min_value
+        for i in range(count):
+            yield n
+            n = self.add(n, interval)
+        yield n
+
+
+class BoundedLexicographicSpace:
+    """Lexicographic space of arbitrary dimensions, where the lowest point may be non-zero.
+
+    i.e. a space resticted by a "bounding-box" between two arbitrary points.
+    """
+
+    def __init__(self, min_bound: Vector, max_bound: Vector) -> None:
+        super().__init__()
+
+        dims = tuple(mx - mn for mn, mx in safezip(min_bound, max_bound))
+        if not all(d >= 0 for d in dims):
+            raise ValueError("Error: Negative dimension!")
+        if not (dims[0] > 0):
+            raise ValueError("First dimension must be non-zero!")
+
+        self.min_bound = min_bound
+        self.max_bound = max_bound
+
+        self.uspace = LexicographicSpace(dims)
+
+    def __contains__(self, p: Vector) -> bool:
+        return all(mn <= i < mx for i, mn, mx in safezip(p, self.min_bound, self.max_bound))
+
+    def to_uspace(self, v: Vector) -> Vector:
+        assert v in self
+        return sub_v(v, self.min_bound)
+
+    def from_uspace(self, v: Vector) -> Vector:
+        res = add_v(v, self.min_bound)
+        assert res in self
+        return res
+
+    def add_interval(self, v1: Vector, interval: Interval) -> Vector:
+        return self.from_uspace(self.uspace.add(self.to_uspace(v1), interval))
+
+    def sub_interval(self, v1: Vector, interval: Interval) -> Vector:
+        return self.from_uspace(self.uspace.sub(self.to_uspace(v1), interval))
+
+    def sub(self, v1: Vector, v2: Vector) -> Interval:
+        return self.uspace.sub(self.to_uspace(v1), self.to_uspace(v2))
+
+    def range(self, min_value: Vector, max_value: Vector, count: int):
+        return [
+            self.from_uspace(v) for v in self.uspace.range(self.to_uspace(min_value), self.to_uspace(max_value), count)
+        ]
+
+
+def test_lex_space():
+    # Test add
+    binspace = LexicographicSpace((2, 2, 2, 2))
+    zero = (0, 0, 0, 0)
+    one = (0, 0, 0, 1)
+    bin_nums = [zero]
+    for i in range(15):
+        last = bin_nums[-1]
+        bin_nums.append(binspace.add(last, one))
+    five = bin_nums[5]
+    seven = bin_nums[7]
+    eight = bin_nums[8]
+    fifteen = bin_nums[15]
+
+    assert binspace.add(binspace.add(one, five), one) == seven
+    assert binspace.add(one, seven) == eight
+    assert binspace.add(seven, eight) == fifteen
+
+    assert binspace.sub(eight, one) == seven
+    assert binspace.sub(fifteen, seven) == eight
+
+    r = list(binspace.range(one, seven, 4))
+    assert r == [one, bin_nums[3], five, seven], r
+
+    decspace = LexicographicSpace((10, 10, 10))
+    assert decspace.divide((4, 5, 2), 2) == (2, 2, 6)
+    assert decspace.divide((3, 0, 2), 2) == (1, 5, 1)
+
+    # Restricted space
+
+    rspace1 = BoundedLexicographicSpace((2, 2), (8, 8))
+    assert rspace1.add_interval((2, 2), (0, 0)) == (2, 2)
+    assert rspace1.add_interval((2, 2), (0, 1)) == (2, 3)
+    assert rspace1.add_interval((2, 2), (0, 6)) == (3, 2)
+    assert rspace1.add_interval((2, 2), (0, 7)) == (3, 3)
+    # space.add((2,2), (6, 0))    # Overflow
+
+    rspace2 = BoundedLexicographicSpace((4, 4, 4, 4), (6, 6, 6, 6))
+    _one = (4, 4, 4, 5)
+    _three = (4, 4, 5, 5)
+    _five = (4, 5, 4, 5)
+    _seven = (4, 5, 5, 5)
+    assert rspace2.add_interval(rspace2.add_interval(_five, one), one) == _seven
+    assert rspace2.sub_interval(rspace2.sub_interval(_seven, one), one) == _five
+
+    r = list(rspace2.range(_one, _seven, 4))
+    assert r == [_one, _three, _five, _seven], r
+
+    # Test range -
+    # For random bounds and min/max values, assert that range() generates steps with uniform distances
+    MAX_COLUMNS = 16
+    MAX_DIM = 10000
+    MAX_BISECTION = 128
+
+    for n in range(1, MAX_COLUMNS):
+        min_bound = tuple(randint(0, MAX_DIM) for i in range(n))
+        size = tuple(randint(1, MAX_DIM) for i in range(n))
+        max_bound = add_v(min_bound, size)
+
+        sp = BoundedLexicographicSpace(min_bound, max_bound)
+
+        max_value = rand_v_in_range(min_bound, max_bound)
+        min_value = rand_v_in_range(min_bound, max_value)
+        for count in range(2, MAX_BISECTION):
+            r = sp.range(min_value, max_value, count)
+            assert len(r) == count
+            diffs = [sp.sub(b, a) for a, b in zip(r[:-1], r[1:])]
+            assert len(set(diffs)) == 1  # Uniform!
+            # print('.', end='')
+
+
+if __name__ == "__main__":
+    test_lex_space()

data_diff/parse_time.py

@@ -0,0 +1,88 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import re
+from datetime import datetime, timedelta
+from difflib import SequenceMatcher
+
+
+class ParseError(ValueError):
+    pass
+
+
+TIME_UNITS = dict(
+    seconds="seconds",
+    minutes="minutes",
+    hours="hours",
+    days="days",
+    weeks="weeks",
+    months="months",
+    years="years",
+    # Shortcuts
+    s="seconds",
+    min="minutes",
+    h="hours",
+    d="days",
+    w="weeks",
+    mon="months",
+    y="years",
+)
+
+EXTRAPOLATED = {"months": (30, "days"), "years": (365, "days")}
+assert set(EXTRAPOLATED) <= set(TIME_UNITS)
+
+TIME_RE = re.compile(r"(\d+)([a-z]+)")
+
+UNITS_STR = ", ".join(sorted(TIME_UNITS.keys()))
+
+
+def string_similarity(a, b) -> SequenceMatcher:
+    return SequenceMatcher(None, a, b).ratio()
+
+
+def parse_time_atom(count, unit):
+    count = int(count)
+    try:
+        unit = TIME_UNITS[unit]
+    except KeyError:
+        most_similar = max(TIME_UNITS, key=lambda k: string_similarity(k, unit))
+        raise ParseError(
+            f"'{unit}' is not a recognized time unit. Did you mean '{most_similar}'?" f"\nSupported units: {UNITS_STR}"
+        )
+
+    if unit in EXTRAPOLATED:
+        mul, unit = EXTRAPOLATED[unit]
+        count *= mul
+    return count, unit
+
+
+def parse_time_delta(t: str) -> timedelta:
+    time_dict = {}
+    while t:
+        m = TIME_RE.match(t)
+        if not m:
+            raise ParseError(f"Cannot parse '{t}': Not a recognized time delta")
+        count, unit = parse_time_atom(*m.groups())
+        if unit in time_dict:
+            raise ParseError(f"Time unit {unit} specified more than once")
+        time_dict[unit] = count
+        t = t[m.end() :]
+
+    if not time_dict:
+        raise ParseError("No time difference specified")
+    return timedelta(**time_dict)
+
+
+def parse_time_before(time: datetime, delta: str) -> datetime:
+    return time - parse_time_delta(delta)

data_diff/queries/__init__.py

@@ -0,0 +1,13 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

data_diff/queries/api.py

@@ -0,0 +1,213 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from data_diff.queries.ast_classes import *
+from data_diff.queries.base import args_as_tuple
+from data_diff.utils import CaseAwareMapping, CaseSensitiveDict
+
+this = This()
+
+
+def join(*tables: ITable) -> Join:
+    """Inner-join a sequence of table expressions"
+
+    When joining, it's recommended to use explicit tables names, instead of `this`, in order to avoid potential name collisions.
+
+    Example:
+        ::
+
+            person = table('person')
+            city = table('city')
+
+            name_and_city = (
+                join(person, city)
+                .on(person['city_id'] == city['id'])
+                .select(person['id'], city['name'])
+            )
+    """
+    return Join(tables)
+
+
+def leftjoin(*tables: ITable) -> Join:
+    """Left-joins a sequence of table expressions.
+
+    See Also: ``join()``
+    """
+    return Join(tables, "LEFT")
+
+
+def rightjoin(*tables: ITable) -> Join:
+    """Right-joins a sequence of table expressions.
+
+    See Also: ``join()``
+    """
+    return Join(tables, "RIGHT")
+
+
+def outerjoin(*tables: ITable) -> Join:
+    """Outer-joins a sequence of table expressions.
+
+    See Also: ``join()``
+    """
+    return Join(tables, "FULL OUTER")
+
+
+def cte(expr: Expr, *, name: Optional[str] = None, params: Sequence[str] = None) -> Cte:
+    """Define a CTE"""
+    return Cte(expr, name, params)
+
+
+def table(*path: str, schema: Union[dict, CaseAwareMapping] = None) -> TablePath:
+    """Defines a table with a path (dotted name), and optionally a schema.
+
+    Parameters:
+        path: A list of names that make up the path to the table.
+        schema: a dictionary of {name: type}
+    """
+    if len(path) == 1 and isinstance(path[0], tuple):
+        (path,) = path
+    if not all(isinstance(i, str) for i in path):
+        raise TypeError(f"All elements of table path must be of type 'str'. Got: {path}")
+    if schema and not isinstance(schema, CaseAwareMapping):
+        assert isinstance(schema, dict)
+        schema = CaseSensitiveDict(schema)
+    return TablePath(path, schema)
+
+
+def or_(*exprs: Expr) -> Union[BinBoolOp, Expr]:
+    """Apply OR between a sequence of boolean expressions"""
+    exprs = args_as_tuple(exprs)
+    if len(exprs) == 1:
+        return exprs[0]
+    return BinBoolOp("OR", exprs)
+
+
+def and_(*exprs: Expr) -> Union[BinBoolOp, Expr]:
+    """Apply AND between a sequence of boolean expressions"""
+    exprs = args_as_tuple(exprs)
+    if len(exprs) == 1:
+        return exprs[0]
+    return BinBoolOp("AND", exprs)
+
+
+def sum_(expr: Expr) -> Func:
+    """Call SUM(expr)"""
+    return Func("sum", [expr])
+
+
+def avg(expr: Expr) -> Func:
+    """Call AVG(expr)"""
+    return Func("avg", [expr])
+
+
+def min_(expr: Expr) -> Func:
+    """Call MIN(expr)"""
+    return Func("min", [expr])
+
+
+def max_(expr: Expr) -> Func:
+    """Call MAX(expr)"""
+    return Func("max", [expr])
+
+
+def exists(expr: Expr) -> Func:
+    """Call EXISTS(expr)"""
+    return Func("exists", [expr])
+
+
+def if_(cond: Expr, then: Expr, else_: Optional[Expr] = None) -> CaseWhen:
+    """Conditional expression, shortcut to when-then-else.
+
+    Example:
+        ::
+
+            # SELECT CASE WHEN b THEN c ELSE d END FROM foo
+            table('foo').select(if_(b, c, d))
+    """
+    return when(cond).then(then).else_(else_)
+
+
+def when(*when_exprs: Expr) -> QB_When:
+    """Start a when-then expression
+
+    Example:
+        ::
+
+            # SELECT CASE
+            #   WHEN (type = 'text') THEN text
+            #   WHEN (type = 'number') THEN number
+            #   ELSE 'unknown type' END
+            # FROM foo
+            rows = table('foo').select(
+                    when(this.type == 'text').then(this.text)
+                    .when(this.type == 'number').then(this.number)
+                    .else_('unknown type')
+                )
+    """
+    return CaseWhen([]).when(*when_exprs)
+
+
+def coalesce(*exprs) -> Func:
+    "Returns a call to COALESCE"
+    exprs = args_as_tuple(exprs)
+    return Func("COALESCE", exprs)
+
+
+def insert_rows_in_batches(db, tbl: TablePath, rows, *, columns=None, batch_size=1024 * 8) -> None:
+    assert batch_size > 0
+    rows = list(rows)
+
+    while rows:
+        batch, rows = rows[:batch_size], rows[batch_size:]
+        db.query(tbl.insert_rows(batch, columns=columns))
+
+
+def current_timestamp() -> CurrentTimestamp:
+    """Returns CURRENT_TIMESTAMP() or NOW()"""
+    return CurrentTimestamp()
+
+
+def code(code: str, **kw: Dict[str, Expr]) -> Code:
+    """Inline raw SQL code.
+
+    It allows users to use features and syntax that Sqeleton doesn't yet support.
+
+    It's the user's responsibility to make sure the contents of the string given to `code()` are correct and safe for execution.
+
+    Strings given to `code()` are actually templates, and can embed query expressions given as arguments:
+
+    Parameters:
+        code: template string of SQL code. Templated variables are signified with '{var}'.
+        kw: optional parameters for SQL template.
+
+    Examples:
+        ::
+
+            # SELECT b, <x> FROM tmp WHERE <y>
+            table('tmp').select(this.b, code("<x>")).where(code("<y>"))
+
+        ::
+
+            def tablesample(tbl, size):
+                return code("SELECT * FROM {tbl} TABLESAMPLE BERNOULLI ({size})", tbl=tbl, size=size)
+
+            nonzero = table('points').where(this.x > 0, this.y > 0)
+
+            # SELECT * FROM points WHERE (x > 0) AND (y > 0) TABLESAMPLE BERNOULLI (10)
+            sample_expr = tablesample(nonzero)
+    """
+    return Code(code, kw)
+
+
+commit = Commit()