dcs-sdk 1.6.5__py3-none-any.whl

data_diff/abcs/database_types.py

@@ -0,0 +1,402 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import decimal
+from abc import ABC, abstractmethod
+from datetime import date, datetime
+from typing import Collection, List, Optional, Tuple, Type, TypeVar, Union
+
+import attrs
+
+from data_diff.utils import (
+    ArithAlphanumeric,
+    ArithDate,
+    ArithDateTime,
+    ArithTimestamp,
+    ArithTimestampTZ,
+    ArithUnicodeString,
+    ArithUUID,
+    Unknown,
+)
+
+DbPath = Tuple[str, ...]
+DbKey = Union[
+    int,
+    str,
+    bytes,
+    ArithUUID,
+    ArithAlphanumeric,
+    ArithUnicodeString,
+    ArithDateTime,
+    ArithDate,
+    ArithTimestamp,
+    ArithTimestampTZ,
+]
+DbTime = datetime
+
+N = TypeVar("N")
+
+
+@attrs.frozen(kw_only=True, eq=False, order=False, unsafe_hash=True)
+class Collation:
+    """
+    A pre-parsed or pre-known record about db collation, per column.
+
+    The "greater" collation should be used as a target collation for textual PKs
+    on both sides of the diff — by coverting the "lesser" collation to self.
+
+    Snowflake easily absorbs the performance losses, so it has a boost to always
+    be greater than any other collation in non-Snowflake databases.
+    Other databases need to negotiate which side absorbs the performance impact.
+    """
+
+    # A boost for special databases that are known to absorb the performance dmaage well.
+    absorbs_damage: bool = False
+
+    # Ordinal soring by ASCII/UTF8 (True), or alphabetic as per locale/country/etc (False).
+    ordinal: Optional[bool] = None
+
+    # Lowercase first (aAbBcC or abcABC). Otherwise, uppercase first (AaBbCc or ABCabc).
+    lower_first: Optional[bool] = None
+
+    # 2-letter lower-case locale and upper-case country codes, e.g. en_US. Ignored for ordinals.
+    language: Optional[str] = None
+    country: Optional[str] = None
+
+    # There are also space-, punctuation-, width-, kana-(in)sensitivity, so on.
+    # Ignore everything not related to xdb alignment. Only case- & accent-sensitivity are common.
+    case_sensitive: Optional[bool] = None
+    accent_sensitive: Optional[bool] = None
+
+    # Purely informational, for debugging:
+    _source: Union[None, str, Collection[str]] = None
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Collation):
+            return NotImplemented
+        if self.ordinal and other.ordinal:
+            # TODO: does it depend on language? what does Albanic_BIN mean in MS SQL?
+            return True
+        return (
+            self.language == other.language
+            and (self.country is None or other.country is None or self.country == other.country)
+            and self.case_sensitive == other.case_sensitive
+            and self.accent_sensitive == other.accent_sensitive
+            and self.lower_first == other.lower_first
+        )
+
+    def __ne__(self, other: object) -> bool:
+        if not isinstance(other, Collation):
+            return NotImplemented
+        return not self.__eq__(other)
+
+    def __gt__(self, other: object) -> bool:
+        if not isinstance(other, Collation):
+            return NotImplemented
+        if self == other:
+            return False
+        if self.absorbs_damage and not other.absorbs_damage:
+            return False
+        if other.absorbs_damage and not self.absorbs_damage:
+            return True  # this one is preferred if it cannot absorb damage as its counterpart can
+        if self.ordinal and not other.ordinal:
+            return True
+        if other.ordinal and not self.ordinal:
+            return False
+        # TODO: try to align the languages & countries?
+        return False
+
+    def __ge__(self, other: object) -> bool:
+        if not isinstance(other, Collation):
+            return NotImplemented
+        return self == other or self.__gt__(other)
+
+    def __lt__(self, other: object) -> bool:
+        if not isinstance(other, Collation):
+            return NotImplemented
+        return self != other and not self.__gt__(other)
+
+    def __le__(self, other: object) -> bool:
+        if not isinstance(other, Collation):
+            return NotImplemented
+        return self == other or not self.__gt__(other)
+
+
+@attrs.define(frozen=True, kw_only=True)
+class ColType:
+    # Arbitrary metadata added and fetched at runtime.
+    _notes: List[N] = attrs.field(factory=list, init=False, hash=False, eq=False)
+
+    def add_note(self, note: N) -> None:
+        self._notes.append(note)
+
+    def get_note(self, cls: Type[N]) -> Optional[N]:
+        """Get the latest added note of type ``cls`` or its descendants."""
+        for note in reversed(self._notes):
+            if isinstance(note, cls):
+                return note
+        return None
+
+    @property
+    def supported(self) -> bool:
+        return True
+
+
+@attrs.define(frozen=True)
+class PrecisionType(ColType):
+    precision: int
+    rounds: Union[bool, Unknown] = Unknown
+
+
+@attrs.define(frozen=True)
+class Boolean(ColType):
+    precision = 0
+
+
+@attrs.define(frozen=True)
+class TemporalType(PrecisionType):
+    pass
+
+
+@attrs.define(frozen=True)
+class IKey(ABC):
+    "Interface for ColType, for using a column as a key in table."
+
+    @property
+    @abstractmethod
+    def python_type(self) -> type:
+        "Return the equivalent Python type of the key"
+
+    def make_value(self, value):
+        if isinstance(value, self.python_type):
+            return value
+        return self.python_type(value)
+
+
+@attrs.define(frozen=True)
+class Timestamp(TemporalType, IKey):
+    @property
+    def python_type(self) -> type:
+        return ArithTimestamp
+
+    def make_value(self, value):
+        if isinstance(value, ArithTimestamp):
+            return value
+        return ArithTimestamp(value)
+
+
+@attrs.define(frozen=True)
+class TimestampTZ(TemporalType, IKey):
+    @property
+    def python_type(self) -> type:
+        return ArithTimestampTZ
+
+    def make_value(self, value):
+        if isinstance(value, ArithTimestampTZ):
+            return value
+        return ArithTimestampTZ(value)
+
+
+@attrs.define(frozen=True)
+class Datetime(TemporalType, IKey):
+    @property
+    def python_type(self) -> type:
+        return ArithDateTime
+
+    def make_value(self, value):
+        if isinstance(value, ArithDateTime):
+            return value
+        return ArithDateTime(value)
+
+
+@attrs.define(frozen=True)
+class Date(TemporalType, IKey):
+    @property
+    def python_type(self) -> type:
+        return ArithDate
+
+    def make_value(self, value):
+        if isinstance(value, ArithDate):
+            return value
+        return ArithDate(value)
+
+
+@attrs.define(frozen=True)
+class Time(TemporalType):
+    pass
+
+
+@attrs.define(frozen=True)
+class NumericType(ColType):
+    # 'precision' signifies how many fractional digits (after the dot) we want to compare
+    precision: int
+
+
+@attrs.define(frozen=True)
+class FractionalType(NumericType):
+    pass
+
+
+@attrs.define(frozen=True)
+class Float(FractionalType):
+    python_type = float
+
+
+@attrs.define(frozen=True)
+class Decimal(FractionalType, IKey):  # Snowflake may use Decimal as a key
+    @property
+    def python_type(self) -> type:
+        if self.precision == 0:
+            return int
+        return decimal.Decimal
+
+
+@attrs.define(frozen=True)
+class Float(FractionalType, IKey):
+    @property
+    def python_type(self) -> type:
+        if self.precision == 0:
+            return int
+        return decimal.Decimal
+
+
+@attrs.define(frozen=True)
+class Numeric(FractionalType, IKey):
+    @property
+    def python_type(self) -> type:
+        return int
+
+
+@attrs.define(frozen=True)
+class StringType(ColType):
+    python_type = str
+    collation: Optional[Collation] = attrs.field(default=None, kw_only=True)
+
+
+@attrs.define(frozen=True)
+class ColType_UUID(ColType, IKey):
+    python_type = ArithUUID
+
+
+@attrs.define(frozen=True)
+class ColType_Alphanum(ColType, IKey):
+    python_type = ArithAlphanumeric
+
+
+@attrs.define(frozen=True)
+class ColType_Unicode(ColType, IKey):
+    python_type = ArithUnicodeString
+
+
+@attrs.define(frozen=True)
+class Native_UUID(ColType_UUID):
+    pass
+
+
+@attrs.define(frozen=True)
+class String_UUID(ColType_UUID, StringType):
+    # Case is important for UUIDs stored as regular string, not native UUIDs stored as numbers.
+    # We slice them internally as numbers, but render them back to SQL as lower/upper case.
+    # None means we do not know for sure, behave as with False, but it might be unreliable.
+    lowercase: Optional[bool] = None
+    uppercase: Optional[bool] = None
+
+    def make_value(self, v: str) -> ArithUUID:
+        return self.python_type(v, lowercase=self.lowercase, uppercase=self.uppercase)
+
+
+@attrs.define(frozen=True)
+class String_Alphanum(ColType_Alphanum, StringType):
+    @staticmethod
+    def test_value(value: str) -> bool:
+        try:
+            ArithAlphanumeric(value)
+            return True
+        except ValueError:
+            return False
+
+    def make_value(self, value) -> ArithAlphanumeric:
+        if isinstance(value, ArithAlphanumeric):
+            return value
+        # Coerce non-string primitives (e.g., integers) to string for alphanumeric representation
+        return ArithAlphanumeric(str(value))
+
+
+@attrs.define(frozen=True)
+class String_VaryingAlphanum(String_Alphanum):
+    pass
+
+
+@attrs.define(frozen=True)
+class String_VaryingUnicode(ColType_Unicode, StringType):
+    pass
+
+
+@attrs.define(frozen=True)
+class String_FixedAlphanum(String_Alphanum):
+    length: int
+
+    def make_value(self, value):
+        if isinstance(value, self.python_type):
+            return value
+        if len(value) != self.length:
+            raise ValueError(f"Expected alphanumeric value of length {self.length}, but got '{value}'.")
+        return self.python_type(value, max_len=self.length)
+
+
+@attrs.define(frozen=True)
+class Text(ColType_Unicode, StringType):
+
+    @property
+    def supported(self) -> bool:
+        return False
+
+
+# In majority of DBMSes, it is called JSON/JSONB. Only in Snowflake, it is OBJECT.
+@attrs.define(frozen=True)
+class JSON(ColType):
+    pass
+
+
+@attrs.define(frozen=True)
+class Array(ColType):
+    item_type: ColType
+
+
+# Unlike JSON, structs are not free-form and have a very specific set of fields and their types.
+# We do not parse & use those fields now, but we can do this later.
+# For example, in BigQuery:
+# - https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#struct_type
+# - https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#struct_literals
+@attrs.define(frozen=True)
+class Struct(ColType):
+    pass
+
+
+@attrs.define(frozen=True)
+class Integer(NumericType, IKey):
+    precision: int = 0
+    python_type: type = int
+
+    def __attrs_post_init__(self) -> None:
+        assert self.precision == 0
+
+
+@attrs.define(frozen=True)
+class UnknownColType(ColType):
+    text: str
+
+    @property
+    def supported(self) -> bool:
+        return False

data_diff/config.py

@@ -0,0 +1,141 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+import re
+from typing import Any, Dict
+
+import toml
+
+_ARRAY_FIELDS = (
+    "key_columns",
+    "columns",
+)
+
+
+class ConfigParseError(Exception):
+    pass
+
+
+def is_uri(s: str) -> bool:
+    return "://" in s
+
+
+def _apply_config(config: Dict[str, Any], run_name: str, kw: Dict[str, Any]):
+    _resolve_env(config)
+
+    # Load config
+    databases = config.pop("database", {})
+    runs = config.pop("run", {})
+    if config:
+        raise ConfigParseError(f"Unknown option(s): {config}")
+
+    # Init run_args
+    run_args = runs.get("default") or {}
+    if run_name:
+        if run_name not in runs:
+            raise ConfigParseError(f"Cannot find run '{run_name}' in configuration.")
+        run_args.update(runs[run_name])
+    else:
+        run_name = "default"
+
+    if kw.get("database1") is not None:
+        for attr in ("table1", "database2", "table2"):
+            if kw[attr] is None:
+                raise ValueError(f"Specified database1 but not {attr}. Must specify all 4 arguments, or neither.")
+
+        for index in "12":
+            run_args[index] = {attr: kw.pop(f"{attr}{index}") for attr in ("database", "table")}
+
+    # Make sure array fields are decoded as list, since array fields in toml are decoded as list, but TableSegment object requires tuple type.
+    for field in _ARRAY_FIELDS:
+        if isinstance(run_args.get(field), list):
+            run_args[field] = tuple(run_args[field])
+
+    # Process databases + tables
+    for index in "12":
+        try:
+            args = run_args.pop(index)
+        except KeyError:
+            raise ConfigParseError(
+                f"Could not find source #{index}: Expecting a key of '{index}' containing '.database' and '.table'."
+            )
+        for attr in ("database", "table"):
+            if attr not in args:
+                raise ConfigParseError(f"Running 'run.{run_name}': Connection #{index} is missing attribute '{attr}'.")
+
+        database = args.pop("database")
+        table = args.pop("table")
+        threads = args.pop("threads", None)
+        if args:
+            raise ConfigParseError(f"Unexpected attributes for connection #{index}: {args}")
+
+        if not is_uri(database):
+            if database not in databases:
+                raise ConfigParseError(
+                    f"Database '{database}' not found in list of databases. Available: {list(databases)}."
+                )
+            database = dict(databases[database])
+            assert isinstance(database, dict)
+            if "driver" not in database:
+                raise ConfigParseError(f"Database '{database}' did not specify a driver.")
+
+        run_args[f"database{index}"] = database
+        run_args[f"table{index}"] = table
+        if threads is not None:
+            run_args[f"threads{index}"] = int(threads)
+
+    # Update keywords
+    new_kw = dict(kw)  # Set defaults
+    new_kw.update(run_args)  # Apply config
+    new_kw.update({k: v for k, v in kw.items() if v})  # Apply non-empty defaults
+
+    new_kw["__conf__"] = run_args
+
+    return new_kw
+
+
+# There are no strict requirements for the environment variable name format.
+# But most shells only allow alphanumeric characters and underscores.
+# https://pubs.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap08.html
+# "Environment variable names (...) consist solely of uppercase letters, digits, and the '_' (underscore)"
+_ENV_VAR_PATTERN = r"\$\{([A-Za-z0-9_]+)\}"
+
+
+def _resolve_env(config: Dict[str, Any]) -> None:
+    """
+    Resolve environment variables referenced as ${ENV_VAR_NAME}.
+    Missing environment variables are replaced with an empty string.
+    """
+    for key, value in config.items():
+        if isinstance(value, dict):
+            _resolve_env(value)
+        elif isinstance(value, str):
+            config[key] = re.sub(_ENV_VAR_PATTERN, _replace_match, value)
+
+
+def _replace_match(match: re.Match) -> str:
+    # Lookup referenced variable in environment.
+    # Replace with empty string if not found
+    referenced_var = match.group(1)  # group(0) is the whole string
+    return os.environ.get(referenced_var, "")
+
+
+def apply_config_from_file(path: str, run_name: str, kw: Dict[str, Any]):
+    with open(path) as f:
+        return _apply_config(toml.load(f), run_name, kw)
+
+
+def apply_config_from_string(toml_config: str, run_name: str, kw: Dict[str, Any]):
+    return _apply_config(toml.loads(toml_config), run_name, kw)

data_diff/databases/__init__.py

@@ -0,0 +1,38 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from data_diff.databases._connect import Connect as Connect
+from data_diff.databases._connect import connect as connect
+from data_diff.databases.base import (
+    CHECKSUM_HEXDIGITS,
+    CHECKSUM_OFFSET,
+    MD5_HEXDIGITS,
+    BaseDialect,
+    ConnectError,
+    Database,
+    QueryError,
+)
+from data_diff.databases.bigquery import BigQuery as BigQuery
+from data_diff.databases.clickhouse import Clickhouse as Clickhouse
+from data_diff.databases.databricks import Databricks as Databricks
+from data_diff.databases.duckdb import DuckDB as DuckDB
+from data_diff.databases.mssql import MsSQL as MsSQL
+from data_diff.databases.mysql import MySQL as MySQL
+from data_diff.databases.oracle import Oracle as Oracle
+from data_diff.databases.postgresql import PostgreSQL as PostgreSQL
+from data_diff.databases.presto import Presto as Presto
+from data_diff.databases.redshift import Redshift as Redshift
+from data_diff.databases.snowflake import Snowflake as Snowflake
+from data_diff.databases.trino import Trino as Trino
+from data_diff.databases.vertica import Vertica as Vertica