relationalai 0.13.0__py3-none-any.whl → 0.13.0.dev0__py3-none-any.whl

relationalai/util/naming.py

@@ -0,0 +1,145 @@
+from dataclasses import dataclass, field
+from typing import Hashable
+import re
+
+#------------------------------------------------------
+# Naming helpers
+#------------------------------------------------------
+
+
+def sanitize(name:str) -> str:
+    """ Cleanup the name to make it more palatable to names. """
+    x = re.sub(r"[ ,\.\(\)\|\n]", "_", name)
+    return x[0:-1] if x[-1] == "_" else x
+
+#------------------------------------------------------
+# Namer and NameCache
+#------------------------------------------------------
+
+@dataclass(frozen=True)
+class Namer:
+    """ Simple namer that keeps a single scope/context. """
+    # in range mode, the first name will have a 0 suffix and the second will have a 1 suffix, etc.
+    # otherwise, the first name will have no suffix, the second will have a _2 suffix, etc.
+    range: bool = False
+    next_count: dict[str, int] = field(default_factory=dict)
+
+    def get_name(self, name: str) -> str:
+        if name in self.next_count:
+            c = self.next_count[name]
+            self.next_count[name] = c + 1
+            if self.range:
+                new_name = f"{name}{c}"
+            else:
+                new_name = f"{name}_{c}"
+            if new_name in self.next_count:
+                # if the modified name is also in use, recurse
+                return self.get_name(new_name)
+            return new_name
+        else:
+            if self.range:
+                self.next_count[name] = 1
+                return f"{name}0"
+            else:
+                self.next_count[name] = 2
+                return name
+
+@dataclass
+class NameCache:
+    # support to generate object names with a count when there's collision
+    # the next count to use as a suffix for an object with this name
+    name_next_count: dict[Hashable, dict[str, int]] = field(default_factory=dict)
+    # cache of the precomputed name for the object with this key
+    name_cache: dict[Hashable, str] =  field(default_factory=dict)
+    # whether to use _ or not
+    use_underscore: bool = True
+    # whether the first entry should start with a 1 or with nothing
+    start_from_one: bool = False
+
+    def get_name(self, key: Hashable, name: str, prefix: str = "") -> str:
+        """
+        Generate a unique name for the given key and base name, avoiding name collisions.
+
+        Names are tracked and deduplicated using an internal counter. If a name has already
+        been generated for the given key, the cached name is returned.
+
+        For tuple keys (e.g., (relation_id, var_id)), the first element is used as the scope
+        for counting name collisions. Only tuples of length 2 are supported; an assertion will
+        fail otherwise.
+
+        Parameters:
+            key (Hashable): A unique key identifying the object. Can be an int or a (scope_id, local_id) tuple.
+            name (str): Base name to use.
+            prefix (str): Optional prefix to prepend to the name.
+
+        Returns:
+            str: A unique name string for the given key.
+
+        Examples:
+            # Global naming
+            get_name(1, "var")           => "var"
+            get_name(2, "var")           => "var_2"
+            get_name(3, "var")           => "var_3"
+
+            # Scoped naming (e.g., per relation)
+            get_name((10, 1), "x")       => "x"
+            get_name((10, 2), "x")       => "x_2"
+            get_name((20, 1), "x")       => "x"     # Different scope, starts again
+            get_name((20, 2), "x")       => "x_2"
+
+            # With prefix and custom suffix start
+            n = NameCache()
+            n.get_name(1, "val", prefix="t_")                => "t_val"
+
+            n = NameCache(start_from_one=True)
+            n.get_name(1, "val", prefix="t_")                => "t_val_1"
+        """
+        if key in self.name_cache:
+            return self.name_cache[key]
+
+        # Derive the count scope from the key
+        scope = None
+        if isinstance(key, tuple):
+            assert len(key) == 2, f"Expected tuple key of length 2, got {len(key)}: {key}"
+            scope = key[0]  # e.g., relation_id
+
+        # get the dict specific for the scope, or create one
+        if scope in self.name_next_count:
+            next_count = self.name_next_count[scope]
+        else:
+            next_count = dict()
+            self.name_next_count[scope] = next_count
+
+        # find the next available name
+        name = self._find_next(f"{prefix}{name}", next_count)
+
+        # register it for the key
+        self.name_cache[key] = name
+        return name
+
+    def _find_next(self, name: str, next_count: dict):
+        if name in next_count:
+            # name is already in use, so append the next count
+            c = next_count[name]
+            next_count[name] = c + 1
+            new_name = self._concat(name, c)
+
+        else:
+            # name not in use yet, so register it
+            next_count[name] = 2
+            if self.start_from_one:
+                new_name = self._concat(name, 1)
+            else:
+                new_name = name
+
+        if new_name != name:
+            if new_name in next_count:
+                # if the modified name is also in use, recurse
+                return self._find_next(name, next_count)
+            else:
+                # otherwise, record that the modified name was used
+                next_count[new_name] = 2
+        return new_name
+
+    def _concat(self, name: str, c: int):
+        return f"{name}_{c}" if self.use_underscore else f"{name}{c}"

relationalai/util/python.py

@@ -0,0 +1,35 @@
+from datetime import date, datetime
+from decimal import Decimal as PyDecimal
+from types import NoneType
+import numpy as np
+import pandas as pd
+
+#--------------------------------------------------
+# Python types
+#--------------------------------------------------
+
+pytype_to_concept_name: dict[object, str] = {
+    NoneType:  "Any",
+    int: "Integer", float: "Number(38,14)", str: "String", bool: "Boolean",
+    date: "Date", datetime: "DateTime", PyDecimal: "Number(38,14)",
+    # Int128Dtype(): "Integer",
+
+    # NumPy dtypes
+    **{np.dtype(k): "Integer" for k in ("int8","int16","int32","int64","uint8","uint16","uint32","uint64")},
+    **{np.dtype(k): "Number(38,14)" for k in ("float32","float64")},
+    np.dtype("bool"): "Boolean",
+    np.dtype("object"): "String",
+    np.dtype("datetime64[ns]"): "DateTime",
+    np.dtype("datetime64[ms]"): "DateTime",
+    np.dtype("datetime64[s]"): "DateTime",
+
+    # Pandas extension dtypes
+    **{t(): "Integer" for t in (pd.Int8Dtype, pd.Int16Dtype, pd.Int32Dtype, pd.Int64Dtype,
+                        pd.UInt8Dtype, pd.UInt16Dtype, pd.UInt32Dtype, pd.UInt64Dtype)},
+    **{t(): "Number(38,14)" for t in (pd.Float32Dtype, pd.Float64Dtype)},
+    pd.StringDtype(): "String",
+    pd.BooleanDtype(): "Boolean",
+    "int": "Integer", "float": "Number(38,14)", "str": "String", "bool": "Boolean",
+    "date": "Date", "datetime": "DateTime", "decimal": "Number(38,14)",
+    "datetime.date": "Date", "datetime.datetime": "DateTime"
+}

relationalai/util/runtime.py

@@ -0,0 +1,156 @@
+import os
+import sys
+import warnings
+import more_itertools
+from .tracing import get_tracer
+from .error import Diagnostic, RAIWarning, RichEmitter, HTMLEmitter, RAIException
+import traceback
+
+#------------------------------------------------------
+# Base Env
+#------------------------------------------------------
+
+class EnvBase:
+    """Decides renderer and how to deliver the string. Override to suit your runtime."""
+    # Which renderer to use for this env
+    prefers_html: bool = False
+    seen_diagnostics: set[str] = set()
+
+    def is_active(self) -> bool:  # override
+        return False
+
+    def renderer(self):
+        return HTMLEmitter() if self.prefers_html else RichEmitter()
+
+    # Delivery hooks — override as needed
+    def warn(self, diag: Diagnostic):
+        """Default: integrate with Python warnings and also print to stderr if terminal."""
+        category = RAIWarning
+        rendered = self.renderer().emit(diag)
+        if rendered in self.seen_diagnostics:
+            return  # avoid duplicate output
+        # If not HTML and we have a TTY, echo the pretty string too
+        if not self.prefers_html and sys.stderr and sys.stderr.isatty():
+            sys.stderr.write(rendered.rstrip() + "\n")
+        # route control via warnings package
+        warnings.warn(f"[{diag.name}] {diag.message}", category=category, stacklevel=3)
+        self._trace(diag)
+        self.seen_diagnostics.add(rendered)
+
+
+    def err(self, diag: Diagnostic, exception: bool = False):
+        """Default: print (or display) and raise DiagnosticException."""
+        # Echo before raising for better UX in terminals/CI
+        rendered = self.renderer().emit(diag)
+        if rendered in self.seen_diagnostics:
+            return  # avoid duplicate output
+        if self.prefers_html:
+            # In notebook-y envs, try to display inline
+            try:
+                from IPython.display import display, HTML as _HTML
+                display(_HTML(rendered))
+            except Exception:
+                sys.stderr.write("[HTML diagnostic suppressed]\n")
+        else:
+            sys.stderr.write(rendered.rstrip() + "\n")
+
+        self._trace(diag, error=True)
+        self.seen_diagnostics.add(rendered)
+        if exception:
+            raise RAIException(diag)
+
+    def _trace(self, diag: Diagnostic, error: bool = False):
+        try:
+            tracer = get_tracer()
+            payload = diag.to_dict()
+            tracer.add_event("diagnostic", diagnostic=payload)
+            if error:
+                tracer.set_status(ok=False, message=diag.message)
+        except Exception:
+            pass
+
+#------------------------------------------------------
+# Envs
+#------------------------------------------------------
+
+class CI(EnvBase):
+    def is_active(self) -> bool:
+        return any(os.getenv(k) for k in ("CI", "GITHUB_ACTIONS", "BUILD_NUMBER", "TEAMCITY_VERSION"))
+
+    def warn(self, diag: Diagnostic):
+        # keep CI logs plain and controllable via warnings
+        rendered = self.renderer().emit(diag)
+        warnings.warn(f"[{diag.name}] {diag.message}", category=RAIWarning, stacklevel=3)
+        sys.stderr.write(rendered.rstrip() + "\n")
+        self._trace(diag)
+
+    def err(self, diag: Diagnostic, exception: bool = False):
+        rendered = self.renderer().emit(diag)
+        sys.stderr.write(rendered.rstrip() + "\n")
+        self._trace(diag)
+        if exception:
+            raise RAIException(diag)
+
+class Terminal(EnvBase):
+    def is_active(self) -> bool:
+        try:
+            return sys.stderr.isatty()
+        except Exception:
+            return False
+
+class File(EnvBase):
+    """Non-interactive file/script run."""
+    def is_active(self) -> bool:
+        # If stderr is not a TTY and not a known notebook/colab/hex, treat as file
+        return (not sys.stderr.isatty()) and ("ipykernel" not in sys.modules) and ("google.colab" not in sys.modules)
+
+    def warn(self, diag: Diagnostic):
+        # just warnings integration (no fancy output)
+        warnings.warn(f"[{diag.name}] {diag.message}", category=RAIWarning, stacklevel=3)
+        self._trace(diag)
+
+    def err(self, diag: Diagnostic, exception: bool = False):
+        self._trace(diag)
+        if exception:
+            raise RAIException(diag)
+
+class Jupyter(EnvBase):
+    prefers_html = True
+    def is_active(self) -> bool:
+        try:
+            from IPython import get_ipython #type: ignore
+            ip = get_ipython()
+            return bool(ip and ip.__class__.__name__.lower().endswith("zmqinteractiveshell"))
+        except Exception:
+            return False
+
+class Colab(EnvBase):
+    prefers_html = True
+    def is_active(self) -> bool:
+        return "google.colab" in sys.modules
+
+class Hex(EnvBase):
+    prefers_html = True
+    def is_active(self) -> bool:
+        # Hex notebooks expose 'HEX_RUN' env or similar; keep flexible
+        return bool(os.getenv("HEX_RUN") or "hex" in sys.modules)
+
+#------------------------------------------------------
+# find env
+#------------------------------------------------------
+
+def find_env():
+    return more_itertools.first_true(
+        [
+            CI(),
+            Terminal(),
+            File(),
+            Jupyter(),
+            Colab(),
+            Hex(),
+        ],
+        pred=lambda k: k.is_active(),
+        default=File(),
+    )
+
+ENV = find_env()

relationalai/util/schema.py

@@ -0,0 +1,197 @@
+from collections import defaultdict
+from functools import lru_cache
+import json
+from pathlib import Path
+import re
+import textwrap
+
+from relationalai.util.error import warn
+from ..config import Config
+from ..config.connections.snowflake import SnowflakeConnection
+
+# config = Config()
+# conn = config.get_connection(SnowflakeConnection, name="snowflake")
+
+# print(conn.get_session().sql("SELECT CURRENT_VERSION()").collect())
+
+#------------------------------------------------------
+# General
+#------------------------------------------------------
+
+CACHE_PATH = Path("build/cache/schemas.json")
+CACHE = {}
+
+def fetch(fqn: str) -> dict[str, str]:
+    """
+    Fetch Concept metadata for a fully-qualified table name.
+
+    - Cache file may or may not exist.
+    - Cache structure is assumed to be { "<db>.<schema>.<table>": { ...Concepts... } }
+    - If the requested FQN is missing from the cache, fetch_snowflake is called and
+      the result is stored back into the cache.
+    """
+    global CACHE
+    database, schema, table = fqn.split(".")
+
+    # Load or initialize cache
+    if not CACHE and CACHE_PATH.exists():
+        with CACHE_PATH.open("r", encoding="utf-8") as f:
+            str_cache: dict[str, dict[str, str]] = json.load(f)
+            CACHE = {k: {ck: cv for ck, cv in v.items()} for k, v in str_cache.items()}
+
+    # If already cached, return it
+    if fqn in CACHE:
+        return CACHE[fqn]
+
+    # Not in cache → fetch from Snowflake
+    result = fetch_snowflake(database, schema, [table])
+    if table not in result:
+        return {}
+
+    CACHE[fqn] = result[table]
+
+    # Write updated cache
+    CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
+    with CACHE_PATH.open("w", encoding="utf-8") as f:
+        str_cache = {k: {ck: cv for ck, cv in v.items()} for k, v in CACHE.items()}
+        json.dump(str_cache, f, indent=2)
+
+    return CACHE[fqn]
+
+#------------------------------------------------------
+# Snowflake
+#------------------------------------------------------
+
+SUPPORTED_SNOWFLAKE_TYPES = [
+    'CHAR', 'STRING', 'VARCHAR', 'BINARY', 'NUMBER', 'FLOAT', 'REAL',
+    'BOOLEAN', 'DATE', 'FIXED', 'TEXT', 'TIME', 'TIMESTAMP_LTZ',
+    'TIMESTAMP_NTZ', 'TIMESTAMP_TZ'
+]
+
+SFTypes = {
+    "TEXT": "String",
+    "FIXED": "Decimal",
+    "DATE": "Date",
+    "TIME": "DateTime",
+    "TIMESTAMP": "DateTime",
+    "TIMESTAMP_LTZ": "DateTime",
+    "TIMESTAMP_TZ": "DateTime",
+    "TIMESTAMP_NTZ": "DateTime",
+    "FLOAT": "Float",
+    "REAL": "Float",
+    "BOOLEAN": "Bool",
+}
+
+SF_ID_REGEX = re.compile(r'^[A-Za-z_][A-Za-z0-9_$]*$')
+
+def quoted(ident: str) -> str:
+    return ident if SF_ID_REGEX.match(ident) or ident.startswith('"') else f'"{ident}"'
+
+@lru_cache()
+def get_provider():
+    from ..shims.executor import get_provider
+    return get_provider()
+
+def fetch_snowflake(database:str, schema:str, table_names:list[str]) -> dict[str, dict[str, str]]:
+    name_lookup = {name.upper(): name for name in table_names}
+
+    tables = ", ".join(f"'{name.upper()}', '{name}'" for name in table_names)
+
+    query = textwrap.dedent(f"""
+        begin
+            SHOW COLUMNS IN SCHEMA {quoted(database)}.{quoted(schema)};
+            let r resultset := (
+                select "table_name", "column_name", "data_type"
+                from table(result_scan(-1)) as t
+                where "table_name" in ({tables})
+            );
+            return table(r);
+        end;
+    """)
+
+    # with debugging.span("fetch_schema", sql=query):
+    try:
+        columns = get_provider().sql(query)
+    except Exception as e:
+        columns = []
+    assert isinstance(columns, list)
+
+    unsupported_columns: dict[str, dict[str, str]] = defaultdict(dict)
+
+    schemas: dict[str, dict[str, str]] = defaultdict(dict)
+    for row in columns:
+        table_name, column_name, data_type = row
+        table_name = name_lookup.get(table_name, table_name)
+        assert table_name is not None
+
+        sf_type_info = json.loads(data_type)
+        typ = sf_type_info.get("type")
+
+        if typ not in SUPPORTED_SNOWFLAKE_TYPES:
+            unsupported_columns[table_name][column_name] = typ
+            continue
+
+        if typ == "FIXED":
+            concept_name = sf_numeric_to_type_str(column_name, sf_type_info)
+        else:
+            concept_name = SFTypes[typ]
+        schemas[table_name][column_name] = concept_name
+
+    for table_name, cols in unsupported_columns.items():
+        col_str = ", ".join(f"{col}({typ})" for col, typ in cols.items())
+        warn("Unsupported Column",
+            f"The following columns in '{database}.{schema}.{table_name}' have unsupported types: {col_str}",
+            [
+                "These columns will not be accessible in your model.",
+                "For the list of supported column types see: https://docs.relational.ai/api/cli/imports/stream/#supported-column-types"
+            ])
+
+    return schemas
+
+
+def digits_to_bits(precision)-> int:
+    """
+    Transform from a number of base 10 digits to the number of bits necessary to represent
+    that. If the precision is larger than 38, return None as that is not supported.
+
+    For example, a number with 38 digits requires 128 bits.
+    """
+    if precision <= 2:
+        return 8
+    elif precision <= 4:
+        return 16
+    elif precision <= 9:
+        return 32
+    elif precision <= 18:
+        return 64
+    elif precision <= 38:
+        return 128
+    raise ValueError(f"Invalid numeric precision '{precision}'")
+
+def sf_numeric_to_type_str(column_name: str, sf_type_info: dict) -> str:
+    """
+    Computes the appropriate type to use for this column. This code reflects exactly
+    the logic currently used by RAI's CDC implementation to ensure we map to the exact
+    same number types.
+    """
+    if "scale" not in sf_type_info or "precision" not in sf_type_info:
+        raise ValueError(
+            f"Invalid definition for column '{column_name}': "
+            "'scale' or 'precision' missing"
+        )
+
+    precision = sf_type_info["precision"]
+    scale = sf_type_info["scale"]
+
+    if scale > precision or scale < 0 or scale > 37:
+        raise ValueError(
+            f"Invalid numeric scale '{scale}' for column '{column_name}'"
+        )
+
+    if scale == 0:
+        # Integers (load_csv only supports these two (and not 8/16/32 bit ints)
+        bits = digits_to_bits(precision)
+        # return Int128 if bits == 128 else Int64
+        return f"Decimal(38,0)" if bits == 128 else f"Decimal(18,0)"
+
+    return f"Decimal({precision},{scale})"