relationalai 0.13.0.dev0__py3-none-any.whl → 0.13.1__py3-none-any.whl

relationalai/semantics/std/aggregates.py

@@ -1,149 +0,0 @@
-from __future__ import annotations
-from ..frontend.base import Field, Library, Relationship, Aggregate, Group, TupleVariable, Value, Distinct
-from ..frontend.core import Any, Boolean, Number, String, Integer, TypeVar, ScaledNumber, Numeric, Float
-
-library = Library("aggregates")
-
-AggValue = Value | Distinct
-
-#------------------------------------------------------
-# Aggregates
-#------------------------------------------------------
-
-# TODO - overloads
-_sum = library.Relation("sum", fields=[Field.input("value", Numeric), Field("result", Numeric)],
-                        overloads=[[Number, Number], [Float, Float]])
-_count = library.Relation("count", fields=[Field("result", Integer)])
-_avg = library.Relation("avg", fields=[Field.input("over", Numeric), Field("result", Numeric)],
-                        overloads=[[Number, ScaledNumber], [Float, Float]])
-_min = library.Relation("min", fields=[Field.input("over", TypeVar), Field("result", TypeVar)])
-_max = library.Relation("max", fields=[Field.input("over", TypeVar), Field("result", TypeVar)])
-_string_join = library.Relation("string_join",
-        fields=[Field.input("index", Number), Field.input("sep", String), Field.input("over", String), Field("result", String)])
-
-_sort = library.Relation("sort", fields=[Field.input("limit", Integer), Field.input("args", Any, is_list=True), Field.input("is_asc", Boolean, is_list=True)])
-_rank = library.Relation("rank", fields=[Field.input("args", Any, is_list=True), Field.input("is_asc", Boolean, is_list=True), Field("rank", Integer)])
-_limit = library.Relation("limit", fields=[Field.input("limit", Integer), Field.input("args", Any, is_list=True), Field.input("is_asc", Boolean, is_list=True)])
-
-
-def sum(*args: AggValue) -> Aggregate:
-    return Aggregate(_sum, *args)
-
-def count(*args: AggValue) -> Aggregate:
-    return Aggregate(_count, *args)
-
-def min(*args: AggValue) -> Aggregate:
-    return Aggregate(_min, *args)
-
-def max(*args: AggValue) -> Aggregate:
-    return Aggregate(_max, *args)
-
-def avg(*args: AggValue) -> Aggregate:
-    return Aggregate(_avg, *args)
-
-def string_join(*args: AggValue, sep="", index=1) -> Aggregate:
-    return Aggregate(_string_join, index, sep, *args)
-
-class Ordering:
-    def __init__(self, *values:AggValue, is_asc=True) -> None:
-        self._values = values
-        self._is_asc = is_asc
-
-    @staticmethod
-    def handle_arg(arg: AggValue|Ordering, ordering, ordering_args, is_asc=True) -> bool:
-        has_distinct = False
-        if isinstance(arg, Ordering):
-            for v in arg._values:
-                has_distinct = Ordering.handle_arg(v, ordering, ordering_args, arg._is_asc) or has_distinct
-        elif isinstance(arg, Distinct):
-            has_distinct = True
-            for v in arg._items:
-                Ordering.handle_arg(v, ordering, ordering_args, is_asc)
-        else:
-            ordering.append(is_asc)
-            ordering_args.append(arg)
-        return has_distinct
-
-    @staticmethod
-    def get_ordering_args(args: tuple[AggValue|Ordering, ...]) -> tuple[tuple[AggValue,...], tuple[bool,...], bool]:
-        ordering = []
-        ordering_args = []
-        has_distinct = False
-        for arg in args:
-            has_distinct = Ordering.handle_arg(arg, ordering, ordering_args) or has_distinct
-        return tuple(ordering_args), tuple(ordering), has_distinct
-
-def asc(*args: AggValue) -> Ordering:
-    return Ordering(*args, is_asc=True)
-
-def desc(*args: AggValue) -> Ordering:
-    return Ordering(*args, is_asc=False)
-
-def _sort_agg(limit: int, *args: AggValue|Ordering) -> Aggregate:
-    ordering_args, is_asc, has_distinct = Ordering.get_ordering_args(args)
-    return Aggregate(_sort, limit, TupleVariable(ordering_args), TupleVariable(is_asc), distinct=has_distinct)
-
-def rank(*args: AggValue|Ordering) -> Aggregate:
-    ordering_args, is_asc, has_distinct = Ordering.get_ordering_args(args)
-    return Aggregate(_rank, TupleVariable(ordering_args), TupleVariable(is_asc), distinct=has_distinct)
-
-def limit(limit: int, *args: AggValue|Ordering) -> Aggregate:
-    ordering_args, is_asc, has_distinct = Ordering.get_ordering_args(args)
-    return Aggregate(_limit, limit, TupleVariable(ordering_args), TupleVariable(is_asc), distinct=has_distinct)
-
-def rank_asc(*args: AggValue) -> Aggregate:
-    return Aggregate(_rank, TupleVariable(args), TupleVariable([True for _ in args]))
-
-def rank_desc(*args: AggValue) -> Aggregate:
-    return Aggregate(_rank, TupleVariable(args), TupleVariable([False for _ in args]))
-
-def top(limit: int, *args: AggValue) -> Aggregate:
-    return Aggregate(_limit, limit, TupleVariable(args), TupleVariable([False for _ in args]))
-
-def bottom(limit: int, *args: AggValue) -> Aggregate:
-    return Aggregate(_limit, limit, TupleVariable(args), TupleVariable([True for _ in args]))
-
-#------------------------------------------------------
-# Per
-#------------------------------------------------------
-
-class Per(Group):
-
-    def sum(self, *args: AggValue) -> Aggregate:
-        return sum(*args).per(*self._args)
-
-    def count(self, *args: AggValue) -> Aggregate:
-        return count(*args).per(*self._args)
-
-    def min(self, *args: AggValue) -> Aggregate:
-        return min(*args).per(*self._args)
-
-    def max(self, *args: AggValue) -> Aggregate:
-        return max(*args).per(*self._args)
-
-    def avg(self, *args: AggValue) -> Aggregate:
-        return avg(*args).per(*self._args)
-
-    def string_join(self, *args: AggValue, sep="", index=1) -> Aggregate:
-        return string_join(*args, sep=sep, index=index).per(*self._args)
-
-    def rank(self, *args: AggValue|Ordering) -> Aggregate:
-        return rank(*args).per(*self._args)
-
-    def limit(self, limit_: int, *args: AggValue|Ordering) -> Aggregate:
-        return limit(limit_, *args).per(*self._args)
-
-    def rank_asc(self, *args: AggValue) -> Aggregate:
-        return rank_asc(*args).per(*self._args)
-
-    def rank_desc(self, *args: AggValue) -> Aggregate:
-        return rank_desc(*args).per(*self._args)
-
-    def top(self, limit: int, *args: AggValue) -> Aggregate:
-        return top(limit, *args).per(*self._args)
-
-    def bottom(self, limit: int, *args: AggValue) -> Aggregate:
-        return bottom(limit, *args).per(*self._args)
-
-def per(*args: Value) -> Per:
-    return Per(*args)

relationalai/semantics/std/common.py

@@ -1,44 +0,0 @@
-from __future__ import annotations
-
-from . import StringValue, IntegerValue, DateValue, DateTimeValue, math
-from ..frontend.base import Aggregate, Library, Concept, NumberConcept, Expression, Field, Literal, TupleVariable, Variable, Value
-from ..frontend.core import Any, Int128, Number, String, Integer, Date, DateTime, Hash
-
-from typing import Union, Literal
-import datetime as dt
-
-# the front-end library object
-library = Library("common")
-
-
-_range = library.Relation("range", [Field.input("start", Integer), Field.input("stop", Integer), Field.input("step", Integer), Field("value", Integer)])
-
-def range(*args: IntegerValue) -> Expression:
-    """ Generate a range of integers.
-
-    Supports range(stop), range(start, stop), range(start, stop, step).
-
-    Start is inclusive and defaults to 0.
-    Stop is exclusive.
-    Step defaults to 1.
-    """
-    if len(args) == 1:
-        start, stop, step = 0, args[0], 1
-    elif len(args) == 2:
-        start, stop, step = args[0], args[1], 1
-    elif len(args) == 3:
-        start, stop, step = args
-    else:
-        raise ValueError(f"range expects 1, 2, or 3 arguments, got {len(args)}")
-
-    return _range(start, stop, step)
-
-_hash = library.Relation("hash", [Field.input("args", Any, is_list=True), Field("hash", Hash)])
-def hash(*args: Value) -> Expression:
-    """ Compute a hash value for the given arguments. """
-    return _hash(TupleVariable(args))
-
-_uuid_to_string = library.Relation("uuid_to_string", [Field.input("uuid", Hash), Field("str", String)])
-def uuid_to_string(uuid: Value) -> Expression:
-    """ Convert a UUID (Hash) to its string representation. """
-    return _uuid_to_string(uuid)

relationalai/semantics/std/numbers.py

@@ -1,86 +0,0 @@
-from __future__ import annotations
-
-from . import NumberValue, StringValue
-from .. import literal
-from ..frontend.base import Library, Concept, MetaRef, NumberConcept, Expression, Field, Variable
-from ..frontend.core import Number, String, cast as core_cast
-from decimal import Decimal as PyDecimal
-
-
-# the front-end library object
-library = Library("numbers")
-
-#--------------------------------------------------
-# Constructors
-#--------------------------------------------------
-_parse_number = library.Relation("parse_number", [Field.input("value", String), Field("result", Number)])
-
-def number(value: NumberValue, precision=38, scale=14) -> Variable:
-    """
-    Create an expression that represents a number with this value, precision and scale.
-    """
-    if isinstance(value, Variable):
-        cast_type = Number.size(precision, scale)
-        return core_cast(MetaRef(cast_type), value, cast_type.ref())
-
-    # literals
-    if isinstance(value, int):
-        value = PyDecimal(str(value))
-    if isinstance(value, float):
-        value = PyDecimal(str(value))
-    return literal(value, Number.size(precision, scale))
-
-def parse_number(value: StringValue, precision=38, scale=14) -> Expression:
-    """
-    Create an expression that represents parsing this string value as a number with this
-    precision and scale.
-    """
-    return _parse_number(value, Number.size(precision, scale).ref())
-
-def parse(value: str, number: NumberConcept) -> Expression:
-    """
-    Create an expression that represents parsing this string value as a number with the
-    precision and scale of the number argument.
-    """
-    return parse_number(value, precision(number), scale(number))
-
-#--------------------------------------------------
-# Number information.
-#--------------------------------------------------
-
-def is_number(number: Concept) -> bool:
-    return isinstance(number, NumberConcept)
-
-def precision(number: NumberConcept) -> int:
-    """ Assuming the concept represents a number, return its precision. """
-    return number._precision
-
-def scale(number: NumberConcept) -> int:
-    """ Assuming the concept represents a number, return its scale. """
-    return number._scale
-
-def size(number: NumberConcept) -> int:
-    """
-    Assuming the concept represents a number, return its size, i.e. the number of bits
-    needed to represent the number.
-    """
-    return digits_to_bits(number.precision)
-
-def digits_to_bits(precision)-> int:
-    """
-    Transform from a number of base 10 digits to the number of bits necessary to represent
-    that. If the precision is larger than 38, return None as that is not supported.
-
-    For example, a number with 38 digits requires 128 bits.
-    """
-    if precision <= 2:
-        return 8
-    elif precision <= 4:
-        return 16
-    elif precision <= 9:
-        return 32
-    elif precision <= 18:
-        return 64
-    elif precision <= 38:
-        return 128
-    raise ValueError(f"Invalid numeric precision '{precision}'")

relationalai/shims/executor.py

@@ -1,147 +0,0 @@
-
-#------------------------------------------------------
-# VERY UNSTABLE API TO EXECUTE FRONTEND PROGRAMS USING V0 EXECUTORS
-#------------------------------------------------------
-from functools import lru_cache
-import json
-from v0.relationalai import debugging
-from v0.relationalai.semantics.lqp.executor import LQPExecutor
-from v0.relationalai.semantics.rel.executor import RelExecutor
-from v0.relationalai.semantics.sql.executor.duck_db import DuckDBExecutor
-from v0.relationalai.semantics.sql.executor.snowflake import SnowflakeExecutor
-from v0.relationalai.semantics.metamodel import ir as v0, factory as v0_factory
-from v0.relationalai.semantics.metamodel.visitor import collect_by_type
-from v0.relationalai.semantics.snowflake import Table as v0Table
-from v0.relationalai.clients.snowflake import Provider as v0Provider
-
-
-from relationalai.semantics import Model, Fragment
-from relationalai.semantics.metamodel import metamodel as mm
-from relationalai.shims.mm2v0 import Translator
-from ..semantics.metamodel.typer import Typer
-from ..semantics.metamodel.metamodel_analyzer import Normalize
-
-DEBUG=False
-PRINT_RESULT=False
-TYPER_DEBUGGER=False
-
-# DEBUG=True
-# PRINT_RESULT=True
-# TYPER_DEBUGGER=True
-
-def execute(query: Fragment, model: Model|None = None, executor="lqp", export_to="", update=False):
-    mm_model = model.to_metamodel() if model else None
-    mm_query = query.to_metamodel()
-    assert isinstance(mm_query, mm.Node)
-    return execute_mm(mm_query, mm_model, executor, export_to=export_to, update=update, model=model)
-
-
-def execute_mm(mm_query: mm.Task, mm_model: mm.Model|None = None, executor="lqp", export_to="", update=False, model: Model|None = None):
-    # perform type inference
-    typer = Typer()
-    # normalize the metamodel
-    normalizer = Normalize()
-    # translate the metamodel into a v0 query
-    translator = Translator()
-
-    # for typer debugging
-    debugger_msgs = []
-
-    try:
-        #------------------------------------------------------
-        # Model processing
-        #------------------------------------------------------
-        v0_model = None
-        if mm_model:
-            # type inference
-            debugger_msgs.append(json.dumps({ 'id': 'model', 'content': str(mm_model)}))
-            mm_model = typer.infer_model(mm_model)
-            debugger_msgs.append(json.dumps({ 'id': 'typed_model', 'content': str(mm_model)}))
-            assert(typer.model_net is not None)
-            debugger_msgs.append(json.dumps({ 'id': 'model_net', 'content': str(typer.model_net.to_mermaid())}))
-            # normalization
-            mm_model = mm_model.mut(root=normalizer.normalize(mm_model.root)) # type: ignore
-            assert isinstance(mm_model, mm.Model)
-            if DEBUG:
-                print("V1 Model:")
-                print(mm_model)
-            # translation
-            v0_model = translator.translate_model(mm_model)
-            if DEBUG:
-                print("Translated v0 Model:")
-                print(v0_model)
-
-        #------------------------------------------------------
-        # Query processing
-        #------------------------------------------------------
-        # type inference
-        debugger_msgs.append(json.dumps({ 'id': 'query', 'content': str(mm_query)}))
-        mm_query = typer.infer_query(mm_query)
-        debugger_msgs.append(json.dumps({ 'id': 'typed_query', 'content': str(mm_query)}))
-        assert(typer.last_net is not None)
-        debugger_msgs.append(json.dumps({ 'id': 'query_net', 'content': str(typer.last_net.to_mermaid())}))
-        # normalization
-        mm_query = normalizer.normalize(mm_query)  # type: ignore
-        assert isinstance(mm_query, mm.Task)
-        if DEBUG:
-            print("V1 Query:")
-            print(mm_query)
-        # translation
-        v0_query = translator.translate_query(mm_query)
-        if DEBUG:
-            print("Translated v0 Query:")
-            print(v0_query)
-        assert isinstance(v0_query, v0.Task)
-
-        if v0_model is None:
-            # there was no model, so create one from the elements refered to by the query
-            v0_model = v0_factory.model(
-                collect_by_type(v0.Engine, v0_query),
-                collect_by_type(v0.Relation, v0_query),
-                v0_factory._collect_reachable_types(collect_by_type(v0.Type, v0_query)),
-                v0_factory.logical([])
-            )
-    finally:
-        if TYPER_DEBUGGER:
-            with open("typer_debug.jsonl", "w") as f:
-                for msg in debugger_msgs:
-                    f.write(msg)
-                    f.write('\n')
-
-    # create snowflake tables for all the tables that have been used
-    ts = [v0Table(t.name) for t in translator.used_tables if not t.uri.startswith("dataframe://")]
-    for t in ts:
-        t._lazy_init()
-        v0Table._used_sources.add(t)
-
-    export_table = None
-    if export_to:
-        export_table = v0Table(export_to)
-
-    # get an executor and execute
-    executor = _get_executor(executor, model.name if model else "")
-    with debugging.span("query", tag=None, export_to=export_to) as query_span:
-        if isinstance(executor, (LQPExecutor, RelExecutor)):
-            results = executor.execute(v0_model, v0_query, export_to=export_table, update=update)
-        else:
-            results = executor.execute(v0_model, v0_query)
-        query_span["results"] = results
-    if DEBUG or PRINT_RESULT:
-        print(results)
-    return results
-
-# @lru_cache()
-def _get_executor(name: str, database: str = "ttb_test"):
-    if name == "duckdb":
-        return DuckDBExecutor()
-    elif name == "lqp":
-        return LQPExecutor(database)
-    elif name == "rel":
-        return RelExecutor(database)
-    elif name == "snowflake":
-        return SnowflakeExecutor(database="TEST_DB", schema="PUBLIC")
-    else:
-        raise ValueError(f"Unknown executor: {name}")
-
-def get_provider():
-    return v0Provider()

relationalai/shims/helpers.py

@@ -1,126 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import  Iterable, Sequence as seq
-
-from relationalai.semantics.metamodel import metamodel as mm
-from relationalai.semantics.metamodel.builtins import builtins as b
-from relationalai.semantics.metamodel.rewriter import NO_WALK, Walker
-from relationalai.util.structures import OrderedSet
-
-
-class ContainerWalker(Walker):
-    """ Walker that redirects all container nodes to a common handler. """
-
-    def enter_container(self, container: mm.Container, children: seq[mm.Node]) -> object |  None:
-        return None
-
-    def exit_container(self, container: mm.Container, children: seq[mm.Node]) -> object |  None:
-        return container
-
-    #------------------------------------------------------
-    # Redirect containers to common handlers
-    #------------------------------------------------------
-
-    def enter_logical(self, logical: mm.Logical):
-        return self.enter_container(logical, logical.body)
-
-    def logical(self, logical: mm.Logical):
-        return self.exit_container(logical, logical.body)
-
-    def enter_sequence(self, sequence: mm.Sequence):
-        return self.enter_container(sequence, sequence.tasks)
-
-    def sequence(self, sequence: mm.Sequence):
-        return self.exit_container(sequence, sequence.tasks)
-
-    def enter_union(self, union: mm.Union):
-        return self.enter_container(union, union.tasks)
-
-    def union(self, union: mm.Union):
-        return self.exit_container(union, union.tasks)
-
-    def enter_match(self, match: mm.Match):
-        return self.enter_container(match, match.tasks)
-
-    def match(self, match: mm.Match):
-        return self.exit_container(match, match.tasks)
-
-    def enter_until(self, until: mm.Until):
-        return self.enter_container(until, [until.check, until.body])
-
-    def until(self, until: mm.Until):
-        self.exit_container(until, [until.check, until.body])
-
-    def enter_wait(self, wait: mm.Wait):
-        return self.enter_container(wait, [wait.check])
-
-    def wait(self, wait: mm.Wait):
-        return self.exit_container(wait, [wait.check])
-
-    def enter_loop(self, loop: mm.Loop):
-        return self.enter_container(loop, [loop.body])
-
-    def loop(self, loop: mm.Loop):
-        return self.exit_container(loop, [loop.body])
-
-    def enter_require(self, require: mm.Require):
-        return self.enter_container(require, [require.domain, require.check])
-
-    def require(self, require: mm.Require):
-        return self.exit_container(require, [require.domain, require.check])
-
-    def enter_not(self, not_: mm.Not):
-        return self.enter_container(not_, [not_.task])
-
-    def not_(self, not_: mm.Not):
-        return self.exit_container(not_, [not_.task])
-
-    def enter_exists(self, exists: mm.Exists):
-        return self.enter_container(exists, [exists.task])
-
-    def exists(self, exists: mm.Exists):
-        return self.exit_container(exists, [exists.task])
-
-
-class VarFinder(ContainerWalker):
-    """ Find all variables used in a set of nodes. """
-
-    def enter_container(self, container, children):
-         if self.shallow:
-            return NO_WALK
-         return None
-
-    def find_vars(self, node:mm.Node|Iterable[mm.Node], positive_only=False, shallow=False) -> OrderedSet[mm.Var]:
-        self.vars:OrderedSet[mm.Var] = OrderedSet()
-        self.positive_only = positive_only
-        self.shallow = shallow
-        self(node)
-        return self.vars
-
-    def enter_not(self, not_:mm.Not):
-        if self.positive_only:
-            return NO_WALK
-
-    def var(self, node:mm.Var):
-        self.vars.add(node)
-
-#------------------------------------------------------
-# Helper functions shared across shim modules
-#------------------------------------------------------
-
-def is_output_update(u: mm.Task) -> bool:
-    return (
-        isinstance(u, mm.Update) and
-        len(u.args) > 0 and
-        isinstance(u.args[0], mm.Var) and
-        isinstance(u.args[0].type, mm.Table) and
-        (u.relation is u.args[0].type or u.relation in u.args[0].type.columns)
-        # In v0 all outputs, including exports, end up as v0.Output so we actually
-        # don't want to limit this to just arrow results
-        # u.args[0].type.uri.startswith("dataframe://")
-    )
-
-def is_main_output(u: mm.Task) -> bool:
-    """ This task is an update to dataframe, and represents the main output (it has only keys, no values)."""
-    return isinstance(u, mm.Update) and is_output_update(u) and isinstance(u.relation, mm.Table)