relationalai 0.13.0__py3-none-any.whl → 0.13.0.dev0__py3-none-any.whl

relationalai/config/external/utils.py

@@ -0,0 +1,19 @@
+import os
+
+
+def find_dbt_profiles_file() -> str | None:
+    paths = [
+        "./profiles.yml",
+        os.path.expanduser("~/.dbt/profiles.yml"),
+    ]
+    for path in paths:
+        if os.path.exists(path):
+            return path
+    return None
+
+
+def find_snowflake_config_file() -> str | None:
+    path = os.path.expanduser("~/.snowflake/config.toml")
+    if os.path.exists(path):
+        return path
+    return None

relationalai/semantics/__init__.py

@@ -1,29 +1,153 @@
+"""The RelationalAI semantics API.
+
+This module provides the core classes and functions for defining and querying
+semantic models.
 """
-The RelationalAI Semantics Module.
-"""
 
-# Mark this package's docstrings for inclusion
-# in automatically generated web documentation.
+from pandas import DataFrame
+from ..util.error import exc
+from ..util.docutils import include_in_docs
+from .frontend.base import (
+    Alias, CoreLibrary, Concept, Data, Distinct, Expression, Field, FieldRef, Fragment, Library,
+    Literal, Match, Model, New, Not, Chain, Property, Reading, Relationship, Statement, StatementAndSchema, Table, Union, Value,
+    Variable,
+)
+# populate the core library
+from .frontend import core
+
+# include this module in documentation
 __include_in_docs__ = True
 
-from .internal import (
-    Model, Concept, Relationship, RelationshipReading, Expression, Fragment, Error, Field,
-    AnyEntity, String, Integer, Int64, Int128, Float, Decimal, Bool,
-    Date, DateTime,
-    RawSource, Hash,
-    select, where, require, define, distinct, union, data,
-    rank, asc, desc,
-    count, sum, min, max, avg, per,
-    not_
-)
+    # String, Integer, Int64, Int128, Float, Decimal, Bool,
+    # Date, DateTime,
+    # RawSource, Hash,
+    # select, where, require, define, distinct, union, data,
+    # rank, asc, desc,
+    # count, sum, min, max, avg, per,
+    # not_
+
+#------------------------------------------------------
+# Convenience model functions
+#------------------------------------------------------
+
+def _check_model():
+    if len(Model.all_models) == 0:
+        exc("Missing Model", "No Model has been defined. Please create a Model before using the semantics API.")
+    elif len(Model.all_models) > 1:
+        exc("Ambiguous model", "Multiple Models have been defined. Please use functions on the specific Model.")
+    return Model.all_models[0]
+
+@include_in_docs
+def select(*args:StatementAndSchema) -> Fragment:
+    """Return a selection fragment from the active model.
+
+    Parameters
+    ----------
+    *args : StatementAndSchema
+        Concepts, properties, relationships fields, expressions, or other
+        statements to select.
+
+    Returns
+    -------
+    Fragment
+        A composable query fragment representing the selection.
+
+    Examples
+    --------
+    Select a literal value and materialize the result as a pandas DataFrame:
+
+    >>> select(1).to_df()
+
+    Combine with filters to project properties of specific entities from the
+    current model:
+
+    >>> select(Person.name, Person.age) \
+    ...     .where(Person.age > 21).to_df()  # doctest: +SKIP
+    """
+    model = _check_model()
+    return model.select(*args)
+
+def define(*args:Statement):
+    model = _check_model()
+    return model.define(*args)
+
+def where(*args:Statement):
+    model = _check_model()
+    return model.where(*args)
+
+def require(*args:Statement):
+    model = _check_model()
+    return model.require(*args)
+
+def union(*args:Value):
+    model = _check_model()
+    return model.union(*args)
+
+def data(data: DataFrame | list[tuple] | list[dict], columns: list[str]|None = None):
+    model = _check_model()
+    return model.data(data, columns)
+
+def distinct(*args:Value):
+    model = _check_model()
+    return model.distinct(*args)
+
+def not_(*args:Value):
+    model = _check_model()
+    return model.not_(*args)
+
+def literal(value, type: Concept|None = None) -> Literal:
+    model = _check_model()
+    return Literal(value, model, type)
+
+#------------------------------------------------------
+# Convenience types
+#------------------------------------------------------
+Any = core.Any
+AnyEntity = core.AnyEntity
+TypeVar = core.TypeVar
+EntityTypeVar = core.EntityTypeVar
+Number = core.Number
+String = core.String
+Integer = core.Integer
+Int = core.Int
+Int64 = core.Int64
+Int128 = core.Int128
+Float = core.Float
+Decimal = core.Decimal
+Bool = core.Boolean
+Boolean = Bool
+Date = core.Date
+DateTime = core.DateTime
+# RawSource = core.RawSource
+# Hash = core.Hash
+
+#------------------------------------------------------
+# Convenience aggregates
+# NOTE: these should be deprecated
+#------------------------------------------------------
+
+from .std import aggregates as aggs
+
+rank = aggs.rank
+limit = aggs.limit
+asc = aggs.asc
+desc = aggs.desc
+count = aggs.count
+sum = aggs.sum
+min = aggs.min
+max = aggs.max
+avg = aggs.avg
+string_join = aggs.string_join
+per = aggs.per
+
+#------------------------------------------------------
+# Exports
+#------------------------------------------------------
 
 __all__ = [
-    "Model", "Concept", "Relationship", "RelationshipReading", "Expression", "Fragment", "Error", "Field",
-    "AnyEntity", "String", "Integer", "Int64", "Int128", "Float", "Decimal", "Bool",
-    "Date", "DateTime",
-    "RawSource", "Hash",
-    "select", "where", "require", "define", "distinct", "union", "data",
-    "rank", "asc", "desc",
-    "count", "sum", "min", "max", "avg", "per",
-    "not_",
+    "Alias", "CoreLibrary", "Concept", "Data", "Distinct", "Expression", "Field", "FieldRef", "Fragment",
+    "Library", "Literal", "Match", "Model", "New", "Not", "Chain", "Property", "Reading", "Relationship",
+    "Table", "Union", "Value", "Variable", "core",
+
+    "select", "define", "where", "require", "union", "data", "not_",
 ]

relationalai/semantics/backends/lqp/annotations.py

@@ -0,0 +1,11 @@
+from relationalai.semantics.frontend.base import Field, Library
+
+library = Library("lqp")
+
+Symbol = library.Type("Symbol")
+
+export = library.Relation("export", [])
+external = library.Relation("external", [])
+from_cdc = library.Relation("from_cdc", [])
+adhoc = library.Relation("adhoc", [])
+track = library.Relation("track", [Field.input("library", Symbol), Field.input("relation", Symbol)])

relationalai/semantics/backends/sql/sql_compiler.py

@@ -0,0 +1,327 @@
+from __future__ import annotations
+from collections import deque
+from dataclasses import dataclass, field
+from sqlglot import exp, select
+
+from ....util.structures import OrderedSet
+from ...metamodel.metamodel import Aggregate, Logical, Lookup, Model, Node, Relation, Task, Update, Var, Value
+from ...metamodel.pprint import pprint as mmpp
+from ...metamodel.rewriter import Rewriter, Walker
+from ...metamodel.metamodel_analyzer import Analyzer as MMAnalyzer
+
+from rich import print as rp
+
+#------------------------------------------------------
+# Simple flatten
+#------------------------------------------------------
+
+class SimpleFlattener(Walker):
+    def flatten(self, node: Node) -> list[Task]:
+        self.rules: list[Task] = []
+        self.stack: list[Task] = []
+        self(node)
+        return self.rules
+
+    def enter_logical(self, logical: Logical):
+        self.stack.append(logical)
+
+    def logical(self, logical: Logical):
+        self.stack.pop()
+
+    def update(self, update: Update):
+        # build a rule for just this update
+        body = OrderedSet()
+        for frame in self.stack:
+            if isinstance(frame, Logical):
+                for task in frame.body:
+                    if not isinstance(task, (Logical, Update)):
+                        body.add(task)
+        body.add(update)
+        self.rules.append(Logical(body=tuple(body)))
+
+#------------------------------------------------------
+# SourceWalker
+#------------------------------------------------------
+
+class SourceWalker(Walker):
+    def capture_sources(self, node: Task, analysis:SQLAnalysis):
+        self.stack = [OrderedSet()]
+        self.root = node
+        self.analysis = analysis
+        self(node)
+
+    def enter_logical(self, logical: Logical):
+        self.stack.append(OrderedSet())
+
+    def logical(self, logical: Logical):
+        self.stack.pop()
+
+    def lookup(self, lookup: Lookup):
+        self.analysis.get(lookup.relation)
+        self.stack[-1].add(lookup.relation)
+
+    def update(self, update: Update):
+        info = self.analysis.get(update.relation)
+        info.sources.add(self.root)
+        for frame in self.stack:
+            for dep in frame:
+                info.edges.add(dep)
+                self.analysis.get(dep)
+
+#------------------------------------------------------
+# RecursionWalker
+#------------------------------------------------------
+
+class RecursionWalker(Walker):
+    def is_simple(self, node: Task, analysis: SQLAnalysis) -> bool:
+        self._recursive_count = 0
+        self._has_agg = False
+        self.analysis = analysis
+        self(node)
+        if not self._has_agg and self._recursive_count <= 1:
+            return True
+        return False
+
+    def aggregate(self, aggregate: Aggregate):
+        self._has_agg = True
+
+    def lookup(self, lookup: Lookup):
+        if self.analysis.get(lookup.relation).recursive:
+            self._recursive_count += 1
+
+#------------------------------------------------------
+# VarMapper
+#------------------------------------------------------
+
+class VarMapper(Rewriter):
+    def map_vars(self, mapping: dict[Var, Value], node: Node):
+        self.mapping = mapping
+        return self(node)
+
+    def var(self, var: Var):
+        if var in self.mapping:
+            return self.mapping[var]
+        return var
+
+#------------------------------------------------------
+# Analysis
+#------------------------------------------------------
+
+mapper = VarMapper()
+
+@dataclass
+class RelationInfo:
+    relation: Relation
+    sources: OrderedSet[Task] = field(default_factory=OrderedSet[Task])
+    edges: OrderedSet[Relation] = field(default_factory=OrderedSet[Relation])
+
+    head_vars: list[Var] = field(default_factory=list)
+    body: OrderedSet[Task] = field(default_factory=OrderedSet[Task])
+
+    recursive: bool = field(default=False)
+
+    def map_lookup(self, lookup: Lookup):
+        mapping = {self.head_vars[i]: lookup.args[i] for i in range(len(self.head_vars))}
+        mapped = []
+        for task in self.body:
+            mapped.append(mapper.map_vars(mapping, task))
+        return mapped
+
+@dataclass
+class SQLAnalysis:
+    relations: dict[Relation, RelationInfo] = field(default_factory=dict)
+    source_walker = SourceWalker()
+
+    def get(self, relation: Relation) -> RelationInfo:
+        if relation not in self.relations:
+            self.relations[relation] = RelationInfo(relation=relation)
+        return self.relations[relation]
+
+    def analyze(self, task:Task):
+        self.source_walker.capture_sources(task, self)
+
+    def construct_sccs(self):
+        sccs = find_sccs(self.relations)
+        self.sccs = sccs
+        for info in self.relations.values():
+            info.recursive = False
+        for scc in sccs:
+            for relation in scc.relations:
+                self.get(relation).recursive = scc.recursive
+
+        scc_order = scc_toposort(sccs, self.relations)
+        self.scc_order = scc_order
+
+#------------------------------------------------------
+# SCCs
+#------------------------------------------------------
+
+@dataclass
+class SCC:
+    relations: list[Relation]
+    recursive: bool
+
+    def strategy(self, analysis:SQLAnalysis) -> str:
+        if not self.recursive:
+            return "NORMAL"
+        relations = self.relations
+        if len(relations) == 1:
+            rel = relations[0]
+            sources = analysis.get(rel).sources
+            if all(RecursionWalker().is_simple(source, analysis) for source in sources):
+                return "CTE"
+        return "SEMI_NAIVE"
+
+def find_sccs(relations: dict[Relation, RelationInfo]) -> list[SCC]:
+    # find sccs using Tarjan's algorithm
+    index = 0
+    stack: list[Relation] = []
+    onstack: set[Relation] = set()
+    ix: dict[Relation, int] = {}
+    low: dict[Relation, int] = {}
+    results: list[SCC] = []
+
+    def strongconnect(v: Relation):
+        nonlocal index
+        ix[v] = low[v] = index
+        index += 1
+        stack.append(v)
+        onstack.add(v)
+
+        info = relations.get(v)
+        edges = info.edges if info else ()
+        for w in edges:
+            if w not in ix:
+                strongconnect(w)
+                low[v] = low[v] if low[v] < low[w] else low[w]
+            elif w in onstack:
+                low[v] = low[v] if low[v] < ix[w] else ix[w]
+
+        if low[v] == ix[v]:
+            comp: list[Relation] = []
+            while True:
+                w = stack.pop()
+                onstack.discard(w)
+                comp.append(w)
+                if w == v:
+                    break
+            # size > 1 or a singleton with self-loop
+            comp_info = relations.get(comp[0])
+            comp_edges = comp_info.edges if comp_info else ()
+            recursive = (len(comp) > 1) or (comp[0] in comp_edges)
+            results.append(SCC(comp, recursive))
+
+    # visit all vertices, including isolated
+    nodes = set(relations.keys()) | {w for info in relations.values() for w in info.edges}
+    for v in nodes:
+        if v not in ix:
+            strongconnect(v)
+
+    return results
+
+def scc_toposort(components: list[SCC], relations: dict[Relation, RelationInfo]) -> list[SCC]:
+    comp_id = {v: i for i, comp in enumerate(components) for v in comp.relations}
+    n = len(components)
+
+    # Build reversed condensed DAG: dep_comp -> rel_comp
+    adj: list[set[int]] = [set() for _ in range(n)]
+    indeg: list[int] = [0] * n
+
+    for rel, info in relations.items():
+        cr = comp_id[rel]
+        for dep in info.edges:
+            cd = comp_id.get(dep)
+            if cd is None:
+                continue
+            if cr == cd:
+                continue
+            # edge: dependency SCC -> relation SCC
+            if cr not in adj[cd]:
+                adj[cd].add(cr)
+                indeg[cr] += 1
+
+    # Kahn’s algorithm
+    q: deque[int] = deque(i for i, d in enumerate(indeg) if d == 0)
+    order: list[SCC] = []
+    while q:
+        i = q.popleft()
+        order.append(components[i])
+        for j in adj[i]:
+            indeg[j] -= 1
+            if indeg[j] == 0:
+                q.append(j)
+
+    assert len(order) == n, "SCC DAG unexpectedly cyclic"
+    return order
+
+#------------------------------------------------------
+# Context
+#------------------------------------------------------
+
+class Frame:
+    pass
+
+class Context:
+    pass
+
+#------------------------------------------------------
+# Compiler
+#------------------------------------------------------
+
+class SQLCompiler:
+
+    def compile(self, model:Model):
+        MMAnalyzer().analyze(model)
+        analysis = SQLAnalysis()
+
+        rules = []
+        flattener = SimpleFlattener()
+        # mmpp(model)
+
+        assert isinstance(model.root, Logical)
+        for sub_task in model.root.body:
+            flat_rules = flattener.flatten(sub_task)
+            rules.extend(flat_rules)
+            for rule in flat_rules:
+                analysis.analyze(rule)
+
+        analysis.construct_sccs()
+        sccs = analysis.sccs
+
+        # print("\nStrongly Connected Components:")
+        # for i, scc in enumerate(sccs, start=1):
+        #     tag = f" (recursive {scc.strategy(analysis)})" if scc.recursive else ""
+        #     print(f"  SCC {i}:{tag}")
+        #     for r in scc.relations:
+        #         print(f"    - {r}")
+
+        # # ---- Topological order of SCCs (evaluation phases) ----
+        # ordered_sccs = analysis.scc_order
+
+        # # TODO: handle relations that didn't end up in any SCC?
+
+        # # walk the ordered components forward collapsing as we go
+        # print("\nSCC evaluation order (topologically sorted):")
+        # for phase_idx, scc in enumerate(ordered_sccs, start=1):
+        #     tag = f" (recursive {scc.strategy(analysis)})" if scc.recursive else ""
+        #     rp(f"[yellow bold]  Phase {phase_idx}:{tag}")
+        #     rp("[yellow bold]------------------------------------------------------------")
+        #     for relation in scc.relations:
+        #         info = analysis.get(relation)
+        #         rp(f"[cyan bold]  {relation}")
+        #         # print("  Head Vars:")
+        #         # for hv in info.head_vars:
+        #         #     print(f"    - {hv}")
+        #         # print("  Body:")
+        #         # for task in info.body:
+        #         #     print(f"    - {task}")
+        #         print("    Sources:")
+        #         for source in info.sources:
+        #             # print(f"        - {source}")
+        #             mmpp(source, indent=2)
+        #         print("    Edges:")
+        #         for edge in info.edges:
+        #             print(f"        - {edge}")
+        #     print("\n")
+
+