relationalai 0.12.4__py3-none-any.whl → 0.12.7__py3-none-any.whl

relationalai/{semantics/reasoners/graph → experimental}/paths/tests/tests_one_sided_ball_upto_multiple.py

@@ -1,6 +1,6 @@
 from relationalai.semantics import Model, Integer, define, select
-from relationalai.semantics.reasoners.graph.paths.graph import Graph
-from relationalai.semantics.reasoners.graph.paths.path_algorithms.one_sided_ball_upto import ball_upto
+from relationalai.experimental.paths.graph import Graph
+from relationalai.experimental.paths.path_algorithms.one_sided_ball_upto import ball_upto
 
 
 # First test with grid graph and multiple source and target nodes

relationalai/{semantics/reasoners/graph → experimental}/paths/tests/tests_one_sided_ball_upto_single.py

@@ -1,6 +1,6 @@
 from relationalai.semantics import Model, Integer, define, select
-from relationalai.semantics.reasoners.graph.paths.graph import Graph
-from relationalai.semantics.reasoners.graph.paths.path_algorithms.one_sided_ball_upto import ball_upto
+from relationalai.experimental.paths.graph import Graph
+from relationalai.experimental.paths.path_algorithms.one_sided_ball_upto import ball_upto
 
 
 # First test with grid graph

relationalai/{semantics/reasoners/graph → experimental}/paths/tests/tests_single_paths.py

@@ -1,6 +1,6 @@
 from relationalai.semantics import Model, Integer, define, select
-from relationalai.semantics.reasoners.graph.paths.graph import Graph
-from relationalai.semantics.reasoners.graph.paths.path_algorithms.single import single_shortest_path
+from relationalai.experimental.paths.graph import Graph
+from relationalai.experimental.paths.path_algorithms.single import single_shortest_path
 
 
 model = Model("test_single_paths", dry_run=False)

relationalai/{semantics/reasoners/graph → experimental}/paths/tests/tests_single_walks.py

@@ -1,6 +1,6 @@
 from relationalai.semantics import Model, Integer, define, select
-from relationalai.semantics.reasoners.graph.paths.graph import Graph
-from relationalai.semantics.reasoners.graph.paths.path_algorithms.single import single_walk
+from relationalai.experimental.paths.graph import Graph
+from relationalai.experimental.paths.path_algorithms.single import single_walk
 
 
 model = Model("test_single_paths", dry_run=False)

relationalai/{semantics/reasoners/graph → experimental}/paths/tests/tests_single_walks_undirected.py

@@ -1,6 +1,6 @@
 from relationalai.semantics import Model, Integer, define, select
-from relationalai.semantics.reasoners.graph.paths.graph import Graph
-from relationalai.semantics.reasoners.graph.paths.path_algorithms.single import single_walk
+from relationalai.experimental.paths.graph import Graph
+from relationalai.experimental.paths.path_algorithms.single import single_walk
 
 
 model = Model("test_single_paths", dry_run=False)

relationalai/{semantics/reasoners/graph → experimental}/paths/tests/tests_two_sided_balls_repetition_multiple.py

@@ -1,6 +1,6 @@
 from relationalai.semantics import Model, Integer, define, select
-from relationalai.semantics.reasoners.graph.paths.graph import Graph
-from relationalai.semantics.reasoners.graph.paths.path_algorithms.two_sided_balls_repetition import two_balls_repetition
+from relationalai.experimental.paths.graph import Graph
+from relationalai.experimental.paths.path_algorithms.two_sided_balls_repetition import two_balls_repetition
 
 
 # Test with diamond graph

relationalai/{semantics/reasoners/graph → experimental}/paths/tests/tests_two_sided_balls_repetition_single.py

@@ -1,6 +1,6 @@
 from relationalai.semantics import Model, Integer, define, select
-from relationalai.semantics.reasoners.graph.paths.graph import Graph
-from relationalai.semantics.reasoners.graph.paths.path_algorithms.two_sided_balls_repetition import two_balls_repetition
+from relationalai.experimental.paths.graph import Graph
+from relationalai.experimental.paths.path_algorithms.two_sided_balls_repetition import two_balls_repetition
 
 
 # Test with diamond graph

relationalai/{semantics/reasoners/graph → experimental}/paths/tests/tests_two_sided_balls_upto_multiple.py

@@ -1,6 +1,6 @@
 from relationalai.semantics import Model, Integer, define, select
-from relationalai.semantics.reasoners.graph.paths.graph import Graph
-from relationalai.semantics.reasoners.graph.paths.path_algorithms.two_sided_balls_upto import two_balls_upto
+from relationalai.experimental.paths.graph import Graph
+from relationalai.experimental.paths.path_algorithms.two_sided_balls_upto import two_balls_upto
 
 
 # Test with grid graph

relationalai/{semantics/reasoners/graph → experimental}/paths/tests/tests_two_sided_balls_upto_single.py

@@ -1,6 +1,6 @@
 from relationalai.semantics import Model, Integer, define, select
-from relationalai.semantics.reasoners.graph.paths.graph import Graph
-from relationalai.semantics.reasoners.graph.paths.path_algorithms.two_sided_balls_upto import two_balls_upto
+from relationalai.experimental.paths.graph import Graph
+from relationalai.experimental.paths.path_algorithms.two_sided_balls_upto import two_balls_upto
 
 
 # Test with grid graph

relationalai/{semantics/reasoners/graph → experimental}/paths/tests/tests_usp_nsp_multiple.py

@@ -1,6 +1,6 @@
 from relationalai.semantics import Model, Integer, define, select
-from relationalai.semantics.reasoners.graph.paths.graph import Graph
-from relationalai.semantics.reasoners.graph.paths.path_algorithms.usp import compute_usp, compute_nsp
+from relationalai.experimental.paths.graph import Graph
+from relationalai.experimental.paths.path_algorithms.usp import compute_usp, compute_nsp
 
 
 # First test for usp with grid graph

relationalai/{semantics/reasoners/graph → experimental}/paths/tests/tests_usp_nsp_single.py

@@ -1,6 +1,6 @@
 from relationalai.semantics import Model, Integer, define, select
-from relationalai.semantics.reasoners.graph.paths.graph import Graph
-from relationalai.semantics.reasoners.graph.paths.path_algorithms.usp import compute_usp, compute_nsp
+from relationalai.experimental.paths.graph import Graph
+from relationalai.experimental.paths.path_algorithms.usp import compute_usp, compute_nsp
 
 
 # First test for usp with grid graph

relationalai/semantics/__init__.py

@@ -1,3 +1,7 @@
+"""
+The RelationalAI Semantics Module.
+"""
+
 # Mark this package's docstrings for inclusion
 # in automatically generated web documentation.
 __include_in_docs__ = True

relationalai/semantics/internal/annotations.py

@@ -6,3 +6,4 @@ concept_population = Relationship.builtins["concept_population"]
 function = Relationship.builtins["function"]
 from_cdc = Relationship.builtins["from_cdc"]
 track = Relationship.builtins["track"]
+recursion_config = Relationship.builtins["recursion_config"]

relationalai/semantics/internal/internal.py

@@ -2550,8 +2550,6 @@ class Fragment():
         from .snowflake import Table
         assert isinstance(table, Table), "Only Snowflake tables are supported for now"
 
-        result_cols = table._col_names
-
         clone = Fragment(parent=self)
         clone._is_export = True
         qb_model = clone._model or Model("anon")
@@ -2559,8 +2557,7 @@ class Fragment():
         clone._source = runtime_env.get_source_pos()
         with debugging.span("query", dsl=str(clone), **with_source(clone), meta=clone._meta):
             query_task = qb_model._compiler.fragment(clone)
-            qb_model._to_executor().execute(ir_model, query_task, result_cols=result_cols, export_to=table._fqn, update=update, meta=clone._meta)
-
+            qb_model._to_executor().execute(ir_model, query_task, export_to=table, update=update, meta=clone._meta)
 
 #--------------------------------------------------
 # Select / Where
@@ -3419,6 +3416,8 @@ class Compiler():
             return out
 
         elif isinstance(item, TypeRef):
+            if isinstance(item._thing, Relationship):
+                return self.to_relation(item._thing)
             concept = to_type(item)
             if not concept:
                 raise ValueError(f"Cannot find concept for {item}, {type(item)}")

relationalai/semantics/internal/snowflake.py

@@ -12,7 +12,18 @@ from . import internal as b, annotations as anns
 from relationalai import debugging
 from relationalai.errors import UnsupportedColumnTypesWarning
 from snowflake.snowpark.context import get_active_session
+from typing import ClassVar, Optional
 
+#--------------------------------------------------
+# Iceberg Configuration
+#--------------------------------------------------
+@dataclass
+class IcebergConfig:
+    """Configuration for exporting to Iceberg tables."""
+    external_volume: str | None = None
+    default: ClassVar[Optional["IcebergConfig"]]
+
+IcebergConfig.default = IcebergConfig()
 #--------------------------------------------------
 # Helpers
 #--------------------------------------------------
@@ -191,7 +202,7 @@ class Table():
     _schemas:dict[tuple[str, str], SchemaInfo] = {}
     _used_sources:OrderedSet[Table] = ordered_set()
 
-    def __init__(self, fqn:str, cols:list[str]|None=None, schema:dict[str, str|b.Concept]|None=None) -> None:
+    def __init__(self, fqn:str, cols:list[str]|None=None, schema:dict[str, str|b.Concept]|None=None, config: IcebergConfig|None=None) -> None:
         self._fqn = fqn
         parser = IdentityParser(fqn, require_all_parts=True)
         self._database, self._schema, self._table, self._fqn = parser.to_list()
@@ -201,6 +212,8 @@ class Table():
         self._ref = self._concept.ref("row_id")
         self._cols = {}
         self._col_names = cols
+        self._iceberg_config = config
+        self._is_iceberg = config is not None
         info = self._schemas.get((self._database, self._schema))
         if not info:
             info = self._schemas[(self._database, self._schema)] = SchemaInfo(self._database, self._schema)

relationalai/semantics/lqp/builtins.py

@@ -12,4 +12,5 @@ annotations_to_emit = FrozenOrderedSet([
     adhoc.name,
     builtins.function.name,
     builtins.track.name,
+    builtins.recursion_config.name,
 ])

relationalai/semantics/lqp/constructors.py

@@ -1,6 +1,5 @@
 from typing import Tuple
 from relationalai.semantics.lqp import ir as lqp
-from relationalai.semantics.metamodel.ir import sanitize
 
 def mk_and(args: list[lqp.Formula]) -> lqp.Formula:
     # Flatten nested conjunctions
@@ -49,10 +48,6 @@ def mk_specialized_value(value) -> lqp.SpecializedValue:
 def mk_value(value) -> lqp.Value:
     return lqp.Value(value=value, meta=None)
 
-def mk_var(name: str) -> lqp.Var:
-    _name = '_' if name == '_' else sanitize(name)
-    return lqp.Var(name=_name, meta=None)
-
 def mk_type(typename: lqp.TypeName, parameters: list[lqp.Value]=[]) -> lqp.Type:
     return lqp.Type(type_name=typename, parameters=parameters, meta=None)
 

relationalai/semantics/lqp/executor.py

@@ -4,7 +4,7 @@ import atexit
 import re
 
 from pandas import DataFrame
-from typing import Any, Optional, Literal
+from typing import Any, Optional, Literal, TYPE_CHECKING
 from snowflake.snowpark import Session
 import relationalai as rai
 
@@ -20,10 +20,14 @@ from relationalai.semantics.lqp.ir import convert_transaction, validate_lqp
 from relationalai.clients.config import Config
 from relationalai.clients.snowflake import APP_NAME
 from relationalai.clients.types import TransactionAsyncResponse
-from relationalai.clients.util import IdentityParser
+from relationalai.clients.util import IdentityParser, escape_for_f_string
 from relationalai.tools.constants import USE_DIRECT_ACCESS, QUERY_ATTRIBUTES_HEADER
 from relationalai.tools.query_utils import prepare_metadata_for_headers
 
+if TYPE_CHECKING:
+    from relationalai.semantics.snowflake import Table
+
+
 class LQPExecutor(e.Executor):
     """Executes LQP using the RAI client."""
 
@@ -172,12 +176,12 @@ class LQPExecutor(e.Executor):
         elif len(all_errors) > 1:
             raise errors.RAIExceptionSet(all_errors)
 
-    def _export(self, txn_id: str, export_info: tuple, dest_fqn: str, actual_cols: list[str], declared_cols: list[str], update:bool):
+    def _export(self, txn_id: str, export_info: tuple, dest: Table, actual_cols: list[str], declared_cols: list[str], update: bool):
         # At this point of the export, we assume that a CSV file has already been written
         # to the Snowflake Native App stage area. Thus, the purpose of this method is to
         # copy the data from the CSV file to the destination table.
         _exec = self.resources._exec
-        dest_database, dest_schema, dest_table, _ = IdentityParser(dest_fqn, require_all_parts=True).to_list()
+        dest_database, dest_schema, dest_table, _ = IdentityParser(dest._fqn, require_all_parts=True).to_list()
         filename = export_info[0]
         result_table_name = filename + "_table"
 
@@ -203,8 +207,28 @@ class LQPExecutor(e.Executor):
                 # destination table. This step also cleans up the result table.
                 out_sample = _exec(f"select * from {APP_NAME}.results.{result_table_name} limit 1;")
                 names = self._build_projection(declared_cols, actual_cols, column_fields, out_sample)
+                dest_fqn = dest._fqn
                 try:
                     if not update:
+                        createTableLogic = f"""
+                                        CREATE TABLE {dest_fqn} AS
+                                        SELECT {names}
+                                        FROM {APP_NAME}.results.{result_table_name};
+                        """
+                        if dest._is_iceberg:
+                            assert dest._iceberg_config is not None
+                            external_volume_clause = ""
+                            if dest._iceberg_config.external_volume:
+                                external_volume_clause = f"EXTERNAL_VOLUME = '{dest._iceberg_config.external_volume}'"
+                            createTableLogic = f"""
+                                        CREATE ICEBERG TABLE {dest_fqn}
+                                        CATALOG = "SNOWFLAKE"
+                                        {external_volume_clause}
+                                        AS
+                                        SELECT {names}
+                                        FROM {APP_NAME}.results.{result_table_name};
+                            """
+
                         _exec(f"""
                             BEGIN
                                 -- Check if table exists
@@ -227,9 +251,7 @@ class LQPExecutor(e.Executor):
                                 ELSE
                                     -- Create table based on the SELECT
                                     EXECUTE IMMEDIATE '
-                                        CREATE TABLE {dest_fqn} AS
-                                        SELECT {names}
-                                        FROM {APP_NAME}.results.{result_table_name};
+                                        {escape_for_f_string(createTableLogic)}
                                     ';
                                 END IF;
                             END;
@@ -376,7 +398,7 @@ class LQPExecutor(e.Executor):
         return final_model, export_info, txn_proto
 
     # TODO (azreika): This should probably be split up into exporting and other processing. There are quite a lot of arguments here...
-    def _process_results(self, task: ir.Task, final_model: ir.Model, raw_results: TransactionAsyncResponse, result_cols: Optional[list[str]], export_info: Optional[tuple], export_to: Optional[str], update: bool) -> DataFrame:
+    def _process_results(self, task: ir.Task, final_model: ir.Model, raw_results: TransactionAsyncResponse, export_info: Optional[tuple], export_to: Optional[Table], update: bool) -> DataFrame:
         cols, extra_cols = self._compute_cols(task, final_model)
 
         df, errs = result_helpers.format_results(raw_results, cols)
@@ -391,6 +413,8 @@ class LQPExecutor(e.Executor):
             assert cols, "No columns found in the output"
             assert isinstance(raw_results, TransactionAsyncResponse) and raw_results.transaction, "Invalid transaction result"
 
+            result_cols = export_to._col_names
+
             if result_cols is not None:
                 assert all(col in result_cols or col in extra_cols for col in cols)
             else:
@@ -403,7 +427,7 @@ class LQPExecutor(e.Executor):
         return self._postprocess_df(self.config, df, extra_cols)
 
     def execute(self, model: ir.Model, task: ir.Task, format: Literal["pandas", "snowpark"] = "pandas",
-                result_cols: Optional[list[str]] = None, export_to: Optional[str] = None,
+                export_to: Optional[Table] = None,
                 update: bool = False, meta: dict[str, Any] | None = None) -> DataFrame:
         self.prepare_data()
         previous_model = self._last_model
@@ -433,7 +457,7 @@ class LQPExecutor(e.Executor):
         assert isinstance(raw_results, TransactionAsyncResponse)
 
         try:
-            return self._process_results(task, final_model, raw_results, result_cols, export_info, export_to, update)
+            return self._process_results(task, final_model, raw_results, export_info, export_to, update)
         except Exception as e:
             # If processing the results failed, revert to the previous model.
             self._last_model = previous_model

relationalai/semantics/lqp/intrinsics.py

@@ -1,13 +1,13 @@
 from datetime import datetime, timezone
 
 from relationalai.semantics.lqp import ir as lqp
-from relationalai.semantics.lqp.constructors import mk_abstraction, mk_value, mk_var, mk_type, mk_primitive
+from relationalai.semantics.lqp.constructors import mk_abstraction, mk_value, mk_type, mk_primitive
 from relationalai.semantics.lqp.utils import lqp_hash
 
 def mk_intrinsic_datetime_now() -> lqp.Def:
     """Constructs a definition of the current datetime."""
     id = lqp_hash("__pyrel_lqp_intrinsic_datetime_now")
-    out = mk_var("out")
+    out = lqp.Var(name="out", meta=None)
     out_type = mk_type(lqp.TypeName.DATETIME)
     now = mk_value(lqp.DateTimeValue(value=datetime.now(timezone.utc), meta=None))
     datetime_now = mk_abstraction(

relationalai/semantics/lqp/model2lqp.py

@@ -7,7 +7,7 @@ from relationalai.semantics.lqp.pragmas import pragma_to_lqp_name
 from relationalai.semantics.lqp.types import meta_type_to_lqp
 from relationalai.semantics.lqp.constructors import (
     mk_abstraction, mk_and, mk_exists, mk_or, mk_pragma, mk_primitive,
-    mk_specialized_value, mk_type, mk_value, mk_var, mk_attribute
+    mk_specialized_value, mk_type, mk_value, mk_attribute
 )
 from relationalai.semantics.lqp.utils import TranslationCtx, gen_unique_var
 from relationalai.semantics.lqp.validators import assert_valid_input
@@ -253,7 +253,7 @@ def _translate_rank(ctx: TranslationCtx, rank: ir.Rank, body: lqp.Formula) -> lq
     # to convert it to Int128.
     result_var, _ = _translate_term(ctx, rank.result)
     # The primitive will return an Int64 result, so we need a var to hold the intermediary.
-    result_64_var = gen_unique_var(ctx, "result_64")
+    result_64_var = gen_unique_var(ctx, "v_rank")
     result_64_type = mk_type(lqp.TypeName.INT)
 
     cast = lqp.Cast(input=result_64_var, result=result_var, meta=None)
@@ -273,7 +273,7 @@ def _translate_descending_rank(ctx: TranslationCtx, limit: int, result: lqp.Var,
     result_type = mk_type(lqp.TypeName.INT)
 
     # Rename abstracted args in the body to new variable names
-    var_map = {var.name: gen_unique_var(ctx, var.name) for (var, _) in abstr_args}
+    var_map = {var.name: gen_unique_var(ctx, 't_' + var.name) for (var, _) in abstr_args}
     body = utils.rename_vars_formula(body, var_map)
     new_abstr_args = [(var_map[var.name], typ) for (var, typ) in abstr_args]
 
@@ -306,7 +306,7 @@ def _translate_descending_rank(ctx: TranslationCtx, limit: int, result: lqp.Var,
     aggr_abstr_args = new_abstr_args + [(count_var, count_type)]
     count_aggr = lqp.Reduce(
         op=lqp_operator(
-            ctx.var_names,
+            ctx,
             "count",
             "count",
             mk_type(lqp.TypeName.INT)
@@ -340,7 +340,7 @@ def _translate_ascending_rank(ctx: TranslationCtx, limit: int, result_var: lqp.V
     terms = [result_var] + [v[0] for v in abstr_args]
 
     # Rename abstracted args in the body to new variable names
-    var_map = {var.name: gen_unique_var(ctx, var.name) for (var, _) in abstr_args}
+    var_map = {var.name: gen_unique_var(ctx, 't_' + var.name) for (var, _) in abstr_args}
     body = utils.rename_vars_formula(body, var_map)
     new_abstr_args = [(var_map[var.name], typ) for (var, typ) in abstr_args]
     sort_abstr = mk_abstraction(new_abstr_args, body)
@@ -431,7 +431,7 @@ def _translate_aggregate(ctx: TranslationCtx, aggr: ir.Aggregate, body: lqp.Form
         (sum_var, sum_type) = abstr_args[-2]
 
         result = lqp.Reduce(
-            op=lqp_avg_op(ctx.var_names, aggr.aggregation.name, sum_var.name, sum_type),
+            op=lqp_avg_op(ctx, aggr.aggregation.name, sum_var.name, sum_type),
             body=mk_abstraction(abstr_args, body),
             terms=[sum_result, count_result],
             meta=None,
@@ -461,9 +461,10 @@ def _translate_aggregate(ctx: TranslationCtx, aggr: ir.Aggregate, body: lqp.Form
     # `input_args`` hold the types of the input arguments, but they may have been modified
     # if we're dealing with a count, so we use `abstr_args` to find the type.
     (aggr_arg, aggr_arg_type) = abstr_args[-1]
+
     # Group-bys do not need to be handled at all, since they are introduced outside already
     reduce = lqp.Reduce(
-        op=lqp_operator(ctx.var_names, aggr.aggregation.name, aggr_arg.name, aggr_arg_type),
+        op=lqp_operator(ctx, aggr.aggregation.name, aggr_arg.name, aggr_arg_type),
         body=mk_abstraction(abstr_args, body),
         terms=output_vars,
         meta=None
@@ -522,9 +523,8 @@ def _translate_term(ctx: TranslationCtx, term: ir.Value) -> Tuple[lqp.Term, lqp.
         # TODO: ScalarType is not like other terms, should be handled separately.
         return to_lqp_value(term.name, types.String), meta_type_to_lqp(types.String)
     elif isinstance(term, ir.Var):
-        name = ctx.var_names.get_name_by_id(term.id, term.name)
         t = meta_type_to_lqp(term.type)
-        return mk_var(name), t
+        return _translate_var(ctx, term), t
     else:
         assert isinstance(term, ir.Literal), f"Cannot translate value {term!r} of type {type(term)} to LQP Term; neither Var nor Literal."
         v = to_lqp_value(term.value, term.type)
@@ -542,6 +542,12 @@ def _translate_to_atom(ctx: TranslationCtx, task: ir.Lookup) -> lqp.Formula:
 
     if task.relation == builtins.join:
         return _translate_join(ctx, task)
+    elif task.relation == builtins.infomap:
+        return _translate_infomap(ctx, task)
+    elif task.relation == builtins.louvain:
+        return _translate_louvain(ctx, task)
+    elif task.relation == builtins.label_propagation:
+        return _translate_label_propagation(ctx, task)
 
     terms = []
     term_types = []
@@ -667,6 +673,92 @@ def _extract_pyrel_error_ids(ctx: TranslationCtx, model: ir.Model) -> list[Tuple
 
     return pyrel_error_attrs
 
+# Translate a relation reference into an abstraction over its fields.
+def _translate_relation_ref(ctx: TranslationCtx, relation: ir.Relation) -> lqp.Abstraction:
+    projection = []
+    for field in relation.fields:
+        var = gen_unique_var(ctx, field.name)
+        typ = meta_type_to_lqp(field.type)
+        projection.append((var, typ))
+
+    rid = get_relation_id(ctx, relation, projection)
+    atom = lqp.Atom(name=rid, terms=[var for (var, _) in projection], meta=None)
+    return mk_abstraction(projection, atom)
+
+# Common translation logic for graph algorithms.
+# task.args[0] : normalized weight list (int64, int64, float)
+# task.args[1] : normalized node count (relation or constant int64)
+# task.args[2] : normalized edge count (relation or constant int64)
+# task.args[3:-3] : algorithm parameters (var or constant)
+# task.args[-3] : diagnostic info
+# task.args[-2] : node index
+# task.args[-1] : community ident
+def _translate_graph_common(name: str, ctx: TranslationCtx, task: ir.Lookup):
+    abstractions = []
+
+    assert isinstance(task.args[0], ir.Relation), \
+        f"Expected relation as first arg to {name}, got {task.args[0]}:{type(task.args[0])}"
+    abstractions.append(_translate_relation_ref(ctx, task.args[0]))
+
+    # Allow constant args for node and edge count
+    for arg in task.args[1:3]:
+        if isinstance(arg, ir.Relation):
+            abst = _translate_relation_ref(ctx, arg)
+            typ = abst.vars[0][1]
+            assert typ.type_name == lqp.TypeName.INT, \
+                f"Expected Int64 types for node and edge counts, got type {typ.type_name}"
+            abstractions.append(abst)
+        else:
+            var, typ, eq = binding_to_lqp_var(ctx, arg)
+            assert eq is not None, \
+                f"Expected equality formula for {name} arg {arg}:{type(arg)}"
+            abstractions.append(mk_abstraction([(var, typ)], mk_and([eq])))
+
+    for arg in task.args[3:-3]:
+        var, typ, eq = binding_to_lqp_var(ctx, arg)
+        if eq:
+            abstractions.append(mk_abstraction([(var, typ)], mk_and([eq])))
+        else:
+            print(f"Primitive graph algorithm arg without eq:\n var:{var}, typ:{typ}\n arg:{arg}:{type(arg)}")
+            abstractions.append(mk_abstraction([(var, typ)], mk_and([])))
+
+    terms = []
+    for arg in task.args[-3:]:
+        term, _ = _translate_relterm(ctx, arg)
+        terms.append(term)
+
+    return (abstractions, terms)
+
+def _translate_infomap(ctx: TranslationCtx, task: ir.Lookup) -> lqp.Formula:
+    abstractions, terms = _translate_graph_common("infomap", ctx, task)
+
+    return lqp.FFI(
+        meta=None,
+        name="rel_primitive_infomap",
+        args=abstractions,
+        terms=terms,
+    )
+
+def _translate_louvain(ctx: TranslationCtx, task: ir.Lookup) -> lqp.Formula:
+    abstractions, terms = _translate_graph_common("louvain", ctx, task)
+
+    return lqp.FFI(
+        meta=None,
+        name="rel_primitive_louvain",
+        args=abstractions,
+        terms=terms,
+    )
+
+def _translate_label_propagation(ctx: TranslationCtx, task: ir.Lookup) -> lqp.Formula:
+    abstractions, terms = _translate_graph_common("label_propagation", ctx, task)
+
+    return lqp.FFI(
+        meta=None,
+        name="rel_primitive_async_label_propagation",
+        args=abstractions,
+        terms=terms,
+    )
+
 # Hard-coded implementation of Rel's string_join
 def _translate_join(ctx: TranslationCtx, task: ir.Lookup) -> lqp.Formula:
     assert len(task.args) == 3
@@ -708,3 +800,7 @@ def _translate_join(ctx: TranslationCtx, task: ir.Lookup) -> lqp.Formula:
     output_term = _translate_term(ctx, target)[0]
 
     return lqp.Reduce(meta=None, op=op, body=body, terms=[output_term])
+
+def _translate_var(ctx: TranslationCtx, term: ir.Var) -> lqp.Var:
+    name = ctx.var_names.get_name_by_id(term.id, term.name)
+    return lqp.Var(name=name, meta=None)

relationalai/semantics/lqp/passes.py

@@ -7,16 +7,18 @@ from relationalai.semantics.metamodel.util import FrozenOrderedSet
 from relationalai.semantics.metamodel.rewrite import Flatten
 
 from ..metamodel.rewrite import DischargeConstraints, DNFUnionSplitter, ExtractNestedLogicals
-from .rewrite import CDC, ExtractCommon, ExtractKeys, FDConstraints, QuantifyVars, Splinter
+from .rewrite import CDC, ExtractCommon, ExtractKeys, FunctionAnnotations, QuantifyVars, Splinter
 
 from relationalai.semantics.lqp.utils import output_names
 
 from typing import cast, List, Sequence, Tuple, Union, Optional, Iterable
 from collections import defaultdict
+import pandas as pd
+import hashlib
 
 def lqp_passes() -> list[Pass]:
     return [
-        FDConstraints(),
+        FunctionAnnotations(),
         DischargeConstraints(),
         Checker(),
         CDC(), # specialize to physical relations before extracting nested and typing
@@ -337,7 +339,7 @@ class UnifyDefinitions(Pass):
         )
 
 # Creates intermediary relations for all Data nodes and replaces said Data nodes
-# with a Lookup into these created relations.
+# with a Lookup into these created relations. Reuse duplicate created relations.
 class EliminateData(Pass):
     def rewrite(self, model: ir.Model, options:dict={}) -> ir.Model:
         r = self.DataRewriter()
@@ -350,17 +352,25 @@ class EliminateData(Pass):
         # Counter for naming new relations.
         # It must be that new_count == len new_updates == len new_relations.
         new_count: int
+        # Cache for Data nodes to avoid creating duplicate intermediary relations
+        data_cache: dict[str, ir.Relation]
 
         def __init__(self):
             self.new_relations = []
             self.new_updates = []
             self.new_count = 0
+            self.data_cache = {}
             super().__init__()
 
-        # Create a new intermediary relation representing the Data (and pop it in
-        # new_updates/new_relations) and replace this Data with a Lookup of said
-        # intermediary.
-        def handle_data(self, node: ir.Data, parent: ir.Node) -> ir.Lookup:
+        # Create a cache key for a Data node based on its structure and content
+        def _data_cache_key(self, node: ir.Data) -> str:
+            values = pd.util.hash_pandas_object(node.data).values
+            return hashlib.sha256(bytes(values)).hexdigest()
+
+        def _intermediary_relation(self, node: ir.Data) -> ir.Relation:
+            cache_key = self._data_cache_key(node)
+            if cache_key in self.data_cache:
+                return self.data_cache[cache_key]
             self.new_count += 1
             intermediary_name = f"formerly_Data_{self.new_count}"
 
@@ -379,7 +389,6 @@ class EliminateData(Pass):
                                 f.lookup(rel_builtins.eq, [f.literal(val), var])
                                 for (val, var) in zip(row, node.vars)
                             ],
-                            hoisted = node.vars,
                         )
                         for row in node
                     ],
@@ -390,6 +399,16 @@ class EliminateData(Pass):
             ])
             self.new_updates.append(intermediary_update)
 
+            # Cache the result for reuse
+            self.data_cache[cache_key] = intermediary_relation
+
+            return intermediary_relation
+
+        # Create a new intermediary relation representing the Data (and pop it in
+        # new_updates/new_relations) and replace this Data with a Lookup of said
+        # intermediary.
+        def handle_data(self, node: ir.Data, parent: ir.Node) -> ir.Lookup:
+            intermediary_relation = self._intermediary_relation(node)
             replacement_lookup = f.lookup(intermediary_relation, node.vars)
 
             return replacement_lookup