relationalai 0.13.0.dev0__py3-none-any.whl → 0.13.1__py3-none-any.whl

relationalai/loaders/csv.py

@@ -0,0 +1,195 @@
+from __future__ import annotations
+import csv
+from datetime import date, datetime
+from typing import Any, cast
+
+import numpy
+import pandas
+
+from relationalai.clients.types import ImportSource, ImportSourceFile
+from relationalai.clients.client import ResourcesBase
+from relationalai.dsl import Graph, RelationRef, build, tag
+from relationalai.loaders.loader import TYPE_TO_REL_SCHEMA, Loader, compute_file_hash, compute_str_hash
+from relationalai.rel_utils import Char, assert_no_problems, emit_nested_relation
+from relationalai.loaders.types import Schema, Syntax
+from relationalai.metamodel import ActionType, Builtins, Var
+from relationalai.rel_emitter import sanitize
+from relationalai.std import rel
+
+
+def df_type(df_type_name):
+    if df_type_name == "bool":
+        type = Builtins.Bool
+    elif df_type_name == "int64":
+        type = Builtins.Number
+    elif df_type_name == "float64":
+        type = Builtins.Number
+    else:
+        type = Builtins.String
+    return type
+
+class ExternalRow:
+    def __init__(self, data, columns):
+        self._data = data
+        self._columns = columns
+
+    def __getitem__(self, index):
+        if isinstance(index, str):
+            return getattr(self, index)
+        return self._data[index]
+
+    def __getattr__(self, name):
+        if name in self._columns:
+            return self._data[self._columns.index(name)]
+        return object.__getattribute__(self, name)
+
+# See https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
+# for keyword arg docs.
+def load_file(graph:Graph, csv_file, **kwargs):
+    df = pandas.read_csv(csv_file, **kwargs)
+
+    # create subqueries for each column that consist of a data object
+    # and a bind for the column relation, also create a data ref
+    # add a range for the id
+    # when a dataref is used, it should add a get for the column relation
+    # the rest just works?
+
+    index = getattr(rel, sanitize(csv_file))
+    tag(index, Builtins.Type)
+
+    with graph.scope():
+        index.add(rel.range(0, len(df), 1))
+
+    id = Var(type=Builtins.Number)
+    items = []
+    for col in df.columns:
+        sub = df[[col]]
+        col_type = df_type(df[col].dtype)
+        with graph.scope(dynamic=True):
+            # By setting Builtins.RawData on the task, we're telling the denester to not
+            # put a reference to the parent scope in this one
+            graph._stack.active()._task.parents.append(Builtins.RawData)
+            v1 = []
+            v2 = []
+            for (i, v) in sub.itertuples():
+                if v is not None and v != "" and v is not False and (isinstance(v,str) or isinstance(v, date) or isinstance(v, datetime) or not numpy.isnan(v)):
+                    v1.append(i)
+                    v2.append(v)
+
+            v1_var = Var(type=Builtins.Number)
+            v2_var = Var(name=col, type=col_type)
+            graph._action(
+                build.relation_action(ActionType.Get, Builtins.RawData, [Var(value=v1), Var(value=v2), v1_var, v2_var])
+            )
+            temp = build.relation(sanitize(csv_file) + "_" + col, 2)
+            temp.parents.append(Builtins.Property)
+            graph._action(
+                build.relation_action(ActionType.Bind, temp, [v1_var, v2_var])
+            )
+            usage_var = Var(name=col, type=col_type)
+            items.append(RelationRef(graph, temp, [id, usage_var]))
+    graph._action(
+        build.relation_action(ActionType.Get, index._rel, [id])
+    )
+    return ExternalRow(items, [c for c in df.columns])
+
+class CSVLoader(Loader):
+    """Load CSV files and URLs in RAI DB using `load_csv`."""
+    type = "csv"
+
+    def load(self, provider: ResourcesBase, model: str, source: ImportSource, *, schema: Schema|None = None, syntax: Syntax|None = None, integration: dict|None = None):
+        from railib import api
+        assert isinstance(source, ImportSourceFile), "CSV Loader can only load from files and URLs, not {type(source).__name__}"
+        schema = schema or {}
+        syntax = syntax or {}
+
+        id = compute_str_hash(source.raw_path)
+        prefix = f"""def config[:path]: raw"{source.raw_path}" """
+        inputs = None
+
+        if not source.is_url():
+            with open(source.raw_path, "r") as csv_file:
+                data = csv_file.read()
+
+            id = compute_file_hash(source.raw_path)
+            prefix = "def config[:data]: data"
+            inputs = {"data": data}
+
+        relation = sanitize(source.name)
+
+        for key in ["delim", "quotechar", "escapechar"]:
+            if syntax.get(key, None) and not isinstance(syntax[key], Char):
+                syntax[key] = Char(syntax[key])
+
+        q = f"""
+        declare {relation}
+        def delete[:{relation}]: {relation}
+        def delete[:__resource, k, "{relation}"]: __resource[k, "{relation}"]
+
+        {prefix}
+        {emit_nested_relation("config[:syntax, ", cast(dict, syntax), api._syntax_options)}
+        {emit_nested_relation("config[:schema, ", {col: TYPE_TO_REL_SCHEMA.get(type, "string") for col, type in schema.items()})}
+        {emit_nested_relation("config[:integration, ", integration) if integration else ""}
+        {emit_nested_relation("insert[:__resource, ", {
+        "id": (relation, id),
+        "type": (relation, self.type),
+        "name": (relation, source.name)
+        })}
+        def insert[:__resource, :schema]: ("{relation}", config[:schema])
+        def insert[:{relation}]: load_csv[config]
+        """
+        q = "\n".join(line.strip() for line in q.splitlines())
+        res = provider.exec_raw(model, provider.get_default_engine_name(), q, readonly=False, inputs=inputs)
+        assert_no_problems(res)
+
+    SAMPLE_LINES_COUNT = 100
+
+    @classmethod
+    def guess_syntax(cls, path: str) -> Syntax:
+        sample_lines = []
+        with open(path, "r") as file:
+            try:
+                for _ in range(cls.SAMPLE_LINES_COUNT):
+                    sample_lines.append(next(file))
+            except StopIteration:
+                pass
+
+        sample = "".join(sample_lines)
+        sniffer = csv.Sniffer()
+        dialect = sniffer.sniff(sample)
+
+        # Unused but available: quoting, doublequote, lineterminator, skipinitialspace,
+        return {
+            "delim": dialect.delimiter,
+            "quotechar": dialect.quotechar,
+            "escapechar": dialect.escapechar,
+            "header_row": 1 if sniffer.has_header(sample) else -1
+        }
+
+    @classmethod
+    def guess_schema(cls, path: str, syntax: Syntax) -> tuple[Schema, pandas.DataFrame]:
+        """Guess the schema by reading a chunk of the file."""
+        # @TODO: pick an appropriate magic number of rows to consider for type information.
+
+        pandas_syntax_args:dict[str, Any] = {
+            "delimiter": syntax.get("delim", None),
+            "quotechar": syntax.get("quotechar", None),
+            "escapechar": syntax.get("escapechar", None),
+
+            # @NOTE: For some reason the sniffer always wants \r\n even when its only \n, and pandas (like rel) requires single char line terminators
+            # "lineterminator": dialect.lineterminator,
+            # @NOTE: These can't be passed to load_csv so there's not much point helping pandas with them.
+            # "doublequote": dialect.doublequote,
+            # "quoting": dialect.quoting,
+            # "skipinitialspace": dialect.skipinitialspace,
+        }
+
+        chunk = pandas.read_csv(path, nrows=cls.SAMPLE_LINES_COUNT, **pandas_syntax_args)
+        chunk = chunk.rename(columns={k: k.strip() for k in chunk.columns})
+        inferred_schema = Schema()
+        for col in chunk.columns:
+            inferred_schema[col] = df_type(chunk[col].dtype)
+
+        return (inferred_schema, chunk)
+
+CSVLoader.register_for_extensions(".csv", ".tsv")

relationalai/loaders/loader.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+from contextlib import contextmanager
+import difflib
+import hashlib
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import cast, Type
+
+from relationalai.clients.types import ImportSource, ImportSourceFile
+from relationalai.clients.client import ResourcesBase
+from relationalai.dsl import Context, Graph, Instance, RelationRef, Vars, create_var
+from relationalai.loaders.types import LoadType, UnsupportedTypeError
+from relationalai.metamodel import Builtins
+from relationalai.rel_emitter import sanitize
+from relationalai.rel_utils import assert_no_problems, process_gnf_results
+from relationalai.std import rel
+
+#-------------------------------------------------------------------------------
+# Schema Mapping
+#-------------------------------------------------------------------------------
+
+TYPE_TO_REL_SCHEMA = {
+    Builtins.String: "string",
+    Builtins.Int: "int",
+    Builtins.Number: "float",
+    Builtins.Bool: "boolean",
+    Builtins.Decimal: "decimal" # @TODO: See if this works or requires manually specifying size
+}
+
+# @NOTE: This only works so long as the mapping is bidirectionally unambiguous
+REL_SCHEMA_TO_TYPE = {v: k for k, v in TYPE_TO_REL_SCHEMA.items()}
+
+def rel_schema_to_type(rel_type: str):
+    try:
+        return REL_SCHEMA_TO_TYPE[rel_type]
+    except KeyError:
+        raise Exception(f"Invalid schema field type '{rel_type}'. Available options are ({', '.join(REL_SCHEMA_TO_TYPE.keys())})")
+
+#-------------------------------------------------------------------------------
+# Hashing
+#-------------------------------------------------------------------------------
+
+def compute_file_hash(file_path:str|Path, chunk_size=16384):
+    """Digest the given file to a content-addressable hash."""
+    sha256 = hashlib.sha256()
+    with open(file_path, 'rb') as f:
+        while chunk := f.read(chunk_size):
+            sha256.update(chunk)
+    return sha256.hexdigest()
+
+def compute_str_hash(content: str):
+    """Digest the given string to a hash."""
+    return hashlib.sha256(content.encode()).hexdigest()
+
+#-------------------------------------------------------------------------------
+# Resource Proxy
+#-------------------------------------------------------------------------------
+
+class ResourceProxy:
+    """Represents a (optimistically-assumed to be present) resource relation, such as a CSV."""
+    def __init__(self, graph: Graph, name: str, id: Instance|None = None):
+        if id is None:
+            id = create_var()
+
+        self._graph = graph
+        self._rel = getattr(rel, sanitize(name)) # @FIXME: This isn't necessarily the same as the relation name, gota do something about that.
+        self._refs: dict[str, RelationRef] = {}
+        self._id = cast(Instance, id)
+
+    def __getattr__(self, attr: str):
+        ref = self._refs.get(attr, None)
+        if ref is not None:
+            return ref
+
+        rel = getattr(self._rel, attr)
+        attr_var = Vars(1)
+        rel(self._id, attr_var)
+        return attr_var
+
+# This could instead be a subclass of Context, it behaves exactly as if it was
+# but doing so happened to be larger and (seemingly?) more brittle due to all the
+# kwargs.
+@contextmanager
+def read_resource_context(graph: Graph, name: str, **kwargs):
+    ctx = Context(graph, **kwargs)
+    with ctx:
+        yield ResourceProxy(graph, name)
+
+
+#-------------------------------------------------------------------------------
+# Loader
+#-------------------------------------------------------------------------------
+
+class Loader(ABC):
+    """Base class for objects that handle the particulars of mapping an import source to a resource relation."""
+    type: str = "untyped"
+
+    @abstractmethod
+    def load(self, provider: ResourcesBase, model: str, source: ImportSource):
+        """Load a snapshot of the given source data into the given model."""
+        ...
+
+    @classmethod
+    def register(cls):
+        assert cls.type != "untyped", f"{cls.__name__} does not provide a type property."
+        assert cls.type not in Loader.type_to_loader or Loader.type_to_loader[cls.type] == cls, f"{cls.__name__} has the same type '{cls.type}' as another loader ({Loader.type_to_loader[cls.type].__name__})."
+        Loader.type_to_loader[cls.type] = cls
+
+    @classmethod
+    def register_for_extensions(cls, *exts: str):
+        cls.register()
+        for ext in exts:
+            assert ext not in Loader.ext_to_type or Loader.ext_to_type[ext] == cls.type, f"{cls.__name__} registered to handle '{ext}' but another loader ({Loader.type_to_loader[cls.type].__name__}) already does."
+            Loader.ext_to_type[ext] = cls.type
+
+    ext_to_type: dict[str, LoadType] = {}
+
+    @classmethod
+    def get_type_for(cls, source: ImportSourceFile):
+        try:
+            return Loader.ext_to_type[source.extension]
+        except KeyError:
+            raise UnsupportedTypeError(f"No loader type provided for extension: '{source.extension}'. You can manually specify the loader to use via the `type` kwarg", source.extension)
+
+    type_to_loader: dict[LoadType, 'Type[Loader]'] = {}
+    @classmethod
+    def get_loader_for_type(cls, type: LoadType):
+       try:
+           return Loader.type_to_loader[type]
+       except KeyError:
+           raise UnsupportedTypeError(f"No loader provided for type: '{type}'", type)
+
+#-------------------------------------------------------------------------------
+# Import Utilities
+#-------------------------------------------------------------------------------
+
+def import_file(provider: ResourcesBase, model: str, source: ImportSourceFile, type: LoadType = "auto", **metadata):
+    """Automatically load `source` into `model` using the most appropriate loader."""
+    if type == "auto":
+        type = Loader.get_type_for(source)
+
+    return Loader.get_loader_for_type(type)().load(provider, model, source, **metadata)
+
+def emit_delete_import(name: str) -> str:
+    """Utility fn for emitting the rel to delete the specified import relation and all its metadata."""
+    relation = sanitize(name)
+    return f"""
+    declare {relation}
+    def delete[:{relation}]: {relation}
+    def delete[:__resource, k]: ("{name}", __resource[k, "{name}"])
+    """
+
+#-------------------------------------------------------------------------------
+# Resource Management
+#-------------------------------------------------------------------------------
+
+available_resources:dict[str, dict] = {}
+
+def list_available_resources(provider: ResourcesBase, database: str, engine: str):
+    res = provider.exec_raw(database, engine, """
+    declare __resource
+    def output {{__resource}}
+    """)
+    assert_no_problems(res)
+
+    available_resources = process_gnf_results(res.results, "name")
+    return available_resources
+
+class MissingResourceError(Exception):
+    def __init__(self, name: str):
+        similar_names = difflib.get_close_matches(name, available_resources.keys(), n=3)
+        message = f"No available resource named '{name}'"
+        if similar_names:
+            message += ". Maybe you want one of these?"
+            for name in similar_names:
+                message += f"\n- {name}"
+        super().__init__(message)

relationalai/loaders/types.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+from typing import TypedDict
+from relationalai.metamodel import Type
+
+Schema = dict[str, Type]
+class Syntax(TypedDict, total=False):
+    # header: dict[int, str]|None
+    header_row: int|None
+    datarow: int|None
+    missingstrings: str|None # @TODO: Support specifying multiple homehow
+    delim: str|None
+    quotechar: str|None
+    escapechar: str|None
+    decimalchar: str|None
+    groupmark: str|None
+
+
+LoadType = str
+
+class UnsupportedTypeError(Exception):
+    def __init__(self, message: str, type: str|None = None):
+        super().__init__(message)
+        self.type = type