relationalai 0.13.0__py3-none-any.whl → 0.13.0.dev0__py3-none-any.whl

relationalai/loaders/csv.py

@@ -1,195 +0,0 @@
-from __future__ import annotations
-import csv
-from datetime import date, datetime
-from typing import Any, cast
-
-import numpy
-import pandas
-
-from relationalai.clients.types import ImportSource, ImportSourceFile
-from relationalai.clients.client import ResourcesBase
-from relationalai.dsl import Graph, RelationRef, build, tag
-from relationalai.loaders.loader import TYPE_TO_REL_SCHEMA, Loader, compute_file_hash, compute_str_hash
-from relationalai.rel_utils import Char, assert_no_problems, emit_nested_relation
-from relationalai.loaders.types import Schema, Syntax
-from relationalai.metamodel import ActionType, Builtins, Var
-from relationalai.rel_emitter import sanitize
-from relationalai.std import rel
-
-
-def df_type(df_type_name):
-    if df_type_name == "bool":
-        type = Builtins.Bool
-    elif df_type_name == "int64":
-        type = Builtins.Number
-    elif df_type_name == "float64":
-        type = Builtins.Number
-    else:
-        type = Builtins.String
-    return type
-
-class ExternalRow:
-    def __init__(self, data, columns):
-        self._data = data
-        self._columns = columns
-
-    def __getitem__(self, index):
-        if isinstance(index, str):
-            return getattr(self, index)
-        return self._data[index]
-
-    def __getattr__(self, name):
-        if name in self._columns:
-            return self._data[self._columns.index(name)]
-        return object.__getattribute__(self, name)
-
-# See https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
-# for keyword arg docs.
-def load_file(graph:Graph, csv_file, **kwargs):
-    df = pandas.read_csv(csv_file, **kwargs)
-
-    # create subqueries for each column that consist of a data object
-    # and a bind for the column relation, also create a data ref
-    # add a range for the id
-    # when a dataref is used, it should add a get for the column relation
-    # the rest just works?
-
-    index = getattr(rel, sanitize(csv_file))
-    tag(index, Builtins.Type)
-
-    with graph.scope():
-        index.add(rel.range(0, len(df), 1))
-
-    id = Var(type=Builtins.Number)
-    items = []
-    for col in df.columns:
-        sub = df[[col]]
-        col_type = df_type(df[col].dtype)
-        with graph.scope(dynamic=True):
-            # By setting Builtins.RawData on the task, we're telling the denester to not
-            # put a reference to the parent scope in this one
-            graph._stack.active()._task.parents.append(Builtins.RawData)
-            v1 = []
-            v2 = []
-            for (i, v) in sub.itertuples():
-                if v is not None and v != "" and v is not False and (isinstance(v,str) or isinstance(v, date) or isinstance(v, datetime) or not numpy.isnan(v)):
-                    v1.append(i)
-                    v2.append(v)
-
-            v1_var = Var(type=Builtins.Number)
-            v2_var = Var(name=col, type=col_type)
-            graph._action(
-                build.relation_action(ActionType.Get, Builtins.RawData, [Var(value=v1), Var(value=v2), v1_var, v2_var])
-            )
-            temp = build.relation(sanitize(csv_file) + "_" + col, 2)
-            temp.parents.append(Builtins.Property)
-            graph._action(
-                build.relation_action(ActionType.Bind, temp, [v1_var, v2_var])
-            )
-            usage_var = Var(name=col, type=col_type)
-            items.append(RelationRef(graph, temp, [id, usage_var]))
-    graph._action(
-        build.relation_action(ActionType.Get, index._rel, [id])
-    )
-    return ExternalRow(items, [c for c in df.columns])
-
-class CSVLoader(Loader):
-    """Load CSV files and URLs in RAI DB using `load_csv`."""
-    type = "csv"
-
-    def load(self, provider: ResourcesBase, model: str, source: ImportSource, *, schema: Schema|None = None, syntax: Syntax|None = None, integration: dict|None = None):
-        from railib import api
-        assert isinstance(source, ImportSourceFile), "CSV Loader can only load from files and URLs, not {type(source).__name__}"
-        schema = schema or {}
-        syntax = syntax or {}
-
-        id = compute_str_hash(source.raw_path)
-        prefix = f"""def config[:path]: raw"{source.raw_path}" """
-        inputs = None
-
-        if not source.is_url():
-            with open(source.raw_path, "r") as csv_file:
-                data = csv_file.read()
-
-            id = compute_file_hash(source.raw_path)
-            prefix = "def config[:data]: data"
-            inputs = {"data": data}
-
-        relation = sanitize(source.name)
-
-        for key in ["delim", "quotechar", "escapechar"]:
-            if syntax.get(key, None) and not isinstance(syntax[key], Char):
-                syntax[key] = Char(syntax[key])
-
-        q = f"""
-        declare {relation}
-        def delete[:{relation}]: {relation}
-        def delete[:__resource, k, "{relation}"]: __resource[k, "{relation}"]
-
-        {prefix}
-        {emit_nested_relation("config[:syntax, ", cast(dict, syntax), api._syntax_options)}
-        {emit_nested_relation("config[:schema, ", {col: TYPE_TO_REL_SCHEMA.get(type, "string") for col, type in schema.items()})}
-        {emit_nested_relation("config[:integration, ", integration) if integration else ""}
-        {emit_nested_relation("insert[:__resource, ", {
-        "id": (relation, id),
-        "type": (relation, self.type),
-        "name": (relation, source.name)
-        })}
-        def insert[:__resource, :schema]: ("{relation}", config[:schema])
-        def insert[:{relation}]: load_csv[config]
-        """
-        q = "\n".join(line.strip() for line in q.splitlines())
-        res = provider.exec_raw(model, provider.get_default_engine_name(), q, readonly=False, inputs=inputs)
-        assert_no_problems(res)
-
-    SAMPLE_LINES_COUNT = 100
-
-    @classmethod
-    def guess_syntax(cls, path: str) -> Syntax:
-        sample_lines = []
-        with open(path, "r") as file:
-            try:
-                for _ in range(cls.SAMPLE_LINES_COUNT):
-                    sample_lines.append(next(file))
-            except StopIteration:
-                pass
-
-        sample = "".join(sample_lines)
-        sniffer = csv.Sniffer()
-        dialect = sniffer.sniff(sample)
-
-        # Unused but available: quoting, doublequote, lineterminator, skipinitialspace,
-        return {
-            "delim": dialect.delimiter,
-            "quotechar": dialect.quotechar,
-            "escapechar": dialect.escapechar,
-            "header_row": 1 if sniffer.has_header(sample) else -1
-        }
-
-    @classmethod
-    def guess_schema(cls, path: str, syntax: Syntax) -> tuple[Schema, pandas.DataFrame]:
-        """Guess the schema by reading a chunk of the file."""
-        # @TODO: pick an appropriate magic number of rows to consider for type information.
-
-        pandas_syntax_args:dict[str, Any] = {
-            "delimiter": syntax.get("delim", None),
-            "quotechar": syntax.get("quotechar", None),
-            "escapechar": syntax.get("escapechar", None),
-
-            # @NOTE: For some reason the sniffer always wants \r\n even when its only \n, and pandas (like rel) requires single char line terminators
-            # "lineterminator": dialect.lineterminator,
-            # @NOTE: These can't be passed to load_csv so there's not much point helping pandas with them.
-            # "doublequote": dialect.doublequote,
-            # "quoting": dialect.quoting,
-            # "skipinitialspace": dialect.skipinitialspace,
-        }
-
-        chunk = pandas.read_csv(path, nrows=cls.SAMPLE_LINES_COUNT, **pandas_syntax_args)
-        chunk = chunk.rename(columns={k: k.strip() for k in chunk.columns})
-        inferred_schema = Schema()
-        for col in chunk.columns:
-            inferred_schema[col] = df_type(chunk[col].dtype)
-
-        return (inferred_schema, chunk)
-
-CSVLoader.register_for_extensions(".csv", ".tsv")

relationalai/loaders/loader.py

@@ -1,177 +0,0 @@
-from __future__ import annotations
-from contextlib import contextmanager
-import difflib
-import hashlib
-from abc import ABC, abstractmethod
-from pathlib import Path
-from typing import cast, Type
-
-from relationalai.clients.types import ImportSource, ImportSourceFile
-from relationalai.clients.client import ResourcesBase
-from relationalai.dsl import Context, Graph, Instance, RelationRef, Vars, create_var
-from relationalai.loaders.types import LoadType, UnsupportedTypeError
-from relationalai.metamodel import Builtins
-from relationalai.rel_emitter import sanitize
-from relationalai.rel_utils import assert_no_problems, process_gnf_results
-from relationalai.std import rel
-
-#-------------------------------------------------------------------------------
-# Schema Mapping
-#-------------------------------------------------------------------------------
-
-TYPE_TO_REL_SCHEMA = {
-    Builtins.String: "string",
-    Builtins.Int: "int",
-    Builtins.Number: "float",
-    Builtins.Bool: "boolean",
-    Builtins.Decimal: "decimal" # @TODO: See if this works or requires manually specifying size
-}
-
-# @NOTE: This only works so long as the mapping is bidirectionally unambiguous
-REL_SCHEMA_TO_TYPE = {v: k for k, v in TYPE_TO_REL_SCHEMA.items()}
-
-def rel_schema_to_type(rel_type: str):
-    try:
-        return REL_SCHEMA_TO_TYPE[rel_type]
-    except KeyError:
-        raise Exception(f"Invalid schema field type '{rel_type}'. Available options are ({', '.join(REL_SCHEMA_TO_TYPE.keys())})")
-
-#-------------------------------------------------------------------------------
-# Hashing
-#-------------------------------------------------------------------------------
-
-def compute_file_hash(file_path:str|Path, chunk_size=16384):
-    """Digest the given file to a content-addressable hash."""
-    sha256 = hashlib.sha256()
-    with open(file_path, 'rb') as f:
-        while chunk := f.read(chunk_size):
-            sha256.update(chunk)
-    return sha256.hexdigest()
-
-def compute_str_hash(content: str):
-    """Digest the given string to a hash."""
-    return hashlib.sha256(content.encode()).hexdigest()
-
-#-------------------------------------------------------------------------------
-# Resource Proxy
-#-------------------------------------------------------------------------------
-
-class ResourceProxy:
-    """Represents a (optimistically-assumed to be present) resource relation, such as a CSV."""
-    def __init__(self, graph: Graph, name: str, id: Instance|None = None):
-        if id is None:
-            id = create_var()
-
-        self._graph = graph
-        self._rel = getattr(rel, sanitize(name)) # @FIXME: This isn't necessarily the same as the relation name, gota do something about that.
-        self._refs: dict[str, RelationRef] = {}
-        self._id = cast(Instance, id)
-
-    def __getattr__(self, attr: str):
-        ref = self._refs.get(attr, None)
-        if ref is not None:
-            return ref
-
-        rel = getattr(self._rel, attr)
-        attr_var = Vars(1)
-        rel(self._id, attr_var)
-        return attr_var
-
-# This could instead be a subclass of Context, it behaves exactly as if it was
-# but doing so happened to be larger and (seemingly?) more brittle due to all the
-# kwargs.
-@contextmanager
-def read_resource_context(graph: Graph, name: str, **kwargs):
-    ctx = Context(graph, **kwargs)
-    with ctx:
-        yield ResourceProxy(graph, name)
-
-
-#-------------------------------------------------------------------------------
-# Loader
-#-------------------------------------------------------------------------------
-
-class Loader(ABC):
-    """Base class for objects that handle the particulars of mapping an import source to a resource relation."""
-    type: str = "untyped"
-
-    @abstractmethod
-    def load(self, provider: ResourcesBase, model: str, source: ImportSource):
-        """Load a snapshot of the given source data into the given model."""
-        ...
-
-    @classmethod
-    def register(cls):
-        assert cls.type != "untyped", f"{cls.__name__} does not provide a type property."
-        assert cls.type not in Loader.type_to_loader or Loader.type_to_loader[cls.type] == cls, f"{cls.__name__} has the same type '{cls.type}' as another loader ({Loader.type_to_loader[cls.type].__name__})."
-        Loader.type_to_loader[cls.type] = cls
-
-    @classmethod
-    def register_for_extensions(cls, *exts: str):
-        cls.register()
-        for ext in exts:
-            assert ext not in Loader.ext_to_type or Loader.ext_to_type[ext] == cls.type, f"{cls.__name__} registered to handle '{ext}' but another loader ({Loader.type_to_loader[cls.type].__name__}) already does."
-            Loader.ext_to_type[ext] = cls.type
-
-    ext_to_type: dict[str, LoadType] = {}
-
-    @classmethod
-    def get_type_for(cls, source: ImportSourceFile):
-        try:
-            return Loader.ext_to_type[source.extension]
-        except KeyError:
-            raise UnsupportedTypeError(f"No loader type provided for extension: '{source.extension}'. You can manually specify the loader to use via the `type` kwarg", source.extension)
-
-    type_to_loader: dict[LoadType, 'Type[Loader]'] = {}
-    @classmethod
-    def get_loader_for_type(cls, type: LoadType):
-       try:
-           return Loader.type_to_loader[type]
-       except KeyError:
-           raise UnsupportedTypeError(f"No loader provided for type: '{type}'", type)
-
-#-------------------------------------------------------------------------------
-# Import Utilities
-#-------------------------------------------------------------------------------
-
-def import_file(provider: ResourcesBase, model: str, source: ImportSourceFile, type: LoadType = "auto", **metadata):
-    """Automatically load `source` into `model` using the most appropriate loader."""
-    if type == "auto":
-        type = Loader.get_type_for(source)
-
-    return Loader.get_loader_for_type(type)().load(provider, model, source, **metadata)
-
-def emit_delete_import(name: str) -> str:
-    """Utility fn for emitting the rel to delete the specified import relation and all its metadata."""
-    relation = sanitize(name)
-    return f"""
-    declare {relation}
-    def delete[:{relation}]: {relation}
-    def delete[:__resource, k]: ("{name}", __resource[k, "{name}"])
-    """
-
-#-------------------------------------------------------------------------------
-# Resource Management
-#-------------------------------------------------------------------------------
-
-available_resources:dict[str, dict] = {}
-
-def list_available_resources(provider: ResourcesBase, database: str, engine: str):
-    res = provider.exec_raw(database, engine, """
-    declare __resource
-    def output {{__resource}}
-    """)
-    assert_no_problems(res)
-
-    available_resources = process_gnf_results(res.results, "name")
-    return available_resources
-
-class MissingResourceError(Exception):
-    def __init__(self, name: str):
-        similar_names = difflib.get_close_matches(name, available_resources.keys(), n=3)
-        message = f"No available resource named '{name}'"
-        if similar_names:
-            message += ". Maybe you want one of these?"
-            for name in similar_names:
-                message += f"\n- {name}"
-        super().__init__(message)

relationalai/loaders/types.py

@@ -1,23 +0,0 @@
-from __future__ import annotations
-from typing import TypedDict
-from relationalai.metamodel import Type
-
-Schema = dict[str, Type]
-class Syntax(TypedDict, total=False):
-    # header: dict[int, str]|None
-    header_row: int|None
-    datarow: int|None
-    missingstrings: str|None # @TODO: Support specifying multiple homehow
-    delim: str|None
-    quotechar: str|None
-    escapechar: str|None
-    decimalchar: str|None
-    groupmark: str|None
-
-
-LoadType = str
-
-class UnsupportedTypeError(Exception):
-    def __init__(self, message: str, type: str|None = None):
-        super().__init__(message)
-        self.type = type