uptimely-core 0.2.0.dev1__tar.gz → 0.4.0__tar.gz

{uptimely_core-0.2.0.dev1 → uptimely_core-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: uptimely-core
-Version: 0.2.0.dev1
+Version: 0.4.0
 Summary: Manage data-driven numerical analytics with complex dependencies at scale
 License-Expression: MIT
 License-File: LICENSE

{uptimely_core-0.2.0.dev1 → uptimely_core-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "uptimely-core"
-version = "0.2.0.dev1"
+version = "0.4.0"
 description = "Manage data-driven numerical analytics with complex dependencies at scale"
 authors = [{ name = "Mikael Ahonen", email = "mikael.ahonen@uptimely.ai" }]
 license = "MIT"

{uptimely_core-0.2.0.dev1 → uptimely_core-0.4.0}/src/uptimely/__init__.py

@@ -3,36 +3,28 @@
 from .compiler import CallableGraph, CallableNode, Compiler, ExecutionPlan
 from .engine import Engine
 from .spec import (
-    Call,
     Dimension,
     Entity,
     FeatureGraph,
-    Function,
-    PreparedSpec,
     PythonSpec,
     Spec,
     SubGraph,
     from_python,
     load_spec,
-    prepare_spec,
     write_document,
 )
 
 __all__ = [
     "Spec",
-    "Call",
     "CallableGraph",
     "CallableNode",
     "Compiler",
     "Dimension",
     "Entity",
     "ExecutionPlan",
-    "Function",
     "PythonSpec",
-    "PreparedSpec",
     "from_python",
     "load_spec",
-    "prepare_spec",
     "write_document",
     "SubGraph",
     "FeatureGraph",

{uptimely_core-0.2.0.dev1 → uptimely_core-0.4.0}/src/uptimely/cli/common.py

@@ -4,7 +4,7 @@ import argparse
 import pathlib
 
 from uptimely import spec
-from uptimely.project.build import prepare_project_root
+from uptimely.project import imports
 
 
 def configure_project_arguments(parser: argparse.ArgumentParser) -> None:
@@ -34,5 +34,5 @@ def project_path(
 
 def load_spec(path: pathlib.Path, project_root: pathlib.Path) -> spec.Spec:
     """Load a spec document with project functions on the import path."""
-    prepare_project_root(project_root)
+    imports.prepare_project_root(project_root)
     return spec.load_spec(path)

{uptimely_core-0.2.0.dev1 → uptimely_core-0.4.0}/src/uptimely/cli/engine.py

@@ -9,6 +9,7 @@ import pydantic
 from uptimely import engine as engine_module
 from uptimely import project
 from uptimely.cli import common, plan
+from uptimely.project import imports
 
 
 class EngineSettings(pydantic.BaseModel):
@@ -47,7 +48,7 @@ def run(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
     root = args.project_root or pathlib.Path.cwd()
     config = common.project_path(args.config, args.project_root, project.ENGINE_CONFIG_FILE)
     try:
-        common.prepare_project_root(root)
+        imports.prepare_project_root(root)
         with project.read_build_bundle(root) as bundle:
             settings = EngineSettings.model_validate_json(config.read_text(encoding="utf-8"))
             execution_plan = bundle.execution_plan

{uptimely_core-0.2.0.dev1 → uptimely_core-0.4.0}/src/uptimely/cli/project.py

@@ -15,12 +15,12 @@ def configure_parser(parser: argparse.ArgumentParser) -> None:
     )
     source = build.add_mutually_exclusive_group()
     source.add_argument(
-        "--entrypoint",
+        "--python-spec",
         metavar="MODULE:ATTRIBUTE",
         help="Importable module and attribute containing a PythonSpec instance",
     )
     source.add_argument(
-        "--spec",
+        "--json-spec",
         type=pathlib.Path,
         help="Source spec JSON path (overrides standard source discovery)",
     )
@@ -43,8 +43,8 @@ def run(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
     try:
         bundle = project_build.build_project(
             root,
-            entrypoint=args.entrypoint,
-            source=args.spec,
+            python_spec=args.python_spec,
+            json_spec=args.json_spec,
             features=args.features,
         )
     except (

{uptimely_core-0.2.0.dev1 → uptimely_core-0.4.0}/src/uptimely/compiler/callable_node.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import dataclasses
 
-from uptimely import render, row_context, spec
+from uptimely import render, row_context
 
 
 @dataclasses.dataclass(frozen=True)
@@ -17,7 +17,7 @@ class CallableNode:
     mapping_id: str | None = None
     output_features: tuple[str, ...] = ()
     args: dict[str, object] = dataclasses.field(default_factory=dict, compare=False, hash=False)
-    callable: spec.Call | None = dataclasses.field(default=None, compare=False, hash=False)
+    input_features: tuple[str, ...] = ()
     row_context: row_context.RowContext | None = dataclasses.field(
         default=None, compare=False, hash=False
     )
@@ -49,6 +49,6 @@ class CallableNode:
             feature=self.feature,
             output_features=list(self.output_features),
             depends_on_features=(
-                self.callable.depends_on_features if self.callable is not None else None
+                list(self.input_features) if self.operation == "vectorize" else None
             ),
         )

{uptimely_core-0.2.0.dev1 → uptimely_core-0.4.0}/src/uptimely/compiler/compiler.py

@@ -167,7 +167,10 @@ class Compiler:
             if function is None or not function.entrypoint:
                 continue
             try:
-                signature = entrypoint.read_signature(function.entrypoint, pathlib.Path.cwd())
+                function_entrypoint = entrypoint.Entrypoint(function.entrypoint, pathlib.Path.cwd())
+                if not function.validate_signature:
+                    continue
+                signature = function_entrypoint.function.signature
             except (ImportError, AttributeError, ValueError, TypeError) as error:
                 raise ValueError(
                     f"Could not read signature of function {function.id} "
@@ -183,6 +186,8 @@ class Compiler:
     ) -> None:
         """Require the Python return annotation to satisfy the function contract."""
         returns = function.returns
+        if returns == "any":
+            return
         if annotation is None:
             raise ValueError(
                 f"Function {function.id} must annotate its return as "

{uptimely_core-0.2.0.dev1 → uptimely_core-0.4.0}/src/uptimely/compiler/execution_graph.py

@@ -6,7 +6,7 @@ import graphable as graphable_
 from graphable.views import texttree
 
 from uptimely import contracts, spec
-from uptimely.spec import api as spec_api
+from uptimely.spec import validation
 
 from ..spec import references as bindings
 from . import callable_node as callable_node_module
@@ -25,16 +25,12 @@ class CallableGraph:
             spec: The spec document containing function declarations and invocations.
         """
         self.spec = spec
-        prepared = spec_api.prepare_spec(spec)
+        validation.validate_spec(spec)
         self._datasets = {
             (entity_id, dataset.id): dataset
             for entity_id, entity in spec.entities.items()
             for dataset in entity.datasets
         }
-        self.calls = prepared.calls
-        self.functions = [function for function in prepared.functions if function.role == "spec"]
-        # Index all callables by (function_id, entity, feature) to resolve dependencies
-        self._callable_index: dict[tuple[str, str, str], callable_node_module.CallableNode] = {}
         # Map feature names to the callables that produce them
         self._feature_producers: dict[str, callable_node_module.CallableNode] = {}
         self._dataset_sources: dict[tuple[str, str], list[callable_node_module.CallableNode]] = {}
@@ -53,14 +49,6 @@ class CallableGraph:
                     primary_feature = (
                         mapping.output_features[0] if mapping.output_features else None
                     )
-                    call = next(
-                        (
-                            call
-                            for call in self.calls
-                            if call.entity == entity_id and call.feature in mapping.output_features
-                        ),
-                        None,
-                    )
                     node = callable_node_module.CallableNode(
                         function_id=mapping.binding.function,
                         entity=entity_id,
@@ -70,20 +58,18 @@ class CallableGraph:
                         mapping_id=mapping.id,
                         output_features=tuple(mapping.output_features),
                         args=mapping.binding.args,
-                        callable=call,
+                        input_features=tuple(mapping.input_features),
                         row_context=mapping.row_context,
                     )
                     nodes[node.node_id] = graphable_.graphable.Graphable(node)
                     if operation == "write":
                         self._dataset_sinks.setdefault(dataset_key, []).append(node)
-                    elif operation != "vectorize":
+                    elif operation in contracts.FRAGMENT_OPERATIONS:
                         self._dataset_sources.setdefault(dataset_key, []).append(node)
                     for feature in mapping.output_features:
                         if feature in self.spec.dimensions:
                             continue
                         self._feature_producers[feature] = node
-                        if operation == "vectorize":
-                            self._callable_index[(node.function_id, entity_id, feature)] = node
         graph = graphable_.graph.Graph(set(nodes.values()), discover=False)
         # Source transforms follow reads in their dataset.
         for fragments in self._dataset_sources.values():
@@ -129,7 +115,10 @@ class CallableGraph:
                     graph.add_edge(nodes[producer.node_id], graph_node)
 
         # Vectorize mappings consume the assembled dataset frame.
-        for callable_node in set(self._callable_index.values()):
+        calculation_nodes = [
+            node.reference for node in nodes.values() if node.reference.operation == "vectorize"
+        ]
+        for callable_node in calculation_nodes:
             consumer_node = nodes[callable_node.node_id]
             dataset_key = (callable_node.entity, callable_node.dataset_id)
             for source in self._dataset_sources.get(dataset_key, []):
@@ -137,13 +126,9 @@ class CallableGraph:
 
         for dataset_key, sinks in self._dataset_sinks.items():
             sources = self._dataset_sources.get(dataset_key, [])
-            calculations = list(
-                dict.fromkeys(
-                    node
-                    for node in self._callable_index.values()
-                    if (node.entity, node.dataset_id) == dataset_key
-                )
-            )
+            calculations = [
+                node for node in calculation_nodes if (node.entity, node.dataset_id) == dataset_key
+            ]
             for sink in sinks:
                 sink_node = nodes[sink.node_id]
                 for source in sources:
@@ -220,8 +205,6 @@ class CallableGraph:
         subset = self.__class__.__new__(self.__class__)
         subset.spec = self.spec
         subset._datasets = self._datasets
-        subset.functions = self.functions
-        subset._callable_index = self._callable_index
         subset._feature_producers = self._feature_producers
         subset._dataset_sources = self._dataset_sources
         subset._dataset_sinks = self._dataset_sinks

{uptimely_core-0.2.0.dev1 → uptimely_core-0.4.0}/src/uptimely/contracts.py

@@ -5,13 +5,16 @@ import typing
 FUNCTION_OPERATIONS = (
     "read",
     "write",
+    "escape",
     "vectorize",
     "collapse",
     "expand",
     "conform",
 )
-FunctionOperation = typing.Literal["read", "write", "vectorize", "collapse", "expand", "conform"]
-FunctionReturns = typing.Literal["dataframe", "none"]
+FunctionOperation = typing.Literal[
+    "read", "write", "escape", "vectorize", "collapse", "expand", "conform"
+]
+FunctionReturns = typing.Literal["dataframe", "none", "any"]
 FRAGMENT_OPERATIONS: tuple[FunctionOperation, ...] = ("read", "conform", "expand", "collapse")
 
 FEATURE_PREFIX = "$features."
@@ -22,17 +25,19 @@ ID_SELECTOR = ".id"
 DATAFRAME_SELECTOR = ".dataframe"
 STRUCTURAL_PREFIXES = (FEATURE_PREFIX, DATASET_PREFIX, DIMENSION_PREFIX)
 
-DATAFRAME_INPUT_COUNTS: dict[FunctionOperation, int] = {
+DATAFRAME_INPUT_COUNTS: dict[FunctionOperation, int | None] = {
     "read": 0,
     "collapse": 1,
     "conform": 2,
     "expand": 2,
     "vectorize": 1,
     "write": 1,
+    "escape": None,
 }
 OPERATION_RETURN_KINDS: dict[FunctionOperation, FunctionReturns] = {
     "read": "dataframe",
     "write": "none",
+    "escape": "any",
     "vectorize": "dataframe",
     "collapse": "dataframe",
     "expand": "dataframe",

{uptimely_core-0.2.0.dev1 → uptimely_core-0.4.0}/src/uptimely/engine/backend.py

@@ -155,4 +155,4 @@ class EntrypointLoader:
     """Default callable loader using the package entrypoint implementation."""
 
     def load(self, entrypoint: str) -> abc.Callable[..., typing.Any]:
-        return entrypoint_module.load_callable(entrypoint, pathlib.Path.cwd())
+        return entrypoint_module.Entrypoint(entrypoint, pathlib.Path.cwd()).function.load()

{uptimely_core-0.2.0.dev1 → uptimely_core-0.4.0}/src/uptimely/engine/engine.py

@@ -78,13 +78,18 @@ class Engine:
                 started = time.perf_counter()
                 logger.debug("Running %s callable %s", callable.operation, callable.id)
                 try:
-                    dataframe.validate_inputs(callable)
-                    if contracts.OPERATION_RETURN_KINDS.get(callable.operation) == "dataframe":
-                        self._execute_dataframe_mapping(callable, state, bindings, dataframe)
-                    elif contracts.OPERATION_RETURN_KINDS.get(callable.operation) == "none":
-                        self._execute_sink(callable, state, bindings)
+                    if callable.operation == "escape":
+                        self._execute_escape(callable, bindings)
                     else:
-                        raise ValueError(f"Unsupported callable operation: {callable.operation}")
+                        dataframe.validate_inputs(callable)
+                        if contracts.OPERATION_RETURN_KINDS.get(callable.operation) == "dataframe":
+                            self._execute_dataframe_mapping(callable, state, bindings, dataframe)
+                        elif contracts.OPERATION_RETURN_KINDS.get(callable.operation) == "none":
+                            self._execute_sink(callable, state, bindings)
+                        else:
+                            raise ValueError(
+                                f"Unsupported callable operation: {callable.operation}"
+                            )
                 except Exception as error:
                     statuses[callable.id] = "failed"
                     failures[callable.id] = str(error)
@@ -133,6 +138,15 @@ class Engine:
                     fragment, [*dimensions, output]
                 )
 
+    def _execute_escape(
+        self,
+        callable: plan_module.Callable,
+        bindings: bindings_module.RuntimeBindings,
+    ) -> None:
+        """Invoke an uncontrolled mapping and discard its return value."""
+        kwargs = bindings.resolve_args(callable)
+        self.callable_loader.load(callable.entrypoint)(**kwargs)
+
     def _execute_sink(
         self,
         callable: plan_module.Callable,

uptimely_core-0.4.0/src/uptimely/entrypoint.py

@@ -0,0 +1,210 @@
+"""Load Python function entrypoints and read their annotated signatures."""
+
+from __future__ import annotations
+
+import dataclasses
+import hashlib
+import importlib
+import inspect
+import pathlib
+import sys
+import types
+import typing
+from importlib import abc, util
+from urllib import parse, request
+
+from uptimely import storage as storage_module
+
+REMOTE_SCHEMES = {"https", "s3"}
+REMOTE_READ_TIMEOUT = 30
+
+
+class _RemoteModuleLoader(abc.SourceLoader):
+    """Load one Python source file from a supported remote location."""
+
+    def __init__(self, location: str) -> None:
+        self.location = location
+
+    def get_filename(self, _fullname: str) -> str:
+        return self.location
+
+    def get_data(self, _path: str) -> bytes:
+        if parse.urlsplit(self.location).scheme == "s3":
+            return storage_module.Storage.read(self.location).encode("utf-8")
+        with request.urlopen(self.location, timeout=REMOTE_READ_TIMEOUT) as response:
+            if parse.urlsplit(response.geturl()).scheme != "https":
+                raise ValueError("HTTPS function references must not redirect to another scheme")
+            return response.read()
+
+
+class Module:
+    """Represent and load a Python module from a file, import path, or URL."""
+
+    def __init__(self, location: str, root: pathlib.Path | None = None) -> None:
+        self.location = location
+        self.root = root
+
+    def load(self) -> types.ModuleType:
+        """Import the represented Python module."""
+        if self.root is not None and str(self.root) not in sys.path:
+            sys.path.insert(0, str(self.root))
+
+        scheme = parse.urlsplit(self.location).scheme
+        if scheme in REMOTE_SCHEMES:
+            return self.load_external()
+
+        path = pathlib.Path(self.location)
+        if path.is_absolute() or self.location.startswith(("./", "../")):
+            path = path if path.is_absolute() else (self.root or pathlib.Path.cwd()) / path
+            return self._load_module(str(path.resolve()))
+
+        module_path = self.location[:-3] if self.location.endswith(".py") else self.location
+        return importlib.import_module(module_path.replace("/", "."))
+
+    def load_external(self) -> types.ModuleType:
+        """Load the represented source file from HTTPS or S3."""
+        location = parse.urlsplit(self.location)
+        if location.scheme not in REMOTE_SCHEMES:
+            raise ValueError(f"Unsupported external module scheme: {location.scheme}")
+        if not location.netloc or not location.path.strip("/"):
+            raise ValueError(f"Function reference must point to a source file: {self.location}")
+        return self._load_module(self.location, _RemoteModuleLoader(self.location))
+
+    @staticmethod
+    def _load_module(location: str, loader: abc.Loader | None = None) -> types.ModuleType:
+        """Load a module once, removing failed imports from the cache."""
+        module_name = f"_uptimely_function_{hashlib.sha256(location.encode()).hexdigest()}"
+        if module_name in sys.modules:
+            return sys.modules[module_name]
+        module_spec = (
+            util.spec_from_loader(module_name, loader, origin=location)
+            if loader is not None
+            else util.spec_from_file_location(module_name, location)
+        )
+        if module_spec is None or module_spec.loader is None:
+            raise ImportError(f"Cannot load Python module from {location}")
+        module = util.module_from_spec(module_spec)
+        module.__file__ = location
+        sys.modules[module_name] = module
+        try:
+            module_spec.loader.exec_module(module)
+        except BaseException:
+            del sys.modules[module_name]
+            raise
+        return module
+
+
+class TypeAnnotation:
+    """Format Python type annotations for display."""
+
+    @staticmethod
+    def name(annotation: object) -> str:
+        """Return a readable name for a type annotation."""
+        if annotation is None or annotation is type(None):
+            return "None"
+
+        origin = typing.get_origin(annotation)
+        if origin in (typing.Union, types.UnionType):
+            return " | ".join(TypeAnnotation.name(arg) for arg in typing.get_args(annotation))
+
+        if origin is not None:
+            arguments = ", ".join(TypeAnnotation.name(arg) for arg in typing.get_args(annotation))
+            return (
+                f"{TypeAnnotation.name(origin)}[{arguments}]"
+                if arguments
+                else TypeAnnotation.name(origin)
+            )
+
+        return getattr(annotation, "__name__", str(annotation))
+
+
+@dataclasses.dataclass
+class Signature:
+    """Annotated input and output types of a Python function."""
+
+    args: dict[str, str] = dataclasses.field(default_factory=dict)
+    returns: str | None = None
+
+    @classmethod
+    def read(cls, function: Function) -> Signature:
+        """Read the annotated argument and return types of a function."""
+        signature = inspect.signature(function.load(), eval_str=True)
+        return cls(
+            args={
+                name: TypeAnnotation.name(parameter.annotation)
+                for name, parameter in signature.parameters.items()
+                if parameter.annotation is not inspect.Parameter.empty
+            },
+            returns=(
+                TypeAnnotation.name(signature.return_annotation)
+                if signature.return_annotation is not inspect.Signature.empty
+                else None
+            ),
+        )
+
+
+class Function:
+    """Represent a named callable in a Python module."""
+
+    def __init__(self, module: Module, name: str) -> None:
+        self.module = module
+        self.name = name
+
+    @property
+    def signature(self) -> Signature:
+        """Return the annotated signature of this function."""
+        return Signature.read(self)
+
+    def load(self) -> typing.Callable[..., typing.Any]:
+        """Import the callable from its module."""
+        module = self.module.load()
+        function = getattr(module, self.name)
+        if not callable(function):
+            raise TypeError(f"{self.name} in module {self.module.location} is not callable")
+        return function
+
+
+class Entrypoint:
+    """Parse and expose the module and function addressed by an entrypoint."""
+
+    def __init__(self, entrypoint: str, root: pathlib.Path | None = None) -> None:
+        if ":" not in entrypoint:
+            raise ValueError(
+                f"Invalid entrypoint: {entrypoint}. "
+                "Expected 'module.path:function' or 'path/to/module.py:function'"
+            )
+
+        module_path, function_name = entrypoint.rsplit(":", 1)
+        scheme = parse.urlsplit(module_path).scheme
+        if not module_path.strip() or not function_name.isidentifier():
+            raise ValueError(f"Invalid function entrypoint: {entrypoint}")
+        if (scheme and scheme not in REMOTE_SCHEMES) or module_path.startswith("git@"):
+            raise ValueError(
+                f"Unsupported function reference scheme: {module_path}. "
+                "Use a Python module, local file, https:// URL, or s3:// URI."
+            )
+
+        self.root = root
+        self._module_path = module_path
+        self._function_name = function_name
+
+    @property
+    def module(self) -> Module:
+        """Return the module component of this entrypoint."""
+        return Module(self._module_path, self.root)
+
+    @property
+    def function(self) -> Function:
+        """Return the function component of this entrypoint."""
+        return Function(self.module, self._function_name)
+
+
+__all__ = [
+    "Entrypoint",
+    "Function",
+    "Module",
+    "REMOTE_READ_TIMEOUT",
+    "REMOTE_SCHEMES",
+    "Signature",
+    "TypeAnnotation",
+]

{uptimely_core-0.2.0.dev1 → uptimely_core-0.4.0}/src/uptimely/explorer/graph_render.py

@@ -23,6 +23,7 @@ class GraphStyle:
     function_colors: dict[str, str] = dataclasses.field(
         default_factory=lambda: {
             "read": "#4c78a8",
+            "escape": "#a78bfa",
             "vectorize": "#f58518",
             "collapse": "#e45756",
             "conform": "#72b7b2",

{uptimely_core-0.2.0.dev1 → uptimely_core-0.4.0}/src/uptimely/explorer/lineage.py

@@ -18,6 +18,7 @@ class DataLineageColumn:
     role: str
     color: str
     producer: int | None
+    external: bool = False
 
 
 @dataclasses.dataclass(frozen=True)
@@ -89,8 +90,14 @@ def _build_stages(
                 for name in dimensions
             ]
             for role, features in (
-                ("Input", callable.input_features),
-                ("Output", callable.output_features),
+                (
+                    "Declared input" if callable.operation == "escape" else "Input",
+                    callable.input_features,
+                ),
+                (
+                    "Declared external output" if callable.operation == "escape" else "Output",
+                    callable.output_features,
+                ),
             ):
                 for feature in features:
                     if feature in dimensions:
@@ -107,6 +114,7 @@ def _build_stages(
                             if feature in spec_definition.dimensions
                             else "#cbd5e1",
                             numbers[producer.id] if producer else None,
+                            producer is not None and producer.operation == "escape",
                         )
                     )
             dataset = datasets.get(callable.dataset_id)

{uptimely_core-0.2.0.dev1 → uptimely_core-0.4.0}/src/uptimely/explorer/plan_render.py

@@ -75,11 +75,14 @@ def _render_step(
         "dataset": callable.dataset_id,
         "feature": (
             ", ".join(callable.output_features)
-            if callable.operation == "collapse"
+            if callable.operation in {"collapse", "escape"}
             else callable.output
             if callable.operation == "vectorize"
             else None
         ),
+        "input_features": callable.input_features,
+        "output_features": callable.output_features,
+        "uncontrolled": callable.operation == "escape",
         "row_context": callable.row_context.to_dict() if callable.row_context else None,
         "depends_on_callables": [
             {
@@ -155,7 +158,15 @@ def _create_website_pages(
             "function_id": callable.function_id,
             "entity": callable.entity_id,
             "dataset_id": callable.dataset_id,
-            "feature": callable.output if callable.operation == "vectorize" else None,
+            "feature": (
+                ", ".join(callable.output_features)
+                if callable.operation == "escape"
+                else callable.output
+                if callable.operation == "vectorize"
+                else None
+            ),
+            "input_features": callable.input_features,
+            "output_features": callable.output_features,
             "dependency_ids": sorted(callable.dependency_ids),
         }
         for callable in sorted(plan.callables, key=lambda callable: callable.id)