indexify 0.3.1__py3-none-any.whl → 0.3.3__py3-none-any.whl

indexify/cli/cli.py

@@ -6,6 +6,8 @@ from indexify.utils.logging import (
 
 configure_logging_early()
 
+import importlib.metadata
+import json
 import os
 import shutil
 import signal
@@ -17,6 +19,7 @@ from importlib.metadata import version
 from pathlib import Path
 from typing import Annotated, List, Optional, Tuple
 
+import docker
 import nanoid
 import structlog
 import typer
@@ -190,7 +193,9 @@ def executor(
             "--function",
             "-f",
             help="Function that the executor will run "
-            "specified as <namespace>:<workflow>:<function>:<version>",
+            "specified as <namespace>:<workflow>:<function>:<version>"
+            "version is optional, not specifying it will make the server send any version"
+            "of the function",
         ),
     ] = None,
     config_path: Optional[str] = typer.Option(
@@ -261,16 +266,23 @@ def _parse_function_uris(uri_strs: Optional[List[str]]) -> Optional[List[Functio
     uris: List[FunctionURI] = []
     for uri_str in uri_strs:
         tokens = uri_str.split(":")
-        if len(tokens) != 4:
+        # FIXME bring this back when we have a dynamic scheduler
+        # if len(tokens) != 4:
+        if len(tokens) < 3 and len(tokens) > 4:
             raise typer.BadParameter(
-                "Function should be specified as <namespace>:<workflow>:<function>:<version>"
+                "Function should be specified as <namespace>:<workflow>:<function>:<version> or"
+                "<namespace>:<workflow>:<function>"
             )
+        try:
+            version = tokens[3]
+        except IndexError:
+            version = None
         uris.append(
             FunctionURI(
                 namespace=tokens[0],
                 compute_graph=tokens[1],
                 compute_fn=tokens[2],
-                version=tokens[3],
+                version=version,
             )
         )
     return uris
@@ -285,7 +297,57 @@ def _create_image(image: Image, python_sdk_path):
 
 
 def _build_image(image: Image, python_sdk_path: Optional[str] = None):
-    built_image, output = image.build(python_sdk_path=python_sdk_path)
-    for line in output:
-        print(line)
-    print(f"built image: {built_image.tags[0]}")
+    docker_file = _generate_dockerfile(image, python_sdk_path=python_sdk_path)
+    image_name = f"{image._image_name}:{image._tag}"
+
+    # low_level_client = docker.APIClient(base_url=docker_client.api.base_url)
+    docker_host = os.getenv("DOCKER_HOST", "unix:///var/run/docker.sock")
+    low_level_client = docker.APIClient(base_url=docker_host)
+    docker.api.build.process_dockerfile = lambda dockerfile, path: (
+        "Dockerfile",
+        dockerfile,
+    )
+    generator = low_level_client.build(
+        dockerfile=docker_file,
+        rm=True,
+        path=".",
+        tag=image_name,
+    )
+
+    for output in generator:
+        for line in output.decode().splitlines():
+            json_line = json.loads(line)
+            if "stream" in json_line:
+                print(json_line["stream"], end="")
+
+            elif "errorDetail" in json_line:
+                print(json_line["errorDetail"]["message"])
+
+
+def _generate_dockerfile(image, python_sdk_path: Optional[str] = None):
+    docker_contents = [
+        f"FROM {image._base_image}",
+        "RUN mkdir -p ~/.indexify",
+        f"RUN echo {image._image_name} > ~/.indexify/image_name",  # TODO: Do we still use this in executors?
+        f"RUN echo {image.hash()} > ~/.indexify/image_hash",  # TODO: Do we still use this in executors?
+        "WORKDIR /app",
+    ]
+
+    for build_op in image._build_ops:
+        docker_contents.append(build_op.render())
+
+    if python_sdk_path is not None:
+        print(f"Building image {image._image_name} with local version of the SDK")
+
+        if not os.path.exists(python_sdk_path):
+            print(f"error: {python_sdk_path} does not exist")
+            os.exit(1)
+        docker_contents.append(f"COPY {python_sdk_path} /app/python-sdk")
+        docker_contents.append("RUN (cd /app/python-sdk && pip install .)")
+    else:
+        docker_contents.append(
+            f"RUN pip install indexify=={importlib.metadata.version('indexify')}"
+        )
+
+    docker_file = "\n".join(docker_contents)
+    return docker_file

indexify/executor/api_objects.py

@@ -20,7 +20,7 @@ class FunctionURI(BaseModel):
     namespace: str
     compute_graph: str
     compute_fn: str
-    version: str
+    version: Optional[str] = None
 
 
 class ExecutorMetadata(BaseModel):

indexify/function_executor/service.py

@@ -86,33 +86,37 @@ class Service(FunctionExecutorServicer):
         # If our code raises an exception the grpc framework converts it into GRPC_STATUS_UNKNOWN
         # error with the exception message. Differentiating errors is not needed for now.
         RunTaskRequestValidator(request=request).check()
+        self._check_task_routed_correctly(request)
 
+        return RunTaskHandler(
+            request=request,
+            graph_name=self._graph_name,
+            graph_version=self._graph_version,
+            function_name=self._function_name,
+            function=self._function,
+            invocation_state=ProxiedInvocationState(
+                request.task_id, self._invocation_state_proxy_server
+            ),
+            logger=self._logger,
+        ).run()
+
+    def _check_task_routed_correctly(self, request: RunTaskRequest):
         # Fail with internal error as this happened due to wrong task routing to this Server.
+        # If we run the wrongly routed task then it can steal data from this Server if it belongs
+        # to a different customer.
         if request.namespace != self._namespace:
             raise ValueError(
                 f"This Function Executor is not initialized for this namespace {request.namespace}"
             )
         if request.graph_name != self._graph_name:
             raise ValueError(
-                f"This Function Executor is not initialized for this graph {request.graph_name}"
+                f"This Function Executor is not initialized for this graph_name {request.graph_name}"
             )
         if request.graph_version != self._graph_version:
             raise ValueError(
-                f"This Function Executor is not initialized for this graph version {request.graph_version}"
+                f"This Function Executor is not initialized for this graph_version {request.graph_version}"
             )
         if request.function_name != self._function_name:
             raise ValueError(
-                f"This Function Executor is not initialized for this function {request.function_name}"
+                f"This Function Executor is not initialized for this function_name {request.function_name}"
             )
-
-        return RunTaskHandler(
-            request=request,
-            graph_name=self._graph_name,
-            graph_version=self._graph_version,
-            function_name=self._function_name,
-            function=self._function,
-            invocation_state=ProxiedInvocationState(
-                request.task_id, self._invocation_state_proxy_server
-            ),
-            logger=self._logger,
-        ).run()

{indexify-0.3.1.dist-info → indexify-0.3.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.1
+Version: 0.3.3
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.3.1.dist-info → indexify-0.3.3.dist-info}/RECORD

@@ -1,6 +1,6 @@
-indexify/cli/cli.py,sha256=II73vJlVeEy51pPLnrnBcYJbZQxCPItkFRFEN8MD37c,9122
+indexify/cli/cli.py,sha256=y6JNaMd5by7_v3jE1iah1krlvfzbN4r5A2WkgGTy1Ts,11427
 indexify/executor/README.md,sha256=ozC6_hMkhQQNVCMEpBxwiUALz6lwErPQxNxQfQDqnG4,2029
-indexify/executor/api_objects.py,sha256=QeXOZpS7nkVA_dsRQFlS3EuMvqFvLGaw1Spcr0c-5lA,1015
+indexify/executor/api_objects.py,sha256=k5tKYxaWml0sSECoEDzamCYeJnlD7zO2M7E_qGwyMrg,1032
 indexify/executor/downloader.py,sha256=Vrq1dAW4BifG62tlqFnImiMxdezIgOZcByTRnhDsnnw,6457
 indexify/executor/executor.py,sha256=lln0p0cVN2aNY3bsHFp7Phbaa2q9gnQABi1xEPxo1Nc,5859
 indexify/executor/function_executor/function_executor.py,sha256=a9pAWQWDLJL-i5bp1PEMD2eMYOTvXAh_1jQsXcL_JCQ,5226
@@ -34,11 +34,11 @@ indexify/function_executor/proto/function_executor_pb2.pyi,sha256=TQEOl3W1s4fxtp
 indexify/function_executor/proto/function_executor_pb2_grpc.py,sha256=xrqIDrcl1PTyQTGAAGrgSDKEtZilC445Op0tJ7LmS5Q,8716
 indexify/function_executor/proto/message_validator.py,sha256=OKXPYgy5L9c-spnV9Zjv7PA_yxwzvykfhbYylYx8cwQ,1456
 indexify/function_executor/server.py,sha256=tJzUy_v4BT8Le9G3hgtiuDJo9YVFkAU2dVISSsX36II,1061
-indexify/function_executor/service.py,sha256=3JYcMcWiCgvxXhRPReUUBjTBNLIqUbhhsGZIsqIcVw4,5343
+indexify/function_executor/service.py,sha256=i3bI7RSKSr3cDiNFoZdl4TSY_SMKlXBXPdExjvaAoBY,5606
 indexify/utils/README.md,sha256=2g8-H9GopacOW4YrViZc0QsaJPtK-Fox7GyfX01kcDk,86
 indexify/utils/http_client.py,sha256=deMlmAu4E_ZXV3blCdWNag3uO_cyD-GsMZjFFmO5r7s,3541
 indexify/utils/logging.py,sha256=c6NwzY7uVHMRJc8f2w2KF36rNkeZVoQfGdq7suIg9s8,2025
-indexify-0.3.1.dist-info/METADATA,sha256=4UcqDjQRLNcVGFezaIROqWdtJxCsPP0hLq8-hlFvDn0,1383
-indexify-0.3.1.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.3.1.dist-info/entry_points.txt,sha256=pJG0YRnypesbiNJHuObfHEkjk0p_ZvEDTyyTH0kGVTY,108
-indexify-0.3.1.dist-info/RECORD,,
+indexify-0.3.3.dist-info/METADATA,sha256=VLMu3g8ZamLGLCjjqwPrBxqg8ZStvSZooP9ITl-BbHw,1383
+indexify-0.3.3.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.3.3.dist-info/entry_points.txt,sha256=pJG0YRnypesbiNJHuObfHEkjk0p_ZvEDTyyTH0kGVTY,108
+indexify-0.3.3.dist-info/RECORD,,