indexify 0.2.38__py3-none-any.whl → 0.2.40__py3-none-any.whl

indexify/cli.py

@@ -6,9 +6,11 @@ import subprocess
 import sys
 import threading
 import time
+from importlib.metadata import version
 from typing import Annotated, List, Optional
 
 import nanoid
+import structlog
 import typer
 from rich.console import Console
 from rich.panel import Panel
@@ -32,6 +34,7 @@ custom_theme = Theme(
     }
 )
 
+logging = structlog.get_logger(module=__name__)
 console = Console(theme=custom_theme)
 
 app = typer.Typer(pretty_exceptions_enable=False, no_args_is_help=True)
@@ -158,6 +161,9 @@ def build_default_image():
 @app.command(help="Joins the extractors to the coordinator server")
 def executor(
     server_addr: str = "localhost:8900",
+    dev: Annotated[
+        bool, typer.Option("--dev", "-d", help="Run the executor in development mode")
+    ] = False,
     workers: Annotated[
         int, typer.Option(help="number of worker processes for extraction")
     ] = 1,
@@ -174,19 +180,26 @@ def executor(
         "1", help="Requested Image Version for this executor"
     ),
 ):
+    # configure structured logging
+    if not dev:
+        processors = [
+            structlog.processors.dict_tracebacks,
+            structlog.processors.JSONRenderer(),
+        ]
+        structlog.configure(processors=processors)
+
     id = nanoid.generate()
-    console.print(
-        Panel(
-            f"Number of workers: {workers}\n"
-            f"Config path: {config_path}\n"
-            f"Server address: {server_addr}\n"
-            f"Executor ID: {id}\n"
-            f"Executor cache: {executor_cache}\n"
-            f"Name Alias: {name_alias}"
-            f"Image Version: {image_version}\n",
-            title="Agent Configuration",
-            border_style="info",
-        )
+    executor_version = version("indexify")
+    logging.info(
+        "executor started",
+        workers=workers,
+        server_addr=server_addr,
+        config_path=config_path,
+        executor_id=id,
+        executor_version=executor_version,
+        executor_cache=executor_cache,
+        name_alias=name_alias,
+        image_version=image_version,
     )
 
     from pathlib import Path
@@ -208,8 +221,8 @@ def executor(
 
     try:
         asyncio.get_event_loop().run_until_complete(agent.run())
-    except asyncio.CancelledError as ex:
-        console.print(Text(f"Exiting gracefully: {ex}", style="bold yellow"))
+    except asyncio.CancelledError:
+        logging.info("graceful shutdown")
 
 
 def _create_image(image: Image, python_sdk_path):

indexify/executor/agent.py

@@ -299,7 +299,12 @@ class ExtractorAgent:
                 image_version=image_version,
                 labels=runtime_probe.labels,
             ).model_dump()
-            logging.info("registering_executor", executor_id=self._executor_id, url=url)
+            logging.info(
+                "registering_executor",
+                executor_id=self._executor_id,
+                url=url,
+                executor_version=executor_version,
+            )
             try:
                 async with get_httpx_client(self._config_path, True) as client:
                     async with aconnect_sse(

indexify/functions_sdk/graph.py

@@ -65,7 +65,11 @@ def is_pydantic_model_from_annotation(type_annotation):
 
 class Graph:
     def __init__(
-        self, name: str, start_node: IndexifyFunction, description: Optional[str] = None
+        self,
+        name: str,
+        start_node: IndexifyFunction,
+        description: Optional[str] = None,
+        tags: Dict[str, str] = {},
     ):
         self.name = name
         self.description = description
@@ -73,6 +77,7 @@ class Graph:
         self.routers: Dict[str, List[str]] = defaultdict(list)
         self.edges: Dict[str, List[str]] = defaultdict(list)
         self.accumulator_zero_values: Dict[str, Any] = {}
+        self.tags = tags
 
         self.add_node(start_node)
         if issubclass(start_node, IndexifyRouter):
@@ -204,12 +209,15 @@ class Graph:
                     )
                 )
 
+        print("TAGS", self.tags)
+
         return ComputeGraphMetadata(
             name=self.name,
             description=self.description or "",
             start_node=NodeMetadata(compute_fn=start_node),
             nodes=metadata_nodes,
             edges=metadata_edges,
+            tags=self.tags,
             runtime_information=RuntimeInformation(
                 major_version=sys.version_info.major,
                 minor_version=sys.version_info.minor,

indexify/functions_sdk/graph_definition.py

@@ -44,6 +44,7 @@ class ComputeGraphMetadata(BaseModel):
     name: str
     description: str
     start_node: NodeMetadata
+    tags: Dict[str, str] = {}
     nodes: Dict[str, NodeMetadata]
     edges: Dict[str, List[str]]
     accumulator_zero_values: Dict[str, bytes] = {}

{indexify-0.2.38.dist-info → indexify-0.2.40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.38
+Version: 0.2.40
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.2.38.dist-info → indexify-0.2.40.dist-info}/RECORD

@@ -1,23 +1,22 @@
 indexify/__init__.py,sha256=P0mvM8sbkeS2CjYzRYyzb42CnXGhyJXdz4FdmTBMSWM,697
-indexify/cli.py,sha256=TBycogrXcTK7q8IIcJxcpPPoZYve_rp6FYipWbkiTUI,8528
+indexify/cli.py,sha256=QqaRpm-XywJQ3zh-2pgABi15Jl0IkwA_sz6PRNvQ4wc,8872
 indexify/common_util.py,sha256=LKS6yZ3yv8nF2J-KzisGIjqjTvCn7tLFifQJLT4gHRg,3529
 indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-rdi4,1339
 indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
 indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
 indexify/error.py,sha256=qAWr8R6AxPkjsxHSzXTc8zqYnNO_AjOqqYEPsQvF1Zs,238
-indexify/executor/agent.py,sha256=zWdcroELAA-C8LaChR518ey5UKaUhfTHtkyawpdOkIE,14053
+indexify/executor/agent.py,sha256=6xcGDZLwFHlFDiqW9EOI_Dfxh8ahdhGZq4PZUoyPLEY,14167
 indexify/executor/api_objects.py,sha256=mvmwGbK4paJNQGFvbtNHMPpiH_LpVhrlRnCcrqS6HOQ,859
 indexify/executor/downloader.py,sha256=dHLxoBnX8-Bh4yZtFDYptZNF6rlVtmTk_70JK8Ect5w,4184
 indexify/executor/executor_tasks.py,sha256=A0UIEZ5VaB6zSkFQG81UmTW0E57MTYhGlaXuAbRV8lQ,1884
 indexify/executor/function_worker.py,sha256=wRW2-X9dNI80KhwTD1vD-pcyetsVKVs6vVdg7L7JjcQ,6462
-indexify/executor/image_dependency_installer.py,sha256=ct8GmzgkaPi6NAblk68IJJWo5MecIUubELotmSrgoRQ,1759
 indexify/executor/indexify_executor.py,sha256=2Ut_VX-Su_lm4b4aEROyRJ3gXx-uFHA-V7EN0sWiARE,771
 indexify/executor/runtime_probes.py,sha256=mjw2_mGQ622wRT_39WPGGgPEZQTgtrf3-ICcUUZOeyg,2126
 indexify/executor/task_reporter.py,sha256=XlEhNf_ScNnzG67zbtVwL7_9Bo8MvPZiHLI5UHymUnM,5305
 indexify/executor/task_store.py,sha256=JlRlWwAm4YjFRkTNRx-6GsUcmOzcyvzb5Csa5XDpRTI,3982
 indexify/functions_sdk/data_objects.py,sha256=wXbUa9hjU6rsXmmk19vQ5Kixf3FsI59VBWPNmHasAX0,854
-indexify/functions_sdk/graph.py,sha256=K7ZF8fPhISVWoNzUTkDF8dMfpF-NKw88FSe9dZHISME,13301
-indexify/functions_sdk/graph_definition.py,sha256=rJmGcy9u5A_Sme6Ol33NsCnSKQVjyUfeN9LnH3bU88Y,1732
+indexify/functions_sdk/graph.py,sha256=jZSOVmlQB8Eb8MBz0TeRADl_4y5rnrjdTTy6VdRLISY,13448
+indexify/functions_sdk/graph_definition.py,sha256=W3zRERTOC-UbiRms-NqvtlLDpSZM-33TocKeZXb5o_A,1762
 indexify/functions_sdk/graph_validation.py,sha256=mN2Fcp91GIwFZEQP6z_qGqt4LkLM70SnI7AWBi4CmKQ,2509
 indexify/functions_sdk/image.py,sha256=QK0H6KxLWriB_z4M0kunKzzHdHxYLWL670RPYgYuf_8,1762
 indexify/functions_sdk/indexify_functions.py,sha256=kjGmkRYnA_KO5ypkyaIhqrz0-3UJOFTPGldgOrHSRHY,13401
@@ -28,8 +27,8 @@ indexify/http_client.py,sha256=iLafZagCFnlTS6uHfOjInogjg0uXW_zXEspIN7ttB5I,15903
 indexify/remote_graph.py,sha256=OzBucU4buR5UdTViNJvh1RfnOTmYA34Uel5qXnRsQsA,5006
 indexify/remote_pipeline.py,sha256=oqx57rSPszNS3DToXO_nf-CKqkCZWptm1u_p3orV_gQ,790
 indexify/settings.py,sha256=Ny59mzYI4gbXoK8hjx66a_men6ndbd1J1zCTcKOoyzg,50
-indexify-0.2.38.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-indexify-0.2.38.dist-info/METADATA,sha256=zS1J2a8q0kGIzLLKyGlqSKiJv6nWfHVLXKehq0KXd9U,6199
-indexify-0.2.38.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-indexify-0.2.38.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
-indexify-0.2.38.dist-info/RECORD,,
+indexify-0.2.40.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+indexify-0.2.40.dist-info/METADATA,sha256=x-8XRtgGLpvgQ5fEXUxamZwgoNtGCWPcbyvAgciO2HE,6199
+indexify-0.2.40.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+indexify-0.2.40.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
+indexify-0.2.40.dist-info/RECORD,,

indexify/executor/image_dependency_installer.py

@@ -1,64 +0,0 @@
-import os
-import subprocess
-
-from rich.console import Console
-from rich.text import Text
-from rich.theme import Theme
-
-from indexify.functions_sdk.image import ImageInformation
-
-custom_theme = Theme(
-    {
-        "info": "cyan",
-        "warning": "yellow",
-        "error": "red",
-        "success": "green",
-    }
-)
-
-console = Console(theme=custom_theme)
-
-
-def _record_image_name(name: str, version: int):
-    dir_path = os.path.expanduser("~/.indexify/")
-
-    file_path = os.path.expanduser("~/.indexify/image_name")
-    os.makedirs(dir_path, exist_ok=True)
-    with open(file_path, "w") as file:
-        file.write(name)
-
-    file_path = os.path.expanduser("~/.indexify/image_version")
-    os.makedirs(dir_path, exist_ok=True)
-    with open(file_path, "w") as file:
-        file.write(str(version))
-
-
-def _install_dependencies(run_str: str):
-    # Throw error to the caller if these subprocesses fail.
-    proc = subprocess.run(run_str.split())
-    if proc.returncode != 0:
-        raise Exception(f"Unable to install dep `{run_str}`")
-
-
-def executor_image_builder(
-    image_info: ImageInformation, name_alias: str, image_version: int
-):
-    console.print(Text("Attempting Executor Bootstrap.", style="red bold"))
-
-    run_strs = image_info.run_strs
-    console.print(Text("Attempting to install dependencies.", style="red bold"))
-
-    for run_str in run_strs:
-        console.print(Text(f"Attempting {run_str}", style="red bold"))
-        _install_dependencies(run_str)
-
-    console.print(Text("Install dependencies done.", style="red bold"))
-
-    console.print(
-        Text(
-            f"Recording image name {name_alias} and version {image_version}",
-            style="red bold",
-        )
-    )
-
-    _record_image_name(name_alias, image_version)