matrice-compute 0.1.15__tar.gz → 0.1.17__tar.gz

{matrice_compute-0.1.15 → matrice_compute-0.1.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_compute
-Version: 0.1.15
+Version: 0.1.17
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

{matrice_compute-0.1.15 → matrice_compute-0.1.17}/matrice_compute.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_compute
-Version: 0.1.15
+Version: 0.1.17
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

{matrice_compute-0.1.15 → matrice_compute-0.1.17}/src/matrice_compute/action_instance.py

@@ -75,7 +75,10 @@ class ActionInstance:
             "facial_recognition_setup": facial_recognition_setup_execute,
             "fe_fs_streaming": fe_fs_streaming_execute,
             "inference_ws_server": inference_ws_server_execute,
-            "lpr_setup": lpr_setup_execute
+            "lpr_setup": lpr_setup_execute,
+            "fe_analytics_service": fe_analytics_service_execute,
+            "lpr_setup": lpr_setup_execute,
+            "tracker_server": inference_tracker_setup_execute
         }
         if self.action_type not in self.actions_map:
             raise ValueError(f"Unknown action type: {self.action_type}")
@@ -471,11 +474,12 @@ class ActionInstance:
             # Try to get model codebase URLs from action_details first
             model_codebase_url = job_params.get("model_codebase_url")
             model_requirements_url = job_params.get("model_requirements_url")
+            dockerId = job_params.get("_idDocker")
 
             # Fallback to API calls if not provided in action_details
             if not model_codebase_url:
                 model_codebase_url, error, message = self.scaling.get_model_codebase(
-                    model_family
+                    dockerId
                 )
                 if error:
                     logging.warning(f"Failed to get model codebase URL: {message}")
@@ -486,7 +490,7 @@ class ActionInstance:
                 model_codebase_requirements_url = model_requirements_url
             else:
                 model_codebase_requirements_url, error, message = (
-                    self.scaling.get_model_codebase_requirements(model_family)
+                    self.scaling.get_model_codebase_requirements(dockerId)
                 )
                 if error:
                     logging.warning(
@@ -1034,6 +1038,8 @@ def augmentation_server_creation_execute(
 def database_setup_execute(self: ActionInstance):
     """
     Creates and setup the database for facial recognition server.
+    MongoDB runs on port 27020:27017 (localhost only with --net=host).
+    Qdrant runs on port 6334 (localhost only with --net=host).
     """
     action_details = self.get_action_details()
     if not action_details:
@@ -1044,11 +1050,11 @@ def database_setup_execute(self: ActionInstance):
 
     project_id = action_details["_idProject"]
 
-    # Run docker compose up
-
+    # MongoDB container with --net=host (Port: 27020:27017)
     cmd = (
-        f"docker run --pull=always -p 27020:27017 "
+        f"docker run --pull=always --net=host "
         f"--name mongodbdatabase "
+        f"-v matrice_myvol:/matrice_data "
         f"-e ACTION_RECORD_ID={self.action_record_id} "
         f"-e MATRICE_ACCESS_KEY_ID={self.matrice_access_key_id} "
         f"-e MATRICE_SECRET_ACCESS_KEY={self.matrice_secret_access_key} "
@@ -1056,15 +1062,16 @@ def database_setup_execute(self: ActionInstance):
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f"{image} "
     )
-    print("Docker command", cmd)
+    logging.info("Starting MongoDB container (Port: 27020:27017): %s", cmd)
 
+    # Qdrant container with --net=host (Port: 6334)
     qdrant_cmd = (
-        f"docker run --pull=always "
+        f"docker run --pull=always --net=host "
         f"--name qdrant "
-        f"-p 6333:6333 "
-        f"-p 6334:6334 "
+        f"-v matrice_myvol:/matrice_data "
         f"{'qdrant/qdrant:latest'} "
     )
+    logging.info("Starting Qdrant container (Port: 6334): %s", qdrant_cmd)
 
     # Docker Command run
     self.start(cmd, "database_setup") 
@@ -1075,7 +1082,8 @@ def database_setup_execute(self: ActionInstance):
 @log_errors(raise_exception=False)
 def facial_recognition_setup_execute(self: ActionInstance):
     """
-    Creates and setup the database for facial recognition server.
+    Creates and setup the facial recognition worker server.
+    Facial recognition worker runs on port 8081 (localhost only with --net=host).
     """
     action_details = self.get_action_details()
 
@@ -1085,18 +1093,18 @@ def facial_recognition_setup_execute(self: ActionInstance):
 
     self.setup_action_requirements(action_details)
 
-    # Add worker container run command
+    # Facial recognition worker container with --net=host (Port: 8081)
     worker_cmd = (
-        f"docker run -d --pull=always "
+        f"docker run -d --pull=always --net=host "
         f"--name worker "
-        f"-p 8081:8081 "
+        f"-v matrice_myvol:/matrice_data "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
         f'-e MATRICE_ACCESS_KEY_ID="{self.matrice_access_key_id}" '
         f'-e ACTION_ID="{self.action_record_id}" '
         f"{image}"
     )
-    print("Worker docker run command:", worker_cmd)
+    logging.info("Starting facial recognition worker (Port: 8081): %s", worker_cmd)
 
     # Docker Command run
     self.start(worker_cmd, "facial_recognition_setup")
@@ -1104,30 +1112,30 @@ def facial_recognition_setup_execute(self: ActionInstance):
 @log_errors(raise_exception=False)
 def lpr_setup_execute(self: ActionInstance):
     """
-    Creates and setup the database for license plate server.
+    Creates and setup the license plate recognition server.
+    LPR worker runs on port 8082 (localhost only with --net=host).
     """
     action_details = self.get_action_details()
 
     if not action_details:
         return
     image = self.docker_container
-    external_port = self.scaling.get_open_port()
 
     self.setup_action_requirements(action_details)
 
-    # Add worker container run command
+    # LPR worker container with --net=host (Port: 8082)
     worker_cmd = (
         f"docker run -d --net=host --pull=always "
         f"--name lpr-worker "
-        f"-p {external_port}:8082 "
+        f"-v matrice_myvol:/matrice_data "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
         f'-e MATRICE_ACCESS_KEY_ID="{self.matrice_access_key_id}" '
         f'-e ACTION_ID="{self.action_record_id}" '
-        f'-e PORT={external_port} '
+        f'-e PORT=8082 '
         f"{image}"
     )
-    print("Worker docker run command:", worker_cmd)
+    logging.info("Starting LPR worker (Port: 8082): %s", worker_cmd)
 
     # Docker Command run
     self.start(worker_cmd, "lpr_setup")
@@ -1183,9 +1191,11 @@ def fe_fs_streaming_execute(self: ActionInstance):
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
         f"--name fe_streaming "
+        f"-v matrice_myvol:/matrice_data "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
         f'-e MATRICE_ACCESS_KEY_ID="{self.matrice_access_key_id}" '
+        f"-e PORT=3000 "
         f"{image}"
     )
     logging.info("Starting frontend streaming (Port: 3000): %s", worker_cmd)
@@ -1194,6 +1204,40 @@ def fe_fs_streaming_execute(self: ActionInstance):
     self.start(worker_cmd, "fe_fs_streaming")
 
 
+@log_errors(raise_exception=False)
+def fe_analytics_service_execute(self: ActionInstance):
+    """
+    Creates and setup the frontend analytics service.
+    Frontend analytics service runs on port 3001 (localhost only with --net=host).
+    """
+    action_details = self.get_action_details()
+
+    if not action_details:
+        return
+    image = action_details["actionDetails"].get("docker")
+
+    self.setup_action_requirements(action_details)
+
+    project_id = action_details["_idProject"]
+    
+    # Frontend analytics service with --net=host (Port: 3001)
+    worker_cmd = (
+        f"docker run -d --pull=always --net=host "
+        f"--name fe-analytics "
+        f'-e NEXT_PUBLIC_DEPLOYMENT_ENV="{os.environ.get("ENV", "prod")}" '
+        f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
+        f'-e MATRICE_ACCESS_KEY_ID="{self.matrice_access_key_id}" '
+        f'-e ACTION_ID="{self.action_record_id}" '
+        f"-e PORT=3001 "
+        f'-e PROJECT_ID="{project_id}" '
+        f"{image}"
+    )
+    logging.info("Starting frontend analytics service (Port: 3001): %s", worker_cmd)
+
+    # Docker Command run
+    self.start(worker_cmd, "fe_analytics_service")
+
+
 @log_errors(raise_exception=False)
 def synthetic_dataset_generation_execute(self: ActionInstance):
     """Execute synthetic dataset generation task."""
@@ -1462,7 +1506,10 @@ def streaming_gateway_execute(self: ActionInstance):
 
 @log_errors(raise_exception=False)
 def kafka_setup_execute(self: ActionInstance):
-    """Execute kafka server task."""
+    """
+    Execute kafka server task.
+    Kafka runs on port 9092 (SASL_PLAINTEXT) and 9093 (CONTROLLER) - localhost only with --net=host.
+    """
     action_details = self.get_action_details()
     if not action_details:
         return
@@ -1470,7 +1517,6 @@ def kafka_setup_execute(self: ActionInstance):
     host_ip = (
         urllib.request.urlopen("https://ident.me", timeout=10).read().decode("utf8")
     )
-    container_port = 9092
     # Setup credentials
     self.setup_action_requirements(action_details)
 
@@ -1538,7 +1584,7 @@ def kafka_setup_execute(self: ActionInstance):
         [f"-e {key}={shlex.quote(str(value))}" for key, value in env_vars.items()]
     )
 
-    # Build the docker command directly to match user's pattern
+    # Build the docker command with --net=host
     pypi_index = f"https://{'test.' if env != 'prod' else ''}pypi.org/simple/"
 
     if env == 'dev':
@@ -1547,8 +1593,9 @@ def kafka_setup_execute(self: ActionInstance):
     else:
         pkgs = f"matrice_common matrice"
 
+    # Kafka container with --net=host (Ports: 9092, 9093)
     cmd = (
-        f"docker run -p {host_port}:{container_port} "
+        f"docker run --net=host "
         f"{env_args} "
         f"--shm-size=30G --pull=always "
         f'aiforeveryone/matrice-kafka:latest /bin/bash -c "'
@@ -1561,5 +1608,36 @@ def kafka_setup_execute(self: ActionInstance):
         f'venv/bin/python3 main.py {self.action_record_id} {host_port}"'
     )
 
-    logging.info("cmd is: %s", cmd)
+    logging.info("Starting Kafka container (Ports: 9092, 9093): %s", cmd)
     self.start(cmd, "kafka_setup")
+
+
+@log_errors(raise_exception=False)
+def inference_tracker_setup_execute(self: ActionInstance):
+
+    """
+    Creates and start inference tracker.
+    Inference tracker runs on port 8110 (localhost only with --net=host).
+    """ 
+    
+    action_details = self.get_action_details()
+    if not action_details:
+        return
+    
+    image = self.docker_container
+    
+    self.setup_action_requirements(action_details)
+    
+    # This is the existing Docker run command
+    worker_cmd = (
+        f"docker run -d --pull=always --net=host "
+        f"--name inference-tracker-worker "
+        f"-v matrice_myvol:/matrice_data "
+        f'-e ENV="{os.environ.get("ENV", "prod")}" '
+        f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
+        f'-e MATRICE_ACCESS_KEY_ID="{self.matrice_access_key_id}" '
+        f'-e ACTION_ID="{self.action_record_id}" '
+        f"{image}"
+    )
+    
+    self.start(worker_cmd, "inference_tracker_setup")

{matrice_compute-0.1.15 → matrice_compute-0.1.17}/src/matrice_compute/scaling.py

@@ -674,16 +674,16 @@ class Scaling:
         )
 
     @log_errors(log_error=True)
-    def get_model_codebase_requirements(self, model_family_id):
+    def get_model_codebase_requirements(self, dockerId):
         """Get model codebase requirements.
 
         Args:
-            model_family_id: ID of the model family
+            dockerId: ID of the docker
 
         Returns:
             Tuple of (data, error, message) from API response
         """
-        path = f"/v1/model_store/get_user_requirements_download_path/{model_family_id}"
+        path = f"/v1/model_store/get_user_requirements_download_path/{dockerId}"
         resp = self.rpc.get(path=path)
         return self.handle_response(
             resp,