layernext-beta 2.2.1b7__tar.gz → 2.2.1b8__tar.gz

{layernext-beta-2.2.1b7/layernext_beta.egg-info → layernext-beta-2.2.1b8}/PKG-INFO

@@ -1,13 +1,10 @@
 Metadata-Version: 2.1
 Name: layernext-beta
-Version: 2.2.1b7
+Version: 2.2.1b8
 Summary: LayerNext Python SDK
-Home-page: UNKNOWN
 Author: LayerNext
 Author-email: <support@layernext.ai>
-License: UNKNOWN
 Keywords: python,datalake,datasetsync,ai,annotation,layernext,layernext,machine learning
-Platform: UNKNOWN
 Classifier: Development Status :: 1 - Planning
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3
@@ -40,5 +37,3 @@ url = 'https://api.xxxx.layernext.ai'
 client = layernext.LayerNextClient(api_key, secret, url)
 
 ```
-
-

{layernext-beta-2.2.1b7 → layernext-beta-2.2.1b8}/layernext/__init__.py

@@ -8,7 +8,7 @@ from layernext.datalake.constants import (AnnotationUploadType, ItemType, JobSta
                                           SortFieldName, SortOrder, AnnotationShapeType)
 from deprecated import deprecated
 
-__version__ = '2.2.1b7'
+__version__ = '2.2.1b8'
 
 
 class LayerNextClient:
@@ -2045,10 +2045,23 @@ class LayerNextClient:
             confidence_threshold=0.5  
             ):
         
+        session_id = str(datetime.datetime.now().timestamp())
+        job_name = f"Auto-tagging-generate-{model_id}"
+        
+        res_obj = self.create_or_update_job(session_id, job_name, JobType.GENERATE_AUTO_TAGGING.value, 0, JobStatus.QUEUED.value)
+
+        if("success" in res_obj and res_obj["success"]==True) and res_obj["jobId"] is not None:
+            job_id = res_obj["jobId"]
+        else:
+            print("Error occurred while create job")
+            raise Exception("Error occurred while create job")
+        
         _automatic_analysis_client = automatic_analysis.AutomaticAnalysisClient(
             self.encoded_key_secret, self.layernext_url)
-        _automatic_analysis_client.model_tag_collection(
-            collection_id, model_id, input_resolution,confidence_threshold)
+        res = _automatic_analysis_client.model_tag_collection(
+            collection_id, model_id, input_resolution,confidence_threshold,session_id,job_id,job_name)
+        
+        return res
 
 
     """
@@ -2066,10 +2079,23 @@ class LayerNextClient:
             confidence_threshold=0.5   
             ):
         
+        session_id = str(datetime.datetime.now().timestamp())
+        job_name = f"Auto-tagging-generate-{model_id}"
+        
+        res_obj = self.create_or_update_job(session_id, job_name, JobType.GENERATE_AUTO_TAGGING.value, 0, JobStatus.QUEUED.value)
+
+        if("success" in res_obj and res_obj["success"]==True) and res_obj["jobId"] is not None:
+            job_id = res_obj["jobId"]
+        else:
+            print("Error occurred while create job")
+            raise Exception("Error occurred while create job")
+        
         _automatic_analysis_client = automatic_analysis.AutomaticAnalysisClient(
             self.encoded_key_secret, self.layernext_url)
-        _automatic_analysis_client.model_tag_population(
-            item_type, model_id, input_resolution, confidence_threshold)
+        res = _automatic_analysis_client.model_tag_population(
+            item_type, model_id, input_resolution, confidence_threshold,session_id,job_id,job_name)
+        
+        return res
 
 
     """
@@ -2126,10 +2152,22 @@ class LayerNextClient:
             filters={}        
             ):
         
+        session_id = str(datetime.datetime.now().timestamp())
+        job_name = f"Embedding-generate-{model_id}"
+        
+        res_obj = self.create_or_update_job(session_id, job_name, JobType.GENERATE_EMBEDDING.value, 0, JobStatus.QUEUED.value)
+
+        if("success" in res_obj and res_obj["success"]==True) and res_obj["jobId"] is not None:
+            job_id = res_obj["jobId"]
+        else:
+            print("Error occurred while create job")
+            raise Exception("Error occurred while create job")
+        
         _automatic_analysis_client = automatic_analysis.AutomaticAnalysisClient(
             self.encoded_key_secret, self.layernext_url)
-        _automatic_analysis_client.embedding_model_inference_collection(
-            collection_id, model_id, query, filters)
+        res = _automatic_analysis_client.embedding_model_inference_collection(
+            collection_id, model_id, query, filters, session_id,job_id, job_name)
+        return res
 
 
     def generate_embeddings_to_metalake(
@@ -2140,10 +2178,23 @@ class LayerNextClient:
             filters={}     
             ):
         
+        session_id = str(datetime.datetime.now().timestamp())
+        job_name = f"Embedding-generate-{model_id}"
+        
+        res_obj = self.create_or_update_job(session_id, job_name, JobType.GENERATE_EMBEDDING.value, 0, JobStatus.QUEUED.value)
+
+        if("success" in res_obj and res_obj["success"]==True) and res_obj["jobId"] is not None:
+            job_id = res_obj["jobId"]
+        else:
+            print("Error occurred while create job")
+            raise Exception("Error occurred while create job")
+        
         _automatic_analysis_client = automatic_analysis.AutomaticAnalysisClient(
             self.encoded_key_secret, self.layernext_url)
-        _automatic_analysis_client.model_embedding_population(
-            item_type, model_id, query, filters)
+        res = _automatic_analysis_client.model_embedding_population(
+            item_type, model_id, query, filters, session_id,job_id, job_name)
+        
+        return res
 
     """
     get item list from data lake
@@ -2675,7 +2726,7 @@ class LayerNextClient:
         job_type:int, 
         progress:int, 
         status:int, 
-        job_detail:dict
+        job_detail:dict={}
     ):
         
         if(not isinstance(session_id, str) or not isinstance(job_name, str) or not isinstance(progress, (int, float)) or not isinstance(JobStatus(status), JobStatus) or not isinstance(JobType(job_type), JobType)):

{layernext-beta-2.2.1b7 → layernext-beta-2.2.1b8}/layernext/automatic_analysis/__init__.py

@@ -1,4 +1,5 @@
-from .automatic_analysis_interface import AutomaticAnalysisInterface
+from layernext.automatic_analysis.automatic_analysis_interface import AutomaticAnalysisInterface
+from .automatic_analysis import AutomaticAnalysis
 from .support import make_tarfile_in_memory, validate_input_prompt, validate_input_annotation_type, validate_confidence, validate_task, validate_resolution_input
 """
 Class to initiate AutomaticAnalysisClient and handle functions integrated to it
@@ -14,6 +15,7 @@ class AutomaticAnalysisClient:
         _automatic_analysis_url = f'{layernext_url}/analytics' 
         self.encoded_key_secret=encoded_key_secret
         self.layernext_url=layernext_url
+        self.automatic_analysis = AutomaticAnalysis(encoded_key_secret, _automatic_analysis_url)
         self.automatic_analysis_interface = AutomaticAnalysisInterface(encoded_key_secret, _automatic_analysis_url)
 
 
@@ -50,15 +52,18 @@ class AutomaticAnalysisClient:
             collection_id, 
             model_id, 
             input_resolution, 
-            confidence_threshold
+            confidence_threshold,
+            session_id,
+            job_id,
+            job_name
             ):
         try:
             validate_confidence(confidence_threshold)
             validate_resolution_input(input_resolution)
             application='collection_autotag'
             item_type=""
-            self.automatic_analysis_interface.tagger_detail_send(
-                application, collection_id, item_type, model_id, input_resolution, confidence_threshold)
+            self.automatic_analysis.tagger_detail_send(
+                application, collection_id, item_type, model_id, input_resolution, confidence_threshold,session_id,job_id,job_name)
         except ValueError as e:
             print(f"ValueError: {e}")
 
@@ -74,15 +79,18 @@ class AutomaticAnalysisClient:
             item_type,
             model_id,
             input_resolution, 
-            confidence_threshold
+            confidence_threshold,
+            session_id,
+            job_id,
+            job_name
             ):
         try:
             application='population_autotag'
             collection_id=""
             validate_confidence(confidence_threshold)
             validate_resolution_input(input_resolution)
-            self.automatic_analysis_interface.tagger_detail_send(
-                application,collection_id,item_type,model_id,input_resolution, confidence_threshold)
+            self.automatic_analysis.tagger_detail_send(
+                application,collection_id,item_type,model_id,input_resolution, confidence_threshold,session_id,job_id,job_name)
         except ValueError as e:
             print(f"ValueError: {e}")
 
@@ -115,7 +123,7 @@ class AutomaticAnalysisClient:
                 validate_input_prompt(kwargs["prompt"])
             if "confidence_threshold" in kwargs:
                 validate_confidence(kwargs["confidence_threshold"])
-            self.automatic_analysis_interface.annotater_detail_send(
+            self.automatic_analysis.annotater_detail_send(
                 application,project_id,model_id, annotation_type, **kwargs)
         except ValueError as e:
             print(f"ValueError: {e}")
@@ -144,24 +152,30 @@ class AutomaticAnalysisClient:
             collection_id,
             model_id, 
             query, 
-            filters):
+            filters,
+            session_id,
+            job_id,
+            job_name):
         
         application ='collection_embedding'
         item_type=""
-        self.automatic_analysis_interface.embedding_detail_send(
-            collection_id, model_id, application,item_type, query, filters)
+        return self.automatic_analysis.embedding_detail_send(
+            collection_id, model_id, application,item_type, query, filters, session_id,job_id,job_name)
 
     def model_embedding_population(
             self,
             item_type,
             model_id, 
             query, 
-            filters
+            filters,
+            session_id,
+            job_id,
+            job_name
             ):
         try:
             application='population_embedding'
             collection_id=""
-            self.automatic_analysis_interface.embedding_detail_send(
-                collection_id, model_id, application,item_type, query, filters)
+            return self.automatic_analysis.embedding_detail_send(
+                collection_id, model_id, application,item_type, query, filters, session_id, job_id, job_name)
         except ValueError as e:
             print(f"ValueError: {e}")

layernext-beta-2.2.1b8/layernext/automatic_analysis/automatic_analysis.py

@@ -0,0 +1,84 @@
+import requests
+import json
+import traceback
+
+from layernext.datalake.constants import Task
+from .support import generate_unique_id
+from .status_check_autoannotate_project import main_annotation
+from .automatic_analysis_interface import AutomaticAnalysisInterface
+"""
+Class to initiate AutomaticAnalysisClientInterface(class which handles all the API request and responses reagrding to automatic analysis) and handle functions integrated to it
+"""
+
+
+class AutomaticAnalysis:
+
+    def __init__(self, auth_token: str, automatic_analysis_url: str):
+        self.auth_token = auth_token
+        self.automatic_analysis_url = automatic_analysis_url
+        self.automatic_analysis_interface = AutomaticAnalysisInterface(auth_token, automatic_analysis_url)
+
+    """
+    Making the payload and sending it to the given API endpoint in and handle responses in regarding to autotagging
+    """
+
+    def tagger_detail_send(self, application, collection_id, item_type, model_id, input_resolution,confidence_threshold,session_id,job_id,job_name):
+        task = Task.AUTO_TAGGING.value
+        unique_id = generate_unique_id()
+        payload = {
+            "Application":application,
+            "ItemType":item_type,
+            "collectionId":collection_id,
+            "ModelID":model_id,
+            "UniqueID":unique_id,
+            "InputResolution":input_resolution,
+            "Confidence" : confidence_threshold,
+            "Task"   : task,
+            "SessionId":session_id,
+            "JobName":job_name
+        }
+
+        response = self.automatic_analysis_interface.main_analysis(payload,job_id,task)
+        return response
+    
+    
+    def annotater_detail_send(self,application,project_id,model_id, annotation_type, **kwargs):
+        task=Task.AUTO_ANNOTATION.value
+        unique_id = generate_unique_id()
+        auto_annotation_op = kwargs["auto_annotation_operation"] if "auto_annotation_operation" in kwargs else "user-model"
+        payload = {
+            "Application":application,
+            "ProjectID":project_id,
+            "ModelID":model_id,
+            "UniqueID":unique_id,
+            "Prompt": kwargs["prompt"] if "prompt" in kwargs else [],
+            "AnnotationType": annotation_type,
+            "Confidence" : kwargs["confidence_threshold"] if "confidence_threshold" in kwargs else 0.5,
+            "Autoannotation_op" : auto_annotation_op,
+            "Labels" : kwargs["labels"] if "labels" in kwargs else {},
+            "Task"   : task
+        }
+
+        response = self.automatic_analysis_interface.main_analysis(payload,"",task)
+        return response
+
+    def embedding_detail_send(self, collection_id, model_id, application,item_type, query, filters, session_id, job_id, job_name):
+
+        task=Task.EMBEDDING.value
+        unique_id = generate_unique_id()
+        payload = {
+            "collectionId": collection_id,
+            "ModelID": model_id,
+            "UniqueID": unique_id,
+            "Task" : task,
+            "Application" : application,
+            "ItemType":item_type,
+            "Query":query,
+            "Filters":filters,
+            "SessionId":session_id,
+            "JobName":job_name
+        }
+        
+        response = self.automatic_analysis_interface.main_analysis(payload,job_id,task)
+        return response
+        

layernext-beta-2.2.1b8/layernext/automatic_analysis/automatic_analysis_interface.py

@@ -0,0 +1,155 @@
+import traceback
+import requests
+import time
+import threading
+from urllib.parse import quote_plus
+from tqdm import tqdm
+
+from layernext.datalake.constants import Task
+
+
+class AutomaticAnalysisInterface:
+    def __init__(self, auth_token: str, automatic_analysis_url: str):
+        self.auth_token = auth_token
+        self.automatic_analysis_url = automatic_analysis_url
+        self.insufficient_load = False
+
+    """
+    Sending the main request to invokde flask app endpoint and start autotagging on the server side. 
+    @param url:url which the request is sent : {self.automatic_analysis_url} in automatic_analysis_interface.py
+    @param payload:payload of the request sent
+    @param headers: headers of the reqeust sent
+    """ 
+    def send_request(self,url, payload, headers, task):
+
+        try:
+            self.insufficient_load = False
+            if task==Task.EMBEDDING.value:
+                response = requests.post(url=f"{url}/dataIn_embedding", json=payload, headers=headers, timeout=30)
+            elif task==Task.AUTO_TAGGING.value:
+                response = requests.post(url=f"{url}/dataIn", json=payload, headers=headers, timeout=30)
+            elif task==Task.AUTO_ANNOTATION.value:
+                response = requests.post(url=f"{url}/dataIn_annotation", json=payload, headers=headers, timeout=30)
+            status_code = response.status_code
+            if status_code == 200:
+                pass
+                """ Okay to proceed """
+            elif status_code == 204:
+                print(f"No content return from this API call")
+            elif status_code == 401:
+                error_obj= response.json()
+                error = error_obj.get("error")
+                message = error_obj.get("message")
+                print(f"Error: {format(error)}")
+                print(message)
+            else:
+                print(f"Error occur with status code: {status_code}")
+        #Handle connection error
+        except requests.exceptions.ConnectionError as e:
+            print(f"Error: {format(e)}")
+            print(f"Failed to connect with Meta Lake Flask app")
+        #Handle timeout error
+        except requests.exceptions.Timeout as e:
+            # print(f"Error: {format(e)}")
+            print("Timeout error from Meta Lake Flask connection")
+            pass
+        #Handle HTTP errors
+        except requests.exceptions.HTTPError as e:
+            print(f"Error: {format(e)}")
+            print("HTTP error from Meta Lake Flask connection")
+        except requests.exceptions.RequestException as e:
+            print(f"An unexpected request exception occurred: {format(e)}")
+        except Exception as e1:
+            print(f"An unexpected exception occurred: {format(e1)}")
+
+
+    """
+    combining about two processes as two threads to run them parallely
+    @param url:url which the request is sent : {self.automatic_analysis_url} in automatic_analysis_interface.py
+    @param payload:payload of the request sent
+    @param headers: headers of the reqeust sent
+    @param unique_id: a unique id generated to identify which autotagging process to refer. 
+    """    
+    def main_analysis(self, payload, job_id, task):
+        url = f'{self.automatic_analysis_url}' 
+        headers = {'Authorization': 'Basic ' + self.auth_token}
+        request_thread = threading.Thread(target=self.send_request, args=(url, payload, headers, task))
+        request_thread.start()
+
+        return {"jobId": job_id}
+    
+    def inference_model_upload(self, storage_url, bucket_name, object_key, model_id, label_list, model_name, task):
+
+        hed = {'Authorization': 'Basic ' + self.auth_token}
+        payload = {
+            "storage_url": storage_url,
+            "bucket_name": bucket_name,
+            "object_key": object_key,
+            "model_ID": model_id,
+            "model_name": model_name,
+            "label_list": label_list,
+            "task": task
+        }
+        url = f'{self.automatic_analysis_url}/model_setup'    
+        
+        try:
+            response = requests.post(url=url, json=payload, headers=hed)
+            status_code = response.status_code
+            if status_code == 200:
+                print(response.json())
+                """ Okay to proceed """
+            elif status_code == 204:
+                print(f"No content return from this API call")
+            elif status_code == 401:
+                error_obj= response.json()
+                error = error_obj.get("error")
+                message = error_obj.get("message")
+                print(f"Error: {format(error)}")
+                print(message)
+            else:
+                print(f"Error occur with status code: {status_code}")
+        #Handle connection error
+        except requests.exceptions.ConnectionError as e:
+            print(f"Error: {format(e)}")
+            print(f"Failed to connect with Meta Lake Flask app")
+        #Handle timeout error
+        except requests.exceptions.Timeout as e:
+            # print(f"Error: {format(e)}")
+            print("Timeout error from Meta Lake Flask connection")
+        #Handle HTTP errors
+        except requests.exceptions.HTTPError as e:
+            print(f"Error: {format(e)}")
+            print("HTTP error from Meta Lake Flask connection")
+        except requests.exceptions.RequestException as e:
+            print(f"An unexpected request exception occurred: {format(e)}")
+        except Exception as e1:
+            print(f"An unexpected exception occurred: {format(e1)}")
+
+    def prompt_detail_send(self, name, shape, **kwargs):
+        hed = {'Authorization': 'Basic ' + self.auth_token}
+        payload = {
+        "bbox": kwargs["bbox"] if "bbox" in kwargs else {},
+        "clicks": kwargs["clicks"] if "clicks" in kwargs else [],
+        "shape" : shape,
+        "uniqueName"   : name
+        }
+        url = f'{self.automatic_analysis_url}' 
+        try:
+            response = requests.post(url=f'{url}/dataIn_prompt_infer', json=payload, headers=hed)
+            # main_annotation(url,payload,hed,unique_id,task, auto_annotation_op)
+            #print(response.json())
+        #Handle connection error
+        except requests.exceptions.ConnectionError as e:
+            print("Failed to connect with Automatic Analysis application")
+        # Handle timeout error
+        except requests.exceptions.Timeout as e:
+            print("Timeout error from Data Lake connection")
+        # Handle HTTP errors
+        except requests.exceptions.HTTPError as e:
+            print("HTTP error from Data Lake connection")
+        except requests.exceptions.RequestException as e:
+            print(f"An unexpected request exception occurred: {format(e)}")
+            traceback.print_exc()
+        except Exception as e1:
+            print(f"An unexpected exception occurred: {format(e1)}")
+            traceback.print_exc()

{layernext-beta-2.2.1b7 → layernext-beta-2.2.1b8}/layernext/datalake/__init__.py

@@ -545,6 +545,7 @@ class DatalakeClient:
         }
 
         res = self.datalake_interface.create_or_update_job(payload)
+        return res
     
     def create_embedding_collection(self, model_name, vector_dimension, index_type = None):
 

{layernext-beta-2.2.1b7 → layernext-beta-2.2.1b8}/layernext/datalake/constants.py

@@ -105,3 +105,8 @@ class JobType(Enum) :
   GENERATE_EMBEDDING = 15
   GENERATE_AUTO_TAGGING = 16
   UNKNOWN = -1
+
+class Task(Enum):
+    EMBEDDING = "embedding"
+    AUTO_TAGGING = "auto_tagging"
+    AUTO_ANNOTATION = "auto_annotation"

{layernext-beta-2.2.1b7 → layernext-beta-2.2.1b8/layernext_beta.egg-info}/PKG-INFO

@@ -1,13 +1,10 @@
 Metadata-Version: 2.1
 Name: layernext-beta
-Version: 2.2.1b7
+Version: 2.2.1b8
 Summary: LayerNext Python SDK
-Home-page: UNKNOWN
 Author: LayerNext
 Author-email: <support@layernext.ai>
-License: UNKNOWN
 Keywords: python,datalake,datasetsync,ai,annotation,layernext,layernext,machine learning
-Platform: UNKNOWN
 Classifier: Development Status :: 1 - Planning
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3
@@ -40,5 +37,3 @@ url = 'https://api.xxxx.layernext.ai'
 client = layernext.LayerNextClient(api_key, secret, url)
 
 ```
-
-

{layernext-beta-2.2.1b7 → layernext-beta-2.2.1b8}/layernext_beta.egg-info/SOURCES.txt

@@ -3,12 +3,9 @@ README.md
 setup.py
 layernext/__init__.py
 layernext/automatic_analysis/__init__.py
+layernext/automatic_analysis/automatic_analysis.py
 layernext/automatic_analysis/automatic_analysis_interface.py
 layernext/automatic_analysis/status_check_autoannotate_project.py
-layernext/automatic_analysis/status_check_autotag_collection.py
-layernext/automatic_analysis/status_check_autotag_population.py
-layernext/automatic_analysis/status_check_embedding_collection.py
-layernext/automatic_analysis/status_check_embedding_population.py
 layernext/automatic_analysis/support.py
 layernext/datalake/__init__.py
 layernext/datalake/annotation.py