code-loader 1.0.43a0__tar.gz → 1.0.44__tar.gz

{code_loader-1.0.43a0 → code_loader-1.0.44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.43a0
+Version: 1.0.44
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.43a0 → code_loader-1.0.44}/code_loader/__init__.py

@@ -1,3 +1,4 @@
 from code_loader.leaploader import LeapLoader
 from code_loader.inner_leap_binder import global_leap_binder as leap_binder
-from code_loader.rt_api import Epoch, Experiment
+from code_loader.experiment_api.experiment import init_experiment
+from code_loader.experiment_api.client import Client

code_loader-1.0.44/code_loader/experiment_api/api.py

@@ -0,0 +1,81 @@
+
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Any
+from code_loader.experiment_api.client import Client
+from code_loader.experiment_api.types import ApiMetrics
+
+
+@dataclass
+class StartExperimentRequest:
+    projectId: str
+    experimentName: str
+    description: str
+    removeUntaggedUploadedModels: bool = True
+    codeIntegrationVersionId: Optional[str] = None
+
+@dataclass
+class StartExperimentResponse:
+    projectId: str
+    versionId: str
+    experimentId: str
+
+@dataclass
+class GetUploadModelSignedUrlRequest:
+    epoch: int
+    experimentId: str
+    versionId: str
+    projectId: str
+    fileType: str
+    origin: Optional[str] = None
+
+@dataclass
+class GetUploadModelSignedUrlResponse:
+    url: str
+    fileName: str
+
+@dataclass
+class AddExternalEpochDataRequest:
+  projectId: str
+  experimentId: str
+  epoch: int
+  metrics: ApiMetrics
+  force: bool = False
+
+@dataclass
+class TagModelRequest:
+  projectId: str
+  experimentId: str
+  epoch: int
+  tags: List[str]
+
+@dataclass
+class SetExperimentNotesRequest:
+    projectId: str
+    experimentId: str
+    notes: Dict[str, Any]
+
+class Api:
+    def __init__(self, client: Client):
+        self.client = client
+    
+    def start_experiment(self, data: StartExperimentRequest) -> StartExperimentResponse:
+        response = self.client.post('/versions/startExperiment', data)
+        self.client.check_response(response)
+        return StartExperimentResponse(**response.json())
+    
+    def get_uploaded_model_signed_url(self, data: GetUploadModelSignedUrlRequest)-> GetUploadModelSignedUrlResponse:
+        response = self.client.post('/versions/getUploadModelSignedUrl', data)
+        self.client.check_response(response)
+        return GetUploadModelSignedUrlResponse(**response.json())
+    
+    def add_external_epoch_data(self, data: AddExternalEpochDataRequest)-> None:
+        response = self.client.post('/externalepochdata/addExternalEpochData', data)
+        self.client.check_response(response)
+    
+    def tag_model(self, data: TagModelRequest)-> None:
+        response = self.client.post('/versions/tagModel', data)
+        self.client.check_response(response)
+
+    def set_experiment_notes(self, data: SetExperimentNotesRequest)-> None:
+        response = self.client.post('/versions/setExperimentNotes', data)
+        self.client.check_response(response)

{code_loader-1.0.43a0/code_loader/rt_api → code_loader-1.0.44/code_loader/experiment_api}/cli_config_utils.py

@@ -22,14 +22,14 @@ def get_cli_conf_file() -> Optional[Config]:
     if not os.path.exists(cli_conf_path):
         return None
     with open(cli_conf_path) as f:
-        y = yaml.safe_load(f)
-        envs_dict = y.get("envs")
+        config_yaml = yaml.safe_load(f)
+        envs_dict = config_yaml.get("envs")
         if envs_dict is None:
             return None
         envs = dict()
         for k, v in envs_dict.items():
             envs[k] = AuthConfig(**v)
-        return Config(envs=envs, current_env=y["current_env"])
+        return Config(envs=envs, current_env=config_yaml["current_env"])
     
 def get_auth_config() -> Optional[AuthConfig]:
     cli_conf = get_cli_conf_file()

code_loader-1.0.44/code_loader/experiment_api/client.py

@@ -0,0 +1,34 @@
+
+from typing import Any, Dict, Optional
+from code_loader.experiment_api.cli_config_utils import get_auth_config
+from code_loader.experiment_api.utils import join_url, to_dict_no_none
+import requests
+
+
+class Client:
+    def __init__(self, url: Optional[str] = None, token: Optional[str] = None):
+        if url is None or token is None:
+            configAuth = get_auth_config()
+            if configAuth is None:
+                raise Exception("No auth config found, either provide url and token or use `leap auth [url] [token]` to setup a config file")
+            url = configAuth.api_url
+            token = configAuth.api_key
+        
+        self.url = url
+        self.token = token
+
+    def __add_auth(self, headers: Dict[str, str]) -> Dict[str, str]:
+        headers['Authorization'] = f'Bearer {self.token}'
+        return headers
+    
+    def post(self, post_path: str, data: Any, headers: Dict[str, str] = {})-> requests.Response:
+        headers = self.__add_auth(headers)
+        if 'Content-Type' not in headers:
+            headers['Content-Type'] = 'application/json'
+        url = join_url(self.url, post_path)
+        json_data = to_dict_no_none(data)
+        return requests.post(url, json=json_data, headers=headers)
+    
+    def check_response(self, response: requests.Response)-> None:
+        if response.status_code >= 400:
+            raise Exception(f"Error: {response.status_code} {response.text}")

{code_loader-1.0.43a0/code_loader/rt_api → code_loader-1.0.44/code_loader/experiment_api}/epoch.py

@@ -1,9 +1,9 @@
 
 from typing import List, Optional
-from code_loader.rt_api import ExperimentContext
-from code_loader.rt_api.types import Metrics
-from code_loader.rt_api.utils import to_api_metric_value, upload_file
-from .api_client import AddExternalEpochDataRequest, GetUploadModelSignedUrlRequest, TagModelRequest
+from code_loader.experiment_api.experiment_context import ExperimentContext
+from code_loader.experiment_api.types import Metrics
+from code_loader.experiment_api.utils import to_api_metric_value, upload_file
+from code_loader.experiment_api.api import AddExternalEpochDataRequest, GetUploadModelSignedUrlRequest, TagModelRequest
 
 
 class Epoch:
@@ -19,12 +19,12 @@ class Epoch:
     def set_metrics(self, metrics: Metrics)-> None:
         self.metrics = metrics
 
-    def upload_model(self, modelFilePath: str)-> None:
+    def _upload_model(self, modelFilePath: str)-> None:
         allowed_extensions = ["h5", "onnx"]
         modelExtension = modelFilePath.split(".")[-1]
         if modelExtension not in allowed_extensions:
             raise Exception(f"Model file extension not allowed. Allowed extensions are {allowed_extensions}")
-        url = self.ctx.client.get_attach_model_upload_signed_url(GetUploadModelSignedUrlRequest(
+        url = self.ctx.api.get_uploaded_model_signed_url(GetUploadModelSignedUrlRequest(
             epoch=self.epoch,
             experimentId=self.ctx.experiment_id,
             versionId=self.ctx.version_id,
@@ -35,29 +35,29 @@ class Epoch:
         upload_file(url.url, modelFilePath)
         print("Model file uploaded")
     
-    def tag_model(self, tags: List[str])-> None:
+    def _tag_model(self, tags: List[str])-> None:
         print(f"Tagging epoch({self.epoch}) model")
-        self.ctx.client.tag_model(TagModelRequest(
+        self.ctx.api.tag_model(TagModelRequest(
             experimentId=self.ctx.experiment_id,
             projectId=self.ctx.project_id,
             epoch=self.epoch,
             tags=tags
         ))
 
-    def save(self, modelFilePath: Optional[str] = None, tags: List[str] = ['latest'])-> None:
+    def log(self, modelFilePath: Optional[str] = None, tags: List[str] = ['latest'])-> None:
         if modelFilePath is not None:
-            self.upload_model(modelFilePath)
+            self._upload_model(modelFilePath)
 
         print(f"Add metrics for epoch({self.epoch}) model")
         api_metrics ={
             key: to_api_metric_value(value) for key, value in self.metrics.items()
         }
-        self.ctx.client.add_external_epoch_data(AddExternalEpochDataRequest(
+        self.ctx.api.add_external_epoch_data(AddExternalEpochDataRequest(
             experimentId=self.ctx.experiment_id,
             projectId=self.ctx.project_id,
             epoch=self.epoch,
             metrics=api_metrics
         ))
         if modelFilePath is not None and len(tags) > 0:
-            self.tag_model(tags)
+            self._tag_model(tags)
    

code_loader-1.0.44/code_loader/experiment_api/experiment.py

@@ -0,0 +1,49 @@
+
+from typing import Any, Dict, List, Optional
+from code_loader.experiment_api.epoch import Epoch
+from code_loader.experiment_api.api import Api, SetExperimentNotesRequest, StartExperimentRequest
+from code_loader.experiment_api.experiment_context import ExperimentContext
+from code_loader.experiment_api.types import Metrics
+from code_loader.experiment_api.workingspace_config_utils import load_workspace_config
+from code_loader.experiment_api.client import  Client
+
+        
+class Experiment:
+    def __init__(self, ctx: ExperimentContext):
+        self.ctx = ctx
+
+    def init_epoch(self, epoch: int) -> Epoch:
+        return Epoch(self.ctx, epoch)
+    
+    def log_epoch(self, epoch: int, metrics: Optional[Metrics] = None, model_path: Optional[str] = None, tags: List[str] = ['latest'])-> None:
+        epoch_o = self.init_epoch(epoch)
+        if metrics is not None:
+            epoch_o.set_metrics(metrics)
+        epoch_o.log(model_path, tags)
+
+    def set_notes(self, notes: Dict[str, Any])-> None:
+        print(f"Setting experiment({self.ctx.experiment_id}) notes")
+        self.ctx.api.set_experiment_notes(SetExperimentNotesRequest(
+            experimentId=self.ctx.experiment_id,
+            projectId=self.ctx.project_id,
+            notes=notes
+        ))
+    
+def init_experiment(experimentName: str, description: str, working_dir: Optional[str] = None, client: Optional[Client] = None) -> 'Experiment':
+    if client is None:
+        client = Client()
+
+    api = Api(client)
+    
+    workspace_config = load_workspace_config(working_dir)
+    if workspace_config is None or workspace_config.projectId is None:
+        raise Exception("No leap workspace config found or projectId is missing, make sure you are in a leap workspace directory or provide a working_dir")
+
+    result = api.start_experiment(StartExperimentRequest(
+        projectId=workspace_config.projectId,
+        experimentName=experimentName,
+        description=description,
+        codeIntegrationVersionId=workspace_config.codeIntegrationId
+    ))
+    ctx = ExperimentContext(api, result.projectId, result.versionId, result.experimentId)
+    return Experiment(ctx)

{code_loader-1.0.43a0/code_loader/rt_api → code_loader-1.0.44/code_loader/experiment_api}/experiment_context.py

@@ -1,10 +1,10 @@
 from dataclasses import dataclass
-from code_loader.rt_api.api_client import ApiClient
+from code_loader.experiment_api.api import Api
 
 
 @dataclass
 class ExperimentContext:
-    client: ApiClient
+    api: Api
     project_id: str
     version_id: str
     experiment_id: str

{code_loader-1.0.43a0/code_loader/rt_api → code_loader-1.0.44/code_loader/experiment_api}/utils.py

@@ -1,20 +1,21 @@
 from dataclasses import asdict, is_dataclass
-from typing import Dict
+from typing import Any, Dict, Union
 from urllib.parse import urljoin
-from code_loader.rt_api.types import ApiMetricValue, MetricValue, NumericMetricValue, StringMetricValue
+from code_loader.experiment_api.types import ApiMetricValue, MetricValue, NumericMetricValue, StringMetricValue
 import requests
 
+
 def upload_file(url: str, file_path: str)-> None:
     with open(file_path, "rb") as f:
         requests.put(url, data=f, timeout=12_000)
 
-def to_dict_no_none(data: any)-> Dict[str, any]: # type: ignore[valid-type]
+def to_dict_no_none(data: Any)-> Union[Dict[str, Any], Any]:
     if is_dataclass(data):
         data = asdict(data)
     if isinstance(data, dict):
         return {k: to_dict_no_none(v) for k, v in data.items() if v is not None}
     elif isinstance(data, list):
-        return [to_dict_no_none(item) for item in data if item is not None]
+        return [to_dict_no_none(item) for item in data]
     else:
         return data
 

{code_loader-1.0.43a0 → code_loader-1.0.44}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "code-loader"
-version = "1.0.43a0"
+version = "1.0.44"
 description = ""
 authors = ["dorhar <doron.harnoy@tensorleap.ai>"]
 license = "MIT"

code_loader-1.0.43a0/code_loader/rt_api/__init__.py

@@ -1,12 +0,0 @@
-
-from code_loader.rt_api.experiment_context import ExperimentContext
-from code_loader.rt_api.epoch import Epoch
-from code_loader.rt_api.experiment import Experiment
-from code_loader.rt_api.api_client import ApiClient
-
-__all__ = [
-    'ExperimentContext',
-    'Epoch',
-    'Experiment',
-    'ApiClient'
-]

code_loader-1.0.43a0/code_loader/rt_api/api_client.py

@@ -1,107 +0,0 @@
-
-from dataclasses import dataclass
-from typing import Dict, List, Optional
-from code_loader.rt_api.types import ApiMetrics
-from code_loader.rt_api.utils import join_url, to_dict_no_none
-from .cli_config_utils import get_auth_config
-import requests
-
-
-@dataclass
-class StartExperimentRequest:
-    projectId: str
-    experimentName: str
-    description: str
-    removeUntaggedUploadedModels: bool = True
-    codeIntegrationVersionId: Optional[str] = None
-
-@dataclass
-class StartExperimentResponse:
-    projectId: str
-    versionId: str
-    experimentId: str
-
-@dataclass
-class GetUploadModelSignedUrlRequest:
-    epoch: int
-    experimentId: str
-    versionId: str
-    projectId: str
-    fileType: str
-    origin: Optional[str] = None
-
-@dataclass
-class GetUploadModelSignedUrlResponse:
-    url: str
-    fileName: str
-
-@dataclass
-class AddExternalEpochDataRequest:
-  projectId: str
-  experimentId: str
-  epoch: int
-  metrics: ApiMetrics
-  force: bool = False
-
-@dataclass
-class TagModelRequest:
-  projectId: str
-  experimentId: str
-  epoch: int
-  tags: List[str]
-
-@dataclass
-class SetExperimentNotesRequest:
-    projectId: str
-    experimentId: str
-    notes: Dict[str, any] # type: ignore[valid-type]
-
-class ApiClient:
-    def __init__(self, url: Optional[str] = None, token: Optional[str] = None):
-        if url is None or token is None:
-            configAuth = get_auth_config()
-            if configAuth is None:
-                raise Exception("No auth config found, either provide url and token or use `leap auth [url] [token]` to setup a config file")
-            url = configAuth.api_url
-            token = configAuth.api_key
-        
-        self.url = url
-        self.token = token
-
-    def __add_auth(self, headers: Dict[str, str]) -> Dict[str, str]:
-        headers['Authorization'] = f'Bearer {self.token}'
-        return headers
-    
-    def __post(self, post_path: str, data: any, headers: Dict[str, str] = {})-> requests.Response: # type: ignore[valid-type]
-        headers = self.__add_auth(headers)
-        if 'Content-Type' not in headers:
-            headers['Content-Type'] = 'application/json'
-        url = join_url(self.url, post_path)
-        json_data = to_dict_no_none(data)
-        return requests.post(url, json=json_data, headers=headers)
-    
-    def __check_response(self, response: requests.Response)-> None:
-        if response.status_code >= 400:
-            raise Exception(f"Error: {response.status_code} {response.text}")
-    
-    def start_experiment(self, data: StartExperimentRequest) -> StartExperimentResponse:
-        response = self.__post('/versions/startExperiment', data)
-        self.__check_response(response)
-        return StartExperimentResponse(**response.json())
-    
-    def get_attach_model_upload_signed_url(self, data: GetUploadModelSignedUrlRequest)-> GetUploadModelSignedUrlResponse:
-        response = self.__post('/versions/getUploadModelSignedUrl', data)
-        self.__check_response(response)
-        return GetUploadModelSignedUrlResponse(**response.json())
-    
-    def add_external_epoch_data(self, data: AddExternalEpochDataRequest)-> None:
-        response = self.__post('/externalepochdata/addExternalEpochData', data)
-        self.__check_response(response)
-    
-    def tag_model(self, data: TagModelRequest)-> None:
-        response = self.__post('/versions/tagModel', data)
-        self.__check_response(response)
-
-    def set_experiment_notes(self, data: SetExperimentNotesRequest)-> None:
-        response = self.__post('/versions/setExperimentNotes', data)
-        self.__check_response(response)

code_loader-1.0.43a0/code_loader/rt_api/experiment.py

@@ -1,47 +0,0 @@
-
-from typing import Dict, List, Optional
-from code_loader.rt_api import Epoch, ExperimentContext
-from code_loader.rt_api.types import Metrics
-from code_loader.rt_api.workingspace_config_utils import load_workspace_config
-from .api_client import  SetExperimentNotesRequest, StartExperimentRequest, ApiClient
-
-        
-class Experiment:
-    def __init__(self, ctx: ExperimentContext):
-        self.ctx = ctx
-
-    def start_epoch(self, epoch: int) -> Epoch:
-        return Epoch(self.ctx, epoch)
-    
-    def add_epoch(self, epoch: int, metrics: Optional[Metrics] = None, model_path: Optional[str] = None, tags: List[str] = ['latest'])-> None:
-        epoch_o = self.start_epoch(epoch)
-        if metrics is not None:
-            epoch_o.set_metrics(metrics)
-        epoch_o.save(model_path, tags)
-
-    def set_notes(self, notes: Dict[str, any])-> None: # type: ignore[valid-type]
-        print(f"Setting experiment({self.ctx.experiment_id}) notes")
-        self.ctx.client.set_experiment_notes(SetExperimentNotesRequest(
-            experimentId=self.ctx.experiment_id,
-            projectId=self.ctx.project_id,
-            notes=notes
-        ))
-    
-    @staticmethod
-    def Start(experimentName: str, description: str, working_dir: Optional[str] = None, client: Optional[ApiClient] = None) -> 'Experiment':
-        
-        if client is None:
-            client = ApiClient()
-        
-        workspace_config = load_workspace_config(working_dir)
-        if workspace_config is None or workspace_config.projectId is None:
-            raise Exception("No leap workspace config found or projectId is missing, make sure you are in a leap workspace directory or provide a working_dir")
-
-        result = client.start_experiment(StartExperimentRequest(
-            projectId=workspace_config.projectId,
-            experimentName=experimentName,
-            description=description,
-            codeIntegrationVersionId=workspace_config.codeIntegrationId
-        ))
-        ctx = ExperimentContext(client, result.projectId, result.versionId, result.experimentId)
-        return Experiment(ctx)