code-loader 1.0.45__py3-none-any.whl → 1.0.46__py3-none-any.whl

code_loader/experiment_api/api.py

@@ -6,18 +6,21 @@ from code_loader.experiment_api.types import ApiMetrics
 
 
 @dataclass
-class StartExperimentRequest:
-    projectId: str
+class InitExperimentRequest:
     experimentName: str
     description: str
+    projectName: Optional[str] = None
+    projectId: Optional[str] = None
     removeUntaggedUploadedModels: bool = True
-    codeIntegrationVersionId: Optional[str] = None
+    codeIntegrationName: Optional[str] = None
+    codeIntegrationId: Optional[str] = None
 
 @dataclass
-class StartExperimentResponse:
+class InitExperimentResponse:
     projectId: str
     versionId: str
     experimentId: str
+    isCreatedProject: bool
 
 @dataclass
 class GetUploadModelSignedUrlRequest:
@@ -34,7 +37,7 @@ class GetUploadModelSignedUrlResponse:
     fileName: str
 
 @dataclass
-class AddExternalEpochDataRequest:
+class LogExternalEpochDataRequest:
   projectId: str
   experimentId: str
   epoch: int
@@ -49,33 +52,33 @@ class TagModelRequest:
   tags: List[str]
 
 @dataclass
-class SetExperimentNotesRequest:
+class SetExperimentPropertiesRequest:
     projectId: str
     experimentId: str
-    notes: Dict[str, Any]
+    properties: Dict[str, Any]
 
 class Api:
     def __init__(self, client: Client):
         self.client = client
     
-    def start_experiment(self, data: StartExperimentRequest) -> StartExperimentResponse:
-        response = self.client.post('/versions/startExperiment', data)
+    def init_experiment(self, data: InitExperimentRequest) -> InitExperimentResponse:
+        response = self.client.post('/versions/initExperiment', data)
         self.client.check_response(response)
-        return StartExperimentResponse(**response.json())
+        return InitExperimentResponse(**response.json())
     
     def get_uploaded_model_signed_url(self, data: GetUploadModelSignedUrlRequest)-> GetUploadModelSignedUrlResponse:
         response = self.client.post('/versions/getUploadModelSignedUrl', data)
         self.client.check_response(response)
         return GetUploadModelSignedUrlResponse(**response.json())
     
-    def add_external_epoch_data(self, data: AddExternalEpochDataRequest)-> None:
-        response = self.client.post('/externalepochdata/addExternalEpochData', data)
+    def log_external_epoch_data(self, data: LogExternalEpochDataRequest)-> None:
+        response = self.client.post('/externalepochdata/logExternalEpochData', data)
         self.client.check_response(response)
     
     def tag_model(self, data: TagModelRequest)-> None:
         response = self.client.post('/versions/tagModel', data)
         self.client.check_response(response)
 
-    def set_experiment_notes(self, data: SetExperimentNotesRequest)-> None:
-        response = self.client.post('/versions/setExperimentNotes', data)
+    def set_experiment_properties(self, data: SetExperimentPropertiesRequest)-> None:
+        response = self.client.post('/versions/setExperimentProperties', data)
         self.client.check_response(response)

code_loader/experiment_api/epoch.py

@@ -1,25 +1,66 @@
-
 from typing import List, Optional
 from code_loader.experiment_api.experiment_context import ExperimentContext
 from code_loader.experiment_api.types import Metrics
 from code_loader.experiment_api.utils import to_api_metric_value, upload_file
-from code_loader.experiment_api.api import AddExternalEpochDataRequest, GetUploadModelSignedUrlRequest, TagModelRequest
+from code_loader.experiment_api.api import LogExternalEpochDataRequest, GetUploadModelSignedUrlRequest, TagModelRequest
 
 
 class Epoch:
-    def __init__(self, ctx: ExperimentContext, epoch: int):
+    """
+    Represents an epoch in an experiment.
+
+    Attributes:
+        experiment (ExperimentContext): The context of the experiment.
+        epoch (int): The epoch number.
+        metrics (Metrics): The metrics associated with the epoch.
+        default_tag (str): The default tag for the epoch.
+        ctx (ExperimentContext): The context of the experiment.
+    """
+
+    def __init__(self, ctx: ExperimentContext, epoch: int, default_tag: str = 'latest'):
+        """
+        Initializes the Epoch instance.
+
+        Args:
+            ctx (ExperimentContext): The context of the experiment.
+            epoch (int): The epoch number.
+            default_tag (str): The default tag for the epoch. Defaults to 'latest', set to '' for no default tag.
+        """
         self.experiment = ExperimentContext
         self.epoch = epoch
         self.metrics: Metrics = {}
+        self.default_tag = default_tag
         self.ctx = ctx
 
-    def add_metric(self, name: str, value: float)-> None:
+    def add_metric(self, name: str, value: float) -> None:
+        """
+        Adds a metric to the epoch.
+
+        Args:
+            name (str): The name of the metric.
+            value (float): The value of the metric.
+        """
         self.metrics[name] = value
 
-    def set_metrics(self, metrics: Metrics)-> None:
+    def set_metrics(self, metrics: Metrics) -> None:
+        """
+        Sets the metrics for the epoch.
+
+        Args:
+            metrics (Metrics): The metrics to set for the epoch.
+        """
         self.metrics = metrics
 
-    def _upload_model(self, modelFilePath: str)-> None:
+    def _upload_model(self, modelFilePath: str) -> None:
+        """
+        Uploads the model file for the epoch.
+
+        Args:
+            modelFilePath (str): The path to the model file.
+
+        Raises:
+            Exception: If the model file extension is not allowed.
+        """
         allowed_extensions = ["h5", "onnx"]
         modelExtension = modelFilePath.split(".")[-1]
         if modelExtension not in allowed_extensions:
@@ -35,7 +76,13 @@ class Epoch:
         upload_file(url.url, modelFilePath)
         print("Model file uploaded")
     
-    def _tag_model(self, tags: List[str])-> None:
+    def _tag_model(self, tags: List[str]) -> None:
+        """
+        Tags the model file for the epoch.
+
+        Args:
+            tags (List[str]): The tags to associate with the model file.
+        """
         print(f"Tagging epoch({self.epoch}) model")
         self.ctx.api.tag_model(TagModelRequest(
             experimentId=self.ctx.experiment_id,
@@ -44,20 +91,35 @@ class Epoch:
             tags=tags
         ))
 
-    def log(self, modelFilePath: Optional[str] = None, tags: List[str] = ['latest'])-> None:
+    def log(self, modelFilePath: Optional[str] = None, tags: Optional[list[str]] = None) -> None:
+        """
+        Logs the epoch with optional model file and tags.
+
+        Args:
+            modelFilePath (Optional[str]): The path to the model file. Defaults to None.
+            tags (Optional[List[str]]): A list of tags to associate with the epoch model. Will always include the default tag. Unless the default tag is set to '', all previous epoch model with the same tag will be removed
+        """
+        if tags is None:
+            tags = []
+
         if modelFilePath is not None:
             self._upload_model(modelFilePath)
+        
+        if len(self.default_tag) > 0 and self.default_tag not in tags:
+            tags.append(self.default_tag)
+
+        if len(tags) == 0 and modelFilePath is not None:
+            raise Exception("No tags provided for the epoch model. Either provide tags or use default_tag")
 
         print(f"Add metrics for epoch({self.epoch}) model")
-        api_metrics ={
+        api_metrics = {
             key: to_api_metric_value(value) for key, value in self.metrics.items()
         }
-        self.ctx.api.add_external_epoch_data(AddExternalEpochDataRequest(
+        self.ctx.api.log_external_epoch_data(LogExternalEpochDataRequest(
             experimentId=self.ctx.experiment_id,
             projectId=self.ctx.project_id,
             epoch=self.epoch,
             metrics=api_metrics
         ))
         if modelFilePath is not None and len(tags) > 0:
-            self._tag_model(tags)
-   
+            self._tag_model(tags)

code_loader/experiment_api/experiment.py

@@ -1,49 +1,138 @@
 
 from typing import Any, Dict, List, Optional
 from code_loader.experiment_api.epoch import Epoch
-from code_loader.experiment_api.api import Api, SetExperimentNotesRequest, StartExperimentRequest
+from code_loader.experiment_api.api import Api, SetExperimentPropertiesRequest, InitExperimentRequest
 from code_loader.experiment_api.experiment_context import ExperimentContext
 from code_loader.experiment_api.types import Metrics
 from code_loader.experiment_api.workingspace_config_utils import load_workspace_config
+from code_loader.experiment_api.utils import generate_experiment_name
 from code_loader.experiment_api.client import  Client
 
         
 class Experiment:
-    def __init__(self, ctx: ExperimentContext):
+    """
+    Represents an experiment in the system.
+
+    Attributes:
+        ctx (ExperimentContext): The context of the experiment, including API client and identifiers.
+        default_epoch_tag (str): The default tag for epochs. Defaults to 'latest'.
+    """
+
+    def __init__(self, ctx: ExperimentContext, default_epoch_tag: str = 'latest'):
+        """
+        Initializes the Experiment instance.
+
+        Args:
+            ctx (ExperimentContext): The context of the experiment.
+            default_epoch_tag (str): The default tag for epochs. Defaults to 'latest'.
+        """
         self.ctx = ctx
+        self.default_epoch_tag = default_epoch_tag
 
     def init_epoch(self, epoch: int) -> Epoch:
-        return Epoch(self.ctx, epoch)
+        """
+        Initializes a new epoch for the experiment.
+
+        Args:
+            epoch (int): The epoch number to initialize.
+
+        Returns:
+            Epoch: An instance of the Epoch class.
+        """
+        return Epoch(self.ctx, epoch, self.default_epoch_tag)
     
-    def log_epoch(self, epoch: int, metrics: Optional[Metrics] = None, model_path: Optional[str] = None, tags: List[str] = ['latest'])-> None:
+    def log_epoch(self, epoch: int, metrics: Optional[Metrics] = None, model_path: Optional[str] = None, tags: Optional[List[str]] = None) -> None:
+        """
+        Logs an epoch with optional metrics, model path, and tags.
+
+        Args:
+            epoch (int): The epoch number to log.
+            metrics (Optional[Metrics]): The metrics to log for the epoch. Defaults to None.
+            model_path (Optional[str]): The path to the model file. Defaults to None.
+            tags (Optional[List[str]]): A list of tags to associate with the epoch model. Will always include the default tag. Unless the default tag is set to '', all previous epoch model with the same tag will be removed.
+        """
         epoch_o = self.init_epoch(epoch)
         if metrics is not None:
             epoch_o.set_metrics(metrics)
         epoch_o.log(model_path, tags)
 
-    def set_notes(self, notes: Dict[str, Any])-> None:
-        print(f"Setting experiment({self.ctx.experiment_id}) notes")
-        self.ctx.api.set_experiment_notes(SetExperimentNotesRequest(
+    def set_properties(self, properties: Dict[str, Any]) -> None:
+        """
+        Sets properties for the experiment.
+
+        Args:
+            properties (Dict[str, Any]): A dictionary of properties to set for the experiment.
+        """
+        print(f"Setting experiment({self.ctx.experiment_id}) properties")
+        self.ctx.api.set_experiment_properties(SetExperimentPropertiesRequest(
             experimentId=self.ctx.experiment_id,
             projectId=self.ctx.project_id,
-            notes=notes
+            properties=properties
         ))
     
-def init_experiment(experimentName: str, description: str, working_dir: Optional[str] = None, client: Optional[Client] = None) -> 'Experiment':
+def init_experiment(
+        experiment_name: Optional[str] = None, 
+        description: str = "", 
+        project_name: Optional[str] = None,
+        code_integration_name: Optional[str] = None,
+        working_dir: Optional[str] = None,
+        client: Optional[Client] = None,
+        default_epoch_tag: str = 'latest',
+        ) -> 'Experiment':
+    """
+    Initializes and starts a new experiment.
+
+    Args:
+        experiment_name (Optional[str]): The name of the experiment. If not provided, a name will be generated.
+            Example generated names:
+            - Bold Crimson Serpent
+            - Brave Sun Raven
+            - Griffin of the Pearl Wings
+        description (str): A description of the experiment.
+        project_name (Optional[str]): The name of the project. If it does not exist, a new project will be created. If not provided, the project ID is loaded from the workspace config.
+        code_integration_name (Optional[str]): The name of the code integration.
+        working_dir (Optional[str]): The working directory. If not provided, the current directory is used.
+            The working directory should contain a leap.yaml file with the project ID and optionally the code integration ID.
+            This configuration is used if the project name is not provided.
+        client (Optional[Client]): The client to use for the experiment. If not provided, the client will be taken from the leap CLI configuration.
+            Ensure that the user has initialized the authentication by running `leap auth [url] [token]`.
+        default_epoch_tag (str): The default tag to use for epoch model. Default is 'latest', set to '' for no default tag.
+
+    Returns:
+        Experiment: An instance of the Experiment class.
+
+    Raises:
+        Exception: If the project name is not supplied and no workspace config is found or the project ID is missing from the leap.yaml file.
+        Exception: If the client is not supplied and the user has not initialized the authentication by running `leap auth [url] [token]`.
+    """
     if client is None:
         client = Client()
+    
+    if experiment_name is None:
+        experiment_name = generate_experiment_name()
+        print(f"Experiment name not provided, generated name: {experiment_name}")
+
 
     api = Api(client)
-    
-    workspace_config = load_workspace_config(working_dir)
-    if workspace_config is None or workspace_config.projectId is None:
-        raise Exception("No leap workspace config found or projectId is missing, make sure you are in a leap workspace directory or provide a working_dir")
+    project_id: Optional[str] = None
+    code_integration_id: Optional[str] = None
+
+    if not project_name:
+        workspace_config = load_workspace_config(working_dir)
+        if workspace_config is None or workspace_config.projectId is None:
+            raise Exception("No leap workspace config found or projectId is missing, make sure you are in a leap workspace directory or provide a working_dir")
+        project_id = workspace_config.projectId
+        code_integration_id = workspace_config.codeIntegrationId
 
-    result = api.start_experiment(StartExperimentRequest(
-        projectId=workspace_config.projectId,
-        experimentName=experimentName,
+    result = api.init_experiment(InitExperimentRequest(
+        projectId=project_id,
+        experimentName=experiment_name,
         description=description,
-        codeIntegrationVersionId=workspace_config.codeIntegrationId
+        codeIntegrationId=code_integration_id,
+        codeIntegrationName=code_integration_name,
+        projectName=project_name
     ))
+    if result.isCreatedProject:
+        print(f"Project name: {project_name} not found, created a new one with id: {result.projectId}")
     ctx = ExperimentContext(api, result.projectId, result.versionId, result.experimentId)
-    return Experiment(ctx)
+    return Experiment(ctx, default_epoch_tag)

code_loader/experiment_api/utils.py

@@ -1,8 +1,9 @@
 from dataclasses import asdict, is_dataclass
+import random
+import requests
 from typing import Any, Dict, Union
 from urllib.parse import urljoin
 from code_loader.experiment_api.types import ApiMetricValue, MetricValue, NumericMetricValue, StringMetricValue
-import requests
 
 
 def upload_file(url: str, file_path: str)-> None:
@@ -32,4 +33,53 @@ def to_api_metric_value(value: MetricValue) -> ApiMetricValue:
     elif isinstance(value, str):
         return StringMetricValue(value=value)
     else:
-        raise Exception(f"Unsupported metric value type: {type(value)}")
+        raise Exception(f"Unsupported metric value type: {type(value)}")
+
+
+def generate_experiment_name() -> str:
+    words = {
+        "adjectives": [
+            "Brave", "Clever", "Fierce", "Mysterious", "Swift", "Ancient", "Luminous", 
+            "Bold", "Majestic", "Noble", "Silent", "Vibrant", "Eternal", "Mystic", 
+            "Radiant", "Whimsical", "Serene", "Fabled", "Shadowy", "Enigmatic", "Fearless"
+        ],
+        "animals": [
+            "Tiger", "Phoenix", "Dragon", "Griffin", "Falcon", "Wolf", "Eagle", 
+            "Lion", "Panther", "Hawk", "Unicorn", "Bear", "Raven", "Fox", "Cobra", 
+            "Leopard", "Serpent", "Shark", "Owl", "Stag", "Hound", "Basilisk"
+        ],
+        "colors": [
+            "Crimson", "Azure", "Emerald", "Golden", "Silver", "Midnight", "Scarlet", 
+            "Ivory", "Obsidian", "Sapphire", "Ruby", "Onyx", "Amber", "Copper", 
+            "Pearl", "Amethyst", "Topaz", "Jade", "Bronze", "Verdant", "Indigo"
+        ],
+        "elements": [
+            "Fire", "Water", "Earth", "Air", "Lightning", "Shadow", "Ice", 
+            "Storm", "Light", "Darkness", "Sun", "Moon", "Void", "Spirit", 
+            "Flame", "Ocean", "Wind", "Frost", "Thunder", "Blaze", "Mist"
+        ],
+        "objects": [
+            "Blade", "Crown", "Shield", "Spear", "Wings", "Orb", "Heart", 
+            "Soul", "Flame", "Key", "Ring", "Sword", "Chalice", "Banner", 
+            "Gem", "Mirror", "Scroll", "Stone", "Throne", "Helm", "Talisman"
+        ]
+    }
+
+    pattern = random.choice([
+        "{adjectives} {animals}",
+        "{colors} {animals}",
+        "{elements} {animals}",
+        "{adjectives} {objects}",
+        "{colors} {objects}",
+        "{elements} {objects}",
+        "{adjectives} {elements}",
+        "{adjectives} {colors}"
+    ])
+
+    return pattern.format(
+        adjectives=random.choice(words["adjectives"]),
+        animals=random.choice(words["animals"]),
+        colors=random.choice(words["colors"]),
+        elements=random.choice(words["elements"]),
+        objects=random.choice(words["objects"])
+    )

code_loader/experiment_api/workingspace_config_utils.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 from os import path
 import os
-from typing import List, Optional
+from typing import Optional
 import yaml
 
 
@@ -10,9 +10,7 @@ class LocalProjectConfig:
     codeIntegrationId: Optional[str] = None
     projectId: Optional[str] = None
     secretId: Optional[str] = None
-    secretManagerId: Optional[str] = None
     entryFile: Optional[str] = None
-    includePatterns: Optional[List[str]] = None
 
 # Loading workspace configuration from leap.yaml
 def load_workspace_config(workspace_dir: Optional[str] = None) -> Optional[LocalProjectConfig]:
@@ -23,5 +21,10 @@ def load_workspace_config(workspace_dir: Optional[str] = None) -> Optional[Local
 
     file_path = path.join(workspace_dir, "leap.yaml")
     with open(file_path) as f:
-        y = yaml.safe_load(f)
-        return LocalProjectConfig(**y)
+        leap_yaml = yaml.safe_load(f)
+        return LocalProjectConfig(
+            codeIntegrationId=leap_yaml.get("codeIntegrationId"),
+            projectId=leap_yaml.get("projectId"),
+            secretId=leap_yaml.get("secretId"),
+            entryFile=leap_yaml.get("entryFile"),
+        )

code_loader/inner_leap_binder/leapbinder.py

@@ -420,6 +420,7 @@ class LeapBinder:
             ]
             for i, (single_metadata_name, single_metadata_result) in enumerate(raw_result.items()):
                 metadata_test_result = metadata_test_result_payloads[i]
+
                 result_shape = get_shape(single_metadata_result)
                 metadata_test_result.shape = result_shape
                 metadata_test_result.raw_result = single_metadata_result

code_loader/utils.py

@@ -1,7 +1,7 @@
 import sys
 from pathlib import Path
 from types import TracebackType
-from typing import List, Union, Tuple
+from typing import List, Union, Tuple, Any
 import traceback
 import numpy as np
 import numpy.typing as npt
@@ -43,7 +43,7 @@ def get_root_exception_file_and_line_number() -> Tuple[int, str, str]:
     return root_exception_line_number, root_exception_file_name, traceback_as_string
 
 
-def get_shape(result: Union[npt.NDArray[np.float32], str, float, int, bool]) -> List[int]:
+def get_shape(result: Any) -> List[int]:
     if not isinstance(result, np.ndarray):
         return [1]
     np_shape = result.shape

{code_loader-1.0.45.dist-info → code_loader-1.0.46.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: code-loader
-Version: 1.0.45
+Version: 1.0.46
 Summary: 
 Home-page: https://github.com/tensorleap/code-loader
 License: MIT

{code_loader-1.0.45.dist-info → code_loader-1.0.46.dist-info}/RECORD

@@ -1,6 +1,5 @@
 LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
 code_loader/__init__.py,sha256=6MMWr0ObOU7hkqQKgOqp4Zp3I28L7joGC9iCbQYtAJg,241
-code_loader/code_inegration_processes_manager.py,sha256=XslWOPeNQk4RAFJ_f3tP5Oe3EgcIR7BE7Y8r9Ty73-o,3261
 code_loader/contract/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/contract/datasetclasses.py,sha256=81TmCcVol7768lzKUp70MatLLipR3ftcR9jgE1r8Yqo,5698
 code_loader/contract/enums.py,sha256=6Lo7p5CUog68Fd31bCozIuOgIp_IhSiPqWWph2k3OGU,1602
@@ -8,22 +7,22 @@ code_loader/contract/exceptions.py,sha256=jWqu5i7t-0IG0jGRsKF4DjJdrsdpJjIYpUkN1F
 code_loader/contract/responsedataclasses.py,sha256=w7xVOv2S8Hyb5lqyomMGiKAWXDTSOG-FX1YW39bXD3A,3969
 code_loader/contract/visualizer_classes.py,sha256=Ka8fJSVKrOeZ12Eg8-dOBGMW0UswYCIkuhnNMd-7z9s,22948
 code_loader/experiment_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-code_loader/experiment_api/api.py,sha256=KI0ml2Dv0LwTNPP0uC5mEIqZqmSE1xaCpTSZBcnIu44,2351
+code_loader/experiment_api/api.py,sha256=p_III5t9WCpvd1REq9L-pF3r1RhjDlxMCBBK-Sn0B1I,2490
 code_loader/experiment_api/cli_config_utils.py,sha256=n6JMyNrquxql3KKxHhAP8jAzezlRT-PV2KWI95kKsm0,1140
 code_loader/experiment_api/client.py,sha256=8kqLWh2Zg7mNA5E9Q0v0qRuPm3OE3Rs7lfGAcXOucU4,1428
-code_loader/experiment_api/epoch.py,sha256=JDJINIrMGhTGdsXdxc4K_aTI5m7_Vwjyygr4egfv8lU,2500
-code_loader/experiment_api/experiment.py,sha256=0d0FesAhiCdxJ3x8uEqq7MdBZzxQV58MT52DKRbwp5w,2151
+code_loader/experiment_api/epoch.py,sha256=OwK8ENR7TJIkreUAZKTRJamPcHDvQYeU85qcDNxSCpo,4700
+code_loader/experiment_api/experiment.py,sha256=oVi6GAJ1e1kOygnOquTvb6SgepMhTlrpH0uVqCwviYM,6349
 code_loader/experiment_api/experiment_context.py,sha256=kdzUbuzXo1pMVslOC3TKeJwW8sx_qWkxDVrswjduH0A,194
 code_loader/experiment_api/types.py,sha256=MY8xFARHwdVA7p4dxyhD60ShmttgTvb4qdp1oEB_NPg,485
-code_loader/experiment_api/utils.py,sha256=5_Sat-NL8iY9DO69YEEHpB22i_Iz-d7QxnLZglQ39UQ,1279
-code_loader/experiment_api/workingspace_config_utils.py,sha256=fNz-xMmkKNTVjePi8r9_baYBFiiH5KO6eUOneCtIvC4,858
+code_loader/experiment_api/utils.py,sha256=XZHtxge12TS4H4-8PjV3sKuhp8Ud6ojAiIzTZJEqBqc,3304
+code_loader/experiment_api/workingspace_config_utils.py,sha256=DLzXQCg4dgTV_YgaSbeTVzq-2ja_SQw4zi7LXwKL9cY,990
 code_loader/inner_leap_binder/__init__.py,sha256=koOlJyMNYzGbEsoIbXathSmQ-L38N_pEXH_HvL7beXU,99
-code_loader/inner_leap_binder/leapbinder.py,sha256=lERXvLPfhPS3EpCJQpClcJuRWt7DyCN3hYq0jHlhb6k,23887
+code_loader/inner_leap_binder/leapbinder.py,sha256=CDiz8AWgNW-TgSPspllUUxy2Q-9V2JcavPJ0CZOAUnU,23888
 code_loader/leaploader.py,sha256=E1DHtRDzz0hyb3kAMl6S_JDbjGxJPR_bZHU77Af4rR4,17521
-code_loader/utils.py,sha256=61I4PgSl-ZBIe4DifLxMNlBELE-HQR2pB9efVYPceIU,2230
+code_loader/utils.py,sha256=TZAoUbA2pE8eK3Le3s5Xr4eRaYdeDMQtxotx6rh-5oE,2185
 code_loader/visualizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 code_loader/visualizers/default_visualizers.py,sha256=VoqO9FN84yXyMjRjHjUTOt2GdTkJRMbHbXJ1cJkREkk,2230
-code_loader-1.0.45.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
-code_loader-1.0.45.dist-info/METADATA,sha256=t7JSA3slllU07MJ4FWE6mpZVRmECnowSLpXXz7bZBQM,888
-code_loader-1.0.45.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-code_loader-1.0.45.dist-info/RECORD,,
+code_loader-1.0.46.dist-info/LICENSE,sha256=qIwWjdspQeSMTtnFZBC8MuT-95L02FPvzRUdWFxrwJY,1067
+code_loader-1.0.46.dist-info/METADATA,sha256=25rGAJqHthPl5csB16XuFX9HkycPAjnmO-5Mo35SI0U,888
+code_loader-1.0.46.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+code_loader-1.0.46.dist-info/RECORD,,

code_loader/code_inegration_processes_manager.py

@@ -1,83 +0,0 @@
-# mypy: ignore-errors
-import traceback
-from dataclasses import dataclass
-
-from typing import List, Tuple, Optional
-
-from multiprocessing import Process, Queue
-
-from code_loader.leap_loader_parallelized_base import LeapLoaderParallelizedBase
-from code_loader.leaploader import LeapLoader
-from code_loader.contract.enums import DataStateEnum
-from code_loader.metric_calculator_parallelized import MetricCalculatorParallelized
-from code_loader.samples_generator_parallelized import SamplesGeneratorParallelized
-
-
-@dataclass
-class SampleSerializableError:
-    state: DataStateEnum
-    index: int
-    leap_script_trace: str
-    exception_as_str: str
-
-
-class CodeIntegrationProcessesManager:
-    def __init__(self, code_path: str, code_entry_name: str, n_workers: Optional[int] = 2,
-                 max_samples_in_queue: int = 128) -> None:
-        self.metric_calculator_parallelized = MetricCalculatorParallelized(code_path, code_entry_name)
-        self.samples_generator_parallelized = SamplesGeneratorParallelized(code_path, code_entry_name)
-
-    def _create_and_start_process(self) -> Process:
-        process = self.multiprocessing_context.Process(
-            target=CodeIntegrationProcessesManager._process_func,
-            args=(self.code_path, self.code_entry_name, self._inputs_waiting_to_be_process,
-                  self._ready_processed_results))
-        process.daemon = True
-        process.start()
-        return process
-
-    def _run_and_warm_first_process(self):
-        process = self._create_and_start_process()
-        self.processes = [process]
-
-        # needed in order to make sure the preprocess func runs once in nonparallel
-        self._start_process_inputs([(DataStateEnum.training, 0)])
-        self._get_next_ready_processed_result()
-
-    def _operation_decider(self):
-        if self.metric_calculator_parallelized._ready_processed_results.empty() and not \
-            self.metric_calculator_parallelized._inputs_waiting_to_be_process.empty():
-            return 'metric'
-
-        if self.samples_generator_parallelized._ready_processed_results.empty() and not \
-            self.samples_generator_parallelized._inputs_waiting_to_be_process.empty():
-            return 'dataset'
-
-
-
-
-    @staticmethod
-    def _process_func(code_path: str, code_entry_name: str,
-                      samples_to_process: Queue, ready_samples: Queue,
-                      metrics_to_process: Queue, ready_metrics: Queue) -> None:
-        import os
-        os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
-
-        leap_loader = LeapLoader(code_path, code_entry_name)
-        while True:
-
-            # decide on sample or metric to process
-            state, idx = samples_to_process.get(block=True)
-            leap_loader._preprocess_result()
-            try:
-                sample = leap_loader.get_sample(state, idx)
-            except Exception as e:
-                leap_script_trace = traceback.format_exc().split('File "<string>"')[-1]
-                ready_samples.put(SampleSerializableError(state, idx, leap_script_trace, str(e)))
-                continue
-
-            ready_samples.put(sample)
-
-    def generate_samples(self, sample_identities: List[Tuple[DataStateEnum, int]]):
-        return self.start_process_inputs(sample_identities)
-