psr-factory 5.0.0b9__py3-none-win_amd64.whl → 5.0.0b12__py3-none-win_amd64.whl

psr/cloud/cloud.py

@@ -139,7 +139,6 @@ class Client:
         self._selected_cluster = kwargs.get("cluster", _DEFAULT_CLUSTER["pretty_name"])
         self._import_desktop = kwargs.get("import_desktop", True)
         self._debug_mode = kwargs.get("debug", False)
-        self._dry_run = kwargs.get("dry_run", False)
         self._timeout = kwargs.get("timeout", None)
         self._python_client = kwargs.get("python_client", False)
 
@@ -546,6 +545,7 @@ class Client:
         :return: True if the requested status is reached, False if timeout occurs.
         """
         status = None
+        last_status = None
         start_time = time()
         original_quiet_flag = self._quiet
         original_verbose_flag = self._verbose
@@ -561,7 +561,10 @@ class Client:
                         f"Timeout reached while waiting for status {requested_status}"
                     )
                     return False
-                status, _ = self.get_status(case_id)
+                status, _ = self.get_status(case_id, quiet=True)
+                if last_status != status:
+                    self._logger.info(f"Status: {STATUS_MAP_TEXT[status]}")
+                last_status = status
                 sleep(20)
         finally:
             self._quiet = original_quiet_flag
@@ -581,7 +584,7 @@ class Client:
                 os.rmdir(dir_path)
 
     @thread_safe()
-    def run_case(self, case: "Case", **kwargs) -> int:
+    def run_case(self, case: "Case", dry_run=False, **kwargs) -> int:
         self._validate_case(case)
         instance_type_map = self._get_instance_type_map()
         instance_type_id = next(
@@ -614,7 +617,6 @@ class Client:
             "s3Dados": "",
             "nproc": case.number_of_processes,
             "repositorioId": "0",
-            "repositorioPai": case.parent_case_id,
             "instanciaTipo": instance_type_id,
             "validacaoModelo": "True",
             "validacaoUsuario": "False",
@@ -632,6 +634,11 @@ class Client:
             "arquivoSaida": case.mymodel_output_file,
         }
 
+        if isinstance(case.parent_case_id, list) and case.parent_case_id is not None:
+            parameters["repositoriosPais"] = ",".join(map(str, case.parent_case_id))
+        else:
+            parameters["repositorioPai"] = case.parent_case_id
+
         if case.budget:
             parameters["budget"] = case.budget
         if case.upload_only is not None:
@@ -639,7 +646,7 @@ class Client:
 
         xml_content = create_case_xml(parameters)
 
-        if self._dry_run:
+        if dry_run:
             return xml_content
 
         if self._python_client:
@@ -675,7 +682,7 @@ class Client:
 
         return case_id
 
-    def get_status(self, case_id: int) -> tuple["ExecutionStatus", str]:
+    def get_status(self, case_id: int, quiet=False) -> tuple["ExecutionStatus", str]:
         delete_xml = not self._debug_mode
         xml_content = ""
         with CreateTempFile(
@@ -717,7 +724,8 @@ class Client:
                     f"Contact PSR support at psrcloud@psr-inc.com with following data:\n\n{xml_str}\n\n"
                 )
 
-            self._logger.info(f"Status: {STATUS_MAP_TEXT[status]}")
+            if not quiet:
+                self._logger.info(f"Status: {STATUS_MAP_TEXT[status]}")
             return status, STATUS_MAP_TEXT[status]
 
     def list_download_files(self, case_id: int) -> List[dict]:

psr/cloud/data.py

@@ -87,7 +87,9 @@ class Case:
 
         self.id: Optional[int] = kwargs.get("id", None)
         self.user: Optional[str] = kwargs.get("user", None)
-        self.parent_case_id: Optional[int] = kwargs.get("parent_case_id", None)
+        self.parent_case_id: Optional[Union[int, list]] = kwargs.get(
+            "parent_case_id", None
+        )
         self.execution_date: Optional[datetime] = kwargs.get("execution_date", None)
         self.budget: Optional[str] = kwargs.get("budget", None)
         if self.budget is not None:

psr/cloud/version.py

@@ -2,4 +2,4 @@
 # Unauthorized copying of this file, via any medium is strictly prohibited
 # Proprietary and confidential
 
-__version__ = "0.3.7"
+__version__ = "0.3.8"

psr/execqueue/client.py

@@ -0,0 +1,113 @@
+import os
+import zipfile
+import requests
+from typing import List, Optional
+
+
+def zip_directory(directory_path, output_zip):
+    """Compress a directory into a zip file."""
+    with zipfile.ZipFile(output_zip, 'w', zipfile.ZIP_DEFLATED) as zipf:
+        for root, _, files in os.walk(directory_path):
+            for file in files:
+                file_path = os.path.join(root, file)
+                arcname = os.path.relpath(file_path, start=directory_path)
+                zipf.write(file_path, arcname=arcname)
+
+
+def upload_case_file(zip_path, server_url):
+    """Upload a zip file to the server."""
+    with open(zip_path, 'rb') as f:
+        files = {'file': (os.path.basename(zip_path), f)}
+        response = requests.post(f"{server_url}/upload", files=files)
+
+    if response.status_code == 200:
+        print("Upload successful!")
+        print("Case ID:", response.json().get('case_id'))
+        return response.json().get('case_id')
+    else:
+        print("Upload failed:", response.text)
+        return None
+
+
+def run_case(case_id: str, server_url: str, cloud_execution: bool = False) -> Optional[str]:
+    """Add a case to the execution queue. For server-local run,
+    returns the execution id. For cloud run, returns the cloud upload id."""
+    data = {"case_id": case_id,'cloud_execution': cloud_execution}
+    response = requests.post(f"{server_url}/run",data=data)
+
+    if response.status_code == 200:
+        if not cloud_execution:
+            print("Added to queue successfully!")
+            print("Execution ID:", response.json().get('execution_id'))
+            return response.json().get('execution_id')
+        else:
+            print("Cloud execution queued!")
+            print("Cloud upload ID:", response.json().get('cloud_upload_id'))
+            return response.json().get('cloud_upload_id')
+    else:
+        print("Upload failed:", response.text)
+        return None
+
+
+def upload_and_run_file(zip_path: str, server_url: str, cloud_execution: bool = False):
+    """Upload a zip file to the server."""
+    with open(zip_path, 'rb') as f:
+        files = {'file': (os.path.basename(zip_path), f)}
+        data = {'cloud_execution': cloud_execution}
+        response = requests.post(f"{server_url}/upload_and_run", files=files, data=data)
+
+    if response.status_code == 200:
+        print("Upload successful! Waiting for execution.")
+
+        if cloud_execution:
+            print("Cloud upload ID:", response.json().get('cloud_upload_id'))
+            return response.json().get('cloud_upload_id')
+        else:
+            print("Local execution ID:", response.json().get('execution_id'))
+            return response.json().get('execution_id')
+    else:
+        print("Upload failed:", response.text)
+        return None
+
+
+def get_execution_status(execution_id: str, server_url: str, cloud_execution: bool = False, return_status_id: bool = False) -> Optional[str]:
+    """Get the status of an execution."""
+    print("Getting status for execution ID:", execution_id)
+    data = {'cloud_execution': cloud_execution, 'return_status_id': return_status_id}
+    response = requests.get(f"{server_url}/status/{execution_id}", data=data)
+
+    if response.status_code == 200:
+        if return_status_id:
+            print("Status ID:", response.json().get('status_id'))
+            return response.json().get('status_id') 
+        print("Status:", response.json().get('status'))
+        return response.json().get('status')
+    else:
+        print("Failed to get status:", response.text)
+        return None
+
+def get_results(execution_id, server_url, cloud_execution=False) -> Optional[List[str]]:
+    """Download the results of an execution."""
+    response = requests.get(f"{server_url}/results/{execution_id}", data={'cloud_execution': cloud_execution})
+
+    if response.status_code == 200:
+        print("Results downloaded successfully!")
+        files = response.json().get('files')
+        print("Files:", files)
+        return files
+    else:
+        print("Download failed:", response.text)
+        return None
+
+
+def download_execution_file(execution_id: str, server_url: str, file: str, download_path: str, cloud_execution: bool = False):
+    """Download the results of an execution."""
+    data = {'cloud_execution': cloud_execution}
+    response = requests.get(f"{server_url}/results/{execution_id}/{file}", data=data)
+
+    # TODO: add validation for download_path existence.
+    if response.status_code == 200:
+        with open(os.path.join(download_path, file), 'wb') as f:
+            f.write(response.content)
+    else:
+        print("Download failed:", response.text)

psr/execqueue/config.py

@@ -0,0 +1,44 @@
+import os
+import tomllib
+
+__version__ = "0.3.0"
+_app_name = "PSR Factory ExecQueue"
+DEFAULT_PORT = 5000
+FLASK_DEBUG = False
+_SETTINGS_FILE_PATH = "server_settings.toml"
+
+if os.name == 'nt':
+    _DEFAULT_SDDP_PATH = r"C:/PSR/Sddp17.3"
+else:
+    _DEFAULT_SDDP_PATH = "/opt/psr/sddp"
+DEFAULT_CLUSTER_NAME = "server"
+DEFAULT_PSRCLOUD_CLUSTER = "external"
+DEFAULT_PSRCLOUD_CLUSTER_URL = ""
+
+
+# read toml settings file
+with open(os.path.join(os.getcwd(), _SETTINGS_FILE_PATH), 'rb') as f:
+    settings = tomllib.load(f)
+
+
+sddp_path = settings.get("sddp_path", _DEFAULT_SDDP_PATH)
+
+cluster_name = settings.get("cluster_name", DEFAULT_CLUSTER_NAME)
+psrcloud_cluster = settings.get("psrcloud_cluster", DEFAULT_PSRCLOUD_CLUSTER)
+psrcloud_cluster_url = settings.get("psrcloud_cluster", DEFAULT_PSRCLOUD_CLUSTER_URL)
+
+# Base server data storage path.
+STORAGE_PATH = settings.get("storage_path", os.path.join(os.getcwd(), 'serverdata'))
+
+# Where uploaded (received) cases will be stored.
+UPLOADS_FOLDER = os.path.join(STORAGE_PATH, 'uploads')
+
+# Where results of local runs will be stored.
+LOCAL_RESULTS_FOLDER = os.path.join(STORAGE_PATH, 'local_results')
+
+# Where results of cloud runs will be stored.
+CLOUD_RESULTS_FOLDER = os.path.join(STORAGE_PATH, 'cloud_results')
+
+# Where temporary extracted case files will be stored
+TEMPORARY_UPLOAD_FOLDER = os.path.join(STORAGE_PATH, 'tmp')
+

psr/execqueue/db.py

@@ -0,0 +1,220 @@
+import datetime
+from enum import Enum
+from typing import (
+    List,
+    Optional
+)
+
+from sqlalchemy import (
+    create_engine,
+    Column,
+    Integer,
+    String,
+    ForeignKey,
+    Text,
+    TIMESTAMP,
+)
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship, sessionmaker
+
+from psr.execqueue.config import *
+
+class CloudStatus(Enum):
+    RUNNING = 1
+    FINISHED = 3
+    ERROR = 4
+    RESULTS_AVAILABLE = 5
+
+
+DB_NAME = "app.db"
+
+
+LOCAL_EXECUTION_RUNNING = 0
+LOCAL_EXECUTION_FINISHED = 1
+LOCAL_EXECUTION_ERROR = 2
+
+
+def get_db_path():
+    return os.path.join(STORAGE_PATH, DB_NAME)
+
+
+Base = declarative_base()
+
+
+class Case(Base):
+    __tablename__ = 'cases'
+
+    case_id = Column(String(26), primary_key=True)
+    upload_time = Column(TIMESTAMP, default=datetime.datetime.utcnow)
+    checksum = Column(Text)
+    removed = Column(Integer, default=0)
+
+    local_executions = relationship("LocalExecution", back_populates="case")
+    cloud_uploads = relationship("CloudUpload", back_populates="case")
+    cloud_executions = relationship("CloudExecution", back_populates="case")
+
+
+class LocalExecution(Base):
+    __tablename__ = 'local_executions'
+
+    execution_id = Column(String(26), primary_key=True)
+    case_id = Column(String(26), ForeignKey('cases.case_id'))
+    start_time = Column(TIMESTAMP, default=datetime.datetime.utcnow)
+    finish_time = Column(TIMESTAMP)
+    status = Column(Integer, default=CloudStatus.RUNNING.value)
+
+    case = relationship("Case", back_populates="local_executions")
+
+
+class CloudUpload(Base):
+    __tablename__ = 'cloud_uploads'
+
+    cloud_upload_id = Column(String(26), primary_key=True)
+    case_id = Column(String(26), ForeignKey('cases.case_id'))
+    start_time = Column(TIMESTAMP, default=datetime.datetime.utcnow)
+
+    case = relationship("Case", back_populates="cloud_uploads")
+    cloud_executions = relationship("CloudExecution", back_populates="cloud_upload")
+
+
+class CloudExecution(Base):
+    __tablename__ = 'cloud_executions'
+
+    repository_id = Column(Integer, primary_key=True)
+    cloud_upload_id = Column(String(26), ForeignKey('cloud_uploads.cloud_upload_id'))
+    case_id = Column(String(26), ForeignKey('cases.case_id'))
+    start_time = Column(TIMESTAMP, default=datetime.datetime.utcnow)
+    archived = Column(Integer, default=0)
+    status = Column(Integer, default=0)
+
+    case = relationship("Case", back_populates="cloud_executions")
+    cloud_upload = relationship("CloudUpload", back_populates="cloud_executions")
+
+
+def initialize():
+    # Create the SQLite database and create the tables
+    _db_path = get_db_path()
+    _create_db = not os.path.exists(_db_path)
+    engine = create_engine(f'sqlite:///{_db_path}')
+    Session = sessionmaker(bind=engine)
+    session = Session()
+
+    if _create_db:
+        Base.metadata.create_all(engine)
+        _first_time_setup(session)
+
+    return session, engine
+
+
+def close(session):
+    session.close()
+
+
+def _first_time_setup(session):
+    pass
+
+
+def register_case(session, case_id, checksum):
+    case = Case(case_id=case_id,
+                checksum=checksum,
+                upload_time=datetime.datetime.utcnow()
+                )
+    session.add(case)
+    session.commit()
+    # registry.configure()
+
+    return case
+
+
+def register_local_execution(session, case_id: str, execution_id: str):
+    case = session.query(Case).filter(Case.case_id == case_id).first()
+    local_execution = LocalExecution(execution_id=execution_id,
+                                     case_id=case_id,
+                                     start_time=datetime.datetime.utcnow()
+                                     )
+    case.local_executions.append(local_execution)
+    session.commit()
+    return local_execution
+
+
+def get_case_id_from_execution_id(session, execution_id: str) -> Optional[str]:
+    local_execution = session.query(LocalExecution).filter(LocalExecution.execution_id == execution_id).first()
+    return local_execution.case_id if local_execution else None
+
+
+def update_local_execution_status(session, execution_id: str, status: int):
+    local_execution = session.query(LocalExecution).filter(LocalExecution.execution_id == execution_id).first()
+    if local_execution:
+        if status not in [LOCAL_EXECUTION_FINISHED, LOCAL_EXECUTION_ERROR,
+                      LOCAL_EXECUTION_RUNNING]:
+            raise ValueError("Wrong status for update.")
+        local_execution.status = status
+        if status in [LOCAL_EXECUTION_FINISHED, LOCAL_EXECUTION_ERROR]:
+            local_execution.finish_time = datetime.datetime.utcnow()
+        session.commit()
+        return True
+
+def update_cloud_execution_status(session, repository_id: int, status: int):
+    cloud_execution = session.query(CloudExecution).filter(CloudExecution.repository_id == repository_id).first()
+    if cloud_execution:
+        if status not in CloudStatus:
+            raise ValueError("Wrong status for update.")
+        cloud_execution.status = status
+        session.commit()
+        return True
+    
+def get_local_execution_status(session, execution_id: str) -> Optional[int]:
+    local_execution = session.query(LocalExecution).filter(LocalExecution.execution_id == execution_id).first()
+    return local_execution.status if local_execution else None
+
+
+def register_cloud_upload(session, case_id: str, cloud_upload_id: str):
+    case = session.query(Case).filter(Case.case_id == case_id).first()
+    cloud_upload = CloudUpload(cloud_upload_id=cloud_upload_id,
+                               case_id=case_id,
+                               start_time=datetime.datetime.utcnow()
+                               )
+    case.cloud_uploads.append(cloud_upload)
+    session.commit()
+    return cloud_upload
+
+
+def register_cloud_execution(session, repository_id: int, cloud_upload_id: str, case_id: str):
+    cloud_upload = session.query(CloudUpload).filter(CloudUpload.cloud_upload_id == cloud_upload_id).first()
+    cloud_execution = CloudExecution(repository_id=repository_id,
+                                     cloud_upload_id=cloud_upload_id,
+                                     case_id=case_id,
+                                     start_time=datetime.datetime.utcnow(),
+                                     status=CloudStatus.RUNNING.value
+                                     )
+    cloud_upload.cloud_executions.append(cloud_execution)
+    session.commit()
+    return cloud_execution
+
+
+def get_case_id_from_cloud_execution_id(session, repository_id: int) -> Optional[str]:
+    cloud_execution = session.query(CloudExecution).filter(CloudExecution.repository_id == repository_id).first()
+    return cloud_execution.case_id if cloud_execution else None
+
+
+def get_case_id_from_repository_id(session, repository_id: int) -> Optional[str]:
+    cloud_execution = session.query(CloudExecution).filter(CloudExecution.repository_id == repository_id).first()
+    return cloud_execution.case_id if cloud_execution else None
+
+def get_repository_id_from_cloud_upload_id(session, cloud_upload_id: str) -> Optional[int]:
+    cloud_execution = session.query(CloudExecution).filter(CloudExecution.cloud_upload_id == cloud_upload_id).first()
+    return cloud_execution.repository_id if cloud_execution else None
+
+def get_repository_ids_from_case_id(session, case_id: str) -> List[int]:
+    cloud_executions = session.query(CloudExecution).filter(CloudExecution.case_id == case_id).all()
+    return [ce.repository_id for ce in cloud_executions]
+
+def get_runing_cloud_executions(session) -> List[CloudExecution]:
+    return session.query(CloudExecution).filter(CloudExecution.status == CloudStatus.RUNNING.value).all()
+
+def get_cloud_execution_status(session, repository_id: int) -> Optional[int]:
+    cloud_execution = session.query(CloudExecution).filter(CloudExecution.repository_id == repository_id).first()
+    return cloud_execution.status if cloud_execution else None
+
+def get_cloud_finished_executions(session) -> List[CloudExecution]:
+    return session.query(CloudExecution).filter(CloudExecution.status == CloudStatus.FINISHED.value).all()