psr-factory 4.1.0b1__py3-none-win_amd64.whl → 4.1.0b2__py3-none-win_amd64.whl

psr/cloud/data.py

@@ -0,0 +1,105 @@
+# PSR Cloud. Copyright (C) PSR, Inc - All Rights Reserved
+# Unauthorized copying of this file, via any medium is strictly prohibited
+# Proprietary and confidential
+
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional, Tuple, Union
+
+
+class CloudInputError(ValueError):
+    """Raised when invalid input is provided."""
+
+    pass
+
+
+class CloudError(RuntimeError):
+    """Raised when case remote execution fails."""
+
+    pass
+
+
+class Case:
+    def __init__(
+        self,
+        name: str,
+        data_path: Optional[Union[str, Path]],
+        program: str,
+        program_version: Union[str, int],
+        execution_type: Union[str, int],
+        price_optimized: bool,
+        number_of_processes: int,
+        memory_per_process_ratio: str,
+        **kwargs,
+    ) -> None:
+        self.name: str = name
+        self._validate_type(self.name, str, "Case name must be a string")
+
+        self.data_path: str = str(data_path)
+        if data_path and not os.path.isabs(data_path):
+            self.data_path = os.path.abspath(data_path)
+        if data_path and not Path(data_path).exists():
+            raise CloudInputError("Data path does not exist")
+
+        self.program: str = program
+        self._validate_type(self.program, str, "Program must be a string")
+
+        self.program_version: Union[str, int] = program_version
+        self._validate_type(
+            self.program_version,
+            (int, str),
+            "Program version must be an integer or string (id or name)",
+        )
+
+        self.execution_type: Union[str, int] = execution_type
+        self._validate_type(
+            self.execution_type,
+            (int, str),
+            "Execution type must be an integer or string (id or name)",
+        )
+
+        self.price_optimized: bool = price_optimized
+        self._validate_type(
+            self.price_optimized, bool, "price_optimized must be a boolean"
+        )
+
+        self.number_of_processes: int = number_of_processes
+        self._validate_type(
+            self.number_of_processes, int, "Number of processes must be an integer"
+        )
+
+        self.memory_per_process_ratio: str = memory_per_process_ratio
+        self._validate_type(
+            self.memory_per_process_ratio,
+            str,
+            "Memory per process ratio must be a string",
+        )
+
+        self.repository_duration: Optional[Union[str, int]] = kwargs.get("repository_duration", 2)
+        self._validate_type(
+            self.repository_duration,
+            (int, str),
+            "Repository duration must be an integer or string (id or name)",
+        )
+
+        self.id: Optional[int] = kwargs.get("id", None)
+        self.user: Optional[str] = kwargs.get("user", None)
+        self.parent_case_id: Optional[int] = kwargs.get("parent_case_id", None)
+        self.execution_date: Optional[datetime] = kwargs.get("execution_date", None)
+        self.budget: Optional[str] = kwargs.get("budget", None)
+        if self.budget is not None:
+            self.budget = self.budget.strip()
+
+        # Save In Cloud
+        self.upload_only = kwargs.get("upload_only", False)
+
+    @staticmethod
+    def _validate_type(
+        value, expected_type: Union[List[type], Tuple[type], type], error_message: str
+    ):
+        if not isinstance(value, expected_type):
+            raise CloudInputError(error_message)
+
+    def __str__(self):
+        return str(self.__dict__)

psr/cloud/desktop.py

@@ -0,0 +1,82 @@
+# PSR Cloud. Copyright (C) PSR, Inc - All Rights Reserved
+# Unauthorized copying of this file, via any medium is strictly prohibited
+# Proprietary and confidential
+
+import os
+import xml.etree.ElementTree as ET
+from datetime import datetime
+from typing import Optional
+
+from .data import Case
+from .xml import create_desktop_xml
+
+
+def import_case(case: Case, console_cluster: str, instance_type_id: int) -> None:
+    case_counter = _get_last_case_id()
+    filepath = os.path.expandvars(
+        os.path.join(r"%appdata%\PSR\PSRCloud\Dados", f"Caso{case_counter}.xml")
+    )
+    now = datetime.now()
+    case_datetime = now.strftime("%d/%m/%Y %H:%M")
+    case_date = now.strftime("%d/%m/%Y")
+    case_lifetime = ""
+    case_budget = "" if case.budget is None else case.budget
+    repositorio_template = (
+        case.parent_case_id
+        if case.parent_case_id is not None and case.parent_case_id != 0
+        else ""
+    )
+
+    parameters = {
+        "Id": str(case_counter),
+        "IdRepositorio": str(case.id),
+        "IdFila": str(case.id),
+        "Modelo": case.program.upper(),
+        "RepositorioTemplate": repositorio_template,
+        "DirDados": case.data_path,
+        "TipoExecucao": str(case.execution_type),
+        "InstanciaTipo": str(instance_type_id),
+        "Cluster": console_cluster,
+        "ClusterSrv": console_cluster,
+        "DtX": case_datetime,
+        "NProc": str(case.number_of_processes),
+        "Versao": str(case.program_version),
+        "Status": "2",
+        "StatusX": "Executando",
+        "Budget": case_budget,
+        "PodeListarFila": "True",
+        "IdX": case.name,
+        "Nom": case.name,
+        "Tag": f"PyCloud\\{case_date.replace('/', '-')}",
+        "DuracaoRepositorio": case_lifetime,
+        "FlagMaqNormalS": "False",
+    }
+
+    with open(filepath, "w", encoding="utf-8") as f:
+        xml_contents = create_desktop_xml(parameters)
+        f.write(xml_contents)
+
+
+def _get_last_case_id() -> Optional[int]:
+    config_path = os.path.expandvars(r"%appdata%\PSR\PSRCloud\ePSRConfig.xml")
+    data_path = os.path.expandvars(r"%appdata%\PSR\PSRCloud\Dados")
+
+    if os.path.isfile(config_path):
+        xml = ET.parse(config_path, parser=ET.XMLParser(encoding="utf-16"))
+        root = xml.getroot()
+        last_case_id = None
+        for child in root.iter("Aplicacao"):
+            last_case_id = int(child.get("idUltimoCaso"))
+            break
+        if last_case_id is not None:
+            # Check for existing files with the same
+            # last_case_id. Increment it if necessary.
+            last_case_id = int(last_case_id)
+            files = os.listdir(data_path)
+            while f"Caso{last_case_id}.xml" in files:
+                last_case_id += 1
+            return last_case_id
+        # No case has been run yet
+        return None
+    else:
+        raise Exception("ERROR: PSR Cloud Desktop is not installed.")

psr/cloud/log.py

@@ -0,0 +1,40 @@
+# PSR Cloud. Copyright (C) PSR, Inc - All Rights Reserved
+# Unauthorized copying of this file, via any medium is strictly prohibited
+# Proprietary and confidential
+
+import logging
+import os
+
+
+def get_logger(
+    id: int, quiet: bool, debug_mode: bool, log_dir: str = os.getcwd()
+) -> logging.Logger:
+    logger = logging.getLogger(str(id))
+    logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter("%(asctime)s - %(message)s")
+
+    if debug_mode:
+        os.makedirs(log_dir, exist_ok=True)
+        file_handler = logging.FileHandler(f"{log_dir}/psr_cloud_console_{id}.log")
+        file_handler.setLevel(logging.DEBUG)
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
+
+    if not quiet:
+        console_handler = logging.StreamHandler()
+        if debug_mode:
+            console_handler.setLevel(logging.DEBUG)
+        else:
+            console_handler.setLevel(logging.INFO)
+        console_handler.setFormatter(formatter)
+        logger.addHandler(console_handler)
+
+    return logger
+
+
+def enable_log_timestamp(logger: logging.Logger, enable: bool) -> None:
+    for handler in logger.handlers:
+        if enable:
+            handler.setFormatter(logging.Formatter("%(asctime)s - %(message)s"))
+        else:
+            handler.setFormatter(logging.Formatter("%(message)s"))

psr/cloud/status.py

@@ -0,0 +1,81 @@
+# PSR Cloud. Copyright (C) PSR, Inc - All Rights Reserved
+# Unauthorized copying of this file, via any medium is strictly prohibited
+# Proprietary and confidential
+
+from enum import Enum
+
+
+class ExecutionStatus(Enum):
+    QUEUE = 0
+    PENDING = 1
+    RUNNING = 2
+    SUCCESS = 3
+    ERROR = 4
+    SYNC_ERROR = 5
+    WAITING_CANCEL = 6
+    CANCELLED = 7
+    NOT_IN_QUEUE = 8
+    CANCELLED_PENDING = 9
+    WAITING_MACHINE = 10
+    WAITING_SPOT = 11
+    SPOT_ACTIVE = 12
+    SPOT_CANCELLED = 13
+    SPOT_CANCELLED_PENDING = 14
+    SERVER_CANCELLED = 15
+    ABORTED = 16
+    WAITING_STOP = 17
+    STOPPED = 18
+    CANCELLED_IDLE = 19
+    WAITING_START = 20
+    WAITING_MACHINE_ON = 21
+    WAITING_ARCHITECTURE = 22
+    WARNING = 23
+    PARENT_CASE_FAILED = 24
+    RESTART_INTERRUPTED = 27
+    SPECIAL_WAITING = 30
+    WAITING_UPLOAD_QUEUE = 31
+    UPLOADED = 32
+
+
+FAULTY_TERMINATION_STATUS = [
+    ExecutionStatus.ERROR,
+    ExecutionStatus.SYNC_ERROR,
+    ExecutionStatus.CANCELLED,
+    ExecutionStatus.NOT_IN_QUEUE,
+    ExecutionStatus.ABORTED,
+    ExecutionStatus.CANCELLED_IDLE,
+    ExecutionStatus.RESTART_INTERRUPTED,
+]
+
+FINISHED_STATUS = FAULTY_TERMINATION_STATUS + [ExecutionStatus.SUCCESS]
+
+STATUS_MAP_TEXT = {
+    ExecutionStatus.QUEUE: "Item in the queue without machine allocations",
+    ExecutionStatus.PENDING: "Waiting to turn machines on",
+    ExecutionStatus.RUNNING: "Running",
+    ExecutionStatus.SUCCESS: "Finished successfully",
+    ExecutionStatus.ERROR: "Finished with errors",
+    ExecutionStatus.SYNC_ERROR: "Finished due to lack of update request in a synchronous execution",
+    ExecutionStatus.WAITING_CANCEL: "The cancellation was requested, but there is still no response from the queue processor",
+    ExecutionStatus.CANCELLED: "Cancelled process",
+    ExecutionStatus.NOT_IN_QUEUE: "Process is not in the queue",
+    ExecutionStatus.CANCELLED_PENDING: "The cancellation process has already been called, but there is still no response from the Process Handler to complete the cancellation",
+    ExecutionStatus.WAITING_MACHINE: "State where the queue is waiting for the machines to be connected to move to PENDING state",
+    ExecutionStatus.WAITING_SPOT: "Waiting for the spot offer to be made",
+    ExecutionStatus.SPOT_ACTIVE: "The bid is in active mode",
+    ExecutionStatus.SPOT_CANCELLED: "The platform requested cancellation, but there is still no response from the Queue Processor",
+    ExecutionStatus.SPOT_CANCELLED_PENDING: "The cancellation process has already been called by the server, but there is still no response from the Process Handler to complete the cancellation",
+    ExecutionStatus.SERVER_CANCELLED: "Cancelled process by the server",
+    ExecutionStatus.ABORTED: "The model runs more aborts due to lack of input files",
+    ExecutionStatus.WAITING_STOP: "Waiting for STOP by the queue processor",
+    ExecutionStatus.STOPPED: "STOP machines in the round are frozen",
+    ExecutionStatus.CANCELLED_IDLE: "Cancelled after being idle",
+    ExecutionStatus.WAITING_START: "Waiting for START by the queue processor",
+    ExecutionStatus.WAITING_MACHINE_ON: "Waiting for machine to turn on",
+    ExecutionStatus.WAITING_ARCHITECTURE: "Waiting for architecture change",
+    ExecutionStatus.WARNING: "Finished with warnings",
+    ExecutionStatus.PARENT_CASE_FAILED: "Parent case failed",
+    ExecutionStatus.SPECIAL_WAITING: "Special Waiting",
+    ExecutionStatus.WAITING_UPLOAD_QUEUE: "Waiting for uploading case",
+    ExecutionStatus.UPLOADED: "Uploaded and Saved in Cloud",
+}

psr/cloud/tempfile.py

@@ -0,0 +1,117 @@
+# PSR Cloud. Copyright (C) PSR, Inc - All Rights Reserved
+# Unauthorized copying of this file, via any medium is strictly prohibited
+# Proprietary and confidential
+
+import errno
+import io
+import os
+from random import Random
+
+
+class _RandomNameSequence:
+    """An instance of _RandomNameSequence generates an endless
+    sequence of unpredictable strings which can safely be incorporated
+    into file names.  Each string is eight characters long.  Multiple
+    threads can safely use the same instance at the same time.
+
+    _RandomNameSequence is an iterator."""
+
+    # Method extracted from tempfile Python's module.
+
+    characters = "abcdefghijklmnopqrstuvwxyz0123456789_"
+
+    @property
+    def rng(self):
+        cur_pid = os.getpid()
+        if cur_pid != getattr(self, "_rng_pid", None):
+            self._rng = Random()  # nosec
+            self._rng_pid = cur_pid
+        return self._rng
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        c = self.characters
+        choose = self.rng.choice
+        letters = [choose(c) for dummy in range(8)]
+        return "".join(letters)
+
+
+def _get_tempfile_name(base_path: str, prefix: str):
+    """Calculate the default directory to use for temporary files.
+    This routine should be called exactly once.
+
+    We determine whether a candidate temp dir is usable by
+    trying to create and write to a file in that directory.  If this
+    is successful, the test file is deleted.  To prevent denial of
+    service, the name of the test file must be randomized."""
+    # Method extracted from tempfile Python's module.
+
+    _text_openflags = os.O_RDWR | os.O_CREAT | os.O_EXCL
+    if hasattr(os, "O_NOFOLLOW"):
+        _text_openflags |= os.O_NOFOLLOW
+
+    _bin_openflags = _text_openflags
+    if hasattr(os, "O_BINARY"):
+        _bin_openflags |= os.O_BINARY
+
+    namer = _RandomNameSequence()
+
+    if base_path != os.curdir:
+        base_path = os.path.abspath(base_path)
+    # Try only a few names per directory.
+    for seq in range(100):
+        name = next(namer)
+        filename = os.path.join(base_path, prefix + name)
+        try:
+            fd = os.open(filename, _bin_openflags, 0o600)
+            try:
+                try:
+                    with io.open(fd, "wb", closefd=False) as fp:
+                        fp.write(b"blat")
+                finally:
+                    os.close(fd)
+            finally:
+                os.unlink(filename)
+            return filename
+        except FileExistsError:
+            pass
+        except PermissionError:
+            # This exception is thrown when a directory with the chosen name
+            # already exists on windows.
+            if (
+                os.name == "nt"
+                and os.path.isdir(base_path)
+                and os.access(base_path, os.W_OK)
+            ):
+                continue
+            break  # no point trying more names in this directory
+        except OSError:
+            break  # no point trying more names in this directory
+    raise FileNotFoundError(
+        errno.ENOENT, "No usable temporary file found in " % base_path
+    )
+
+
+class CreateTempFile:
+    def __init__(
+        self,
+        base_path: str,
+        prefix: str,
+        xml_content: str,
+        delete_temp_xml: bool = True,
+    ):
+        self.delete_temp_xml = delete_temp_xml
+        # get temp file name
+        self.xml_file_name = _get_tempfile_name(base_path, prefix) + ".xml"
+        self.xml_content = xml_content
+
+    def __enter__(self):
+        with open(self.xml_file_name, "w", encoding="utf-8-sig") as xml_file:
+            xml_file.write(self.xml_content)
+            return xml_file
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.delete_temp_xml:
+            os.remove(self.xml_file_name)

psr/cloud/version.py

@@ -0,0 +1,5 @@
+# PSR Cloud. Copyright (C) PSR, Inc - All Rights Reserved
+# Unauthorized copying of this file, via any medium is strictly prohibited
+# Proprietary and confidential
+
+__version__ = "0.3.0"

psr/cloud/xml.py

@@ -0,0 +1,55 @@
+# PSR Cloud. Copyright (C) PSR, Inc - All Rights Reserved
+# Unauthorized copying of this file, via any medium is strictly prohibited
+# Proprietary and confidential
+
+from typing import Any, Dict
+from xml.etree import ElementTree as ET
+
+
+def create_case_xml(parameters: Dict[str, Any]) -> str:
+    root = ET.Element("ColecaoParametro")
+    for name, value in parameters.items():
+        value = _handle_invalid_xml_chars(value)
+        parameter = ET.SubElement(root, "Parametro", nome=name, tipo="System.String")
+        parameter.text = value
+    ET.indent(root, "  ")
+    return ET.tostring(root, encoding="unicode", method="xml")
+
+
+def create_desktop_xml(parameters: Dict[str, Any]) -> str:
+    # use element tree to write the file contents instead
+    node = ET.Element("Repositorio")
+    case_node = ET.SubElement(node, "CasoOperacao")
+    for key, value in parameters.items():
+        value_escaped = _handle_invalid_xml_chars(value)
+        case_node.set(key, value_escaped)
+    tree = ET.ElementTree(node)
+    return ET.tostring(
+        tree.getroot(), encoding="unicode", method="xml", xml_declaration=False
+    )
+
+
+def _return_invalid_xml_chars(xml_content: str) -> str:
+    special_chars = {
+        "&": "&",
+        "&lt;": "<",
+        "&gt;": ">",
+        "&quot;": '"',
+        "&apos;": "'",
+    }
+    for char, replacement in special_chars.items():
+        xml_content = xml_content.replace(char, replacement)
+    return xml_content
+
+
+def _handle_invalid_xml_chars(xml_content: str) -> str:
+    special_chars = {
+        "&": "&amp;",
+        "<": "&lt;",
+        ">": "&gt;",
+        '"': "&quot;",
+        "'": "&apos;",
+    }
+    for char, replacement in special_chars.items():
+        xml_content = str(xml_content).replace(char, replacement)
+    return xml_content

psr/factory/__init__.py

@@ -2,6 +2,6 @@
 # Unauthorized copying of this file, via any medium is strictly prohibited
 # Proprietary and confidential
 
-__version__ = "4.1.0b1"
+__version__ = "4.1.0b2"
 
 from .api import *

psr/psrfcommon/__init__.py

@@ -0,0 +1,6 @@
+# PSR Factory. Copyright (C) PSR, Inc - All Rights Reserved
+# Unauthorized copying of this file, via any medium is strictly prohibited
+# Proprietary and confidential
+
+from .psrfcommon import *
+

psr/psrfcommon/psrfcommon.py

@@ -0,0 +1,54 @@
+# PSR Factory. Copyright (C) PSR, Inc - All Rights Reserved
+# Unauthorized copying of this file, via any medium is strictly prohibited
+# Proprietary and confidential
+
+import os
+import pathlib
+import subprocess
+import sys
+from contextlib import contextmanager
+from typing import Union, List
+
+
+@contextmanager
+def change_cwd(new_dir: Union[str, pathlib.Path]):
+    last_dir = os.getcwd()
+    os.chdir(new_dir)
+    try:
+        yield
+    finally:
+        os.chdir(last_dir)
+
+
+def exec_cmd(cmd: Union[str, List[str]], **kwargs) -> int:
+    dry_run = kwargs.get("dry_run", False)
+    print_progress = kwargs.get("show_progress", False)
+
+    if print_progress or dry_run:
+        sys.stdout.flush()
+
+    if dry_run:
+        if isinstance(cmd, list):
+            print(" ".join(cmd))
+        else:
+            print(cmd)
+        return_code = 0
+    else:
+        try:
+            return_code = subprocess.call(cmd, shell=True)
+            if return_code > 0:
+                raise RuntimeError(f"Execution error, code {return_code}")
+            else:
+                if print_progress:
+                    print("Execution success", return_code)
+        except OSError as e:
+            msg = f"Execution failed: {e}"
+            if print_progress:
+                print(msg, file=sys.stderr)
+            raise RuntimeError(msg)
+
+    if print_progress or dry_run:
+        sys.stdout.flush()
+    return return_code
+
+

psr/psrfcommon/tempfile.py

@@ -0,0 +1,118 @@
+# PSR Factory. Copyright (C) PSR, Inc - All Rights Reserved
+# Unauthorized copying of this file, via any medium is strictly prohibited
+# Proprietary and confidential
+
+import errno
+import io
+import os
+from random import Random
+
+
+class _RandomNameSequence:
+    """An instance of _RandomNameSequence generates an endless
+    sequence of unpredictable strings which can safely be incorporated
+    into file names.  Each string is eight characters long.  Multiple
+    threads can safely use the same instance at the same time.
+
+    _RandomNameSequence is an iterator."""
+
+    # Method extracted from tempfile Python's module.
+
+    characters = "abcdefghijklmnopqrstuvwxyz0123456789_"
+
+    @property
+    def rng(self):
+        cur_pid = os.getpid()
+        if cur_pid != getattr(self, "_rng_pid", None):
+            self._rng = Random()  # nosec
+            self._rng_pid = cur_pid
+        return self._rng
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        c = self.characters
+        choose = self.rng.choice
+        letters = [choose(c) for dummy in range(8)]
+        return "".join(letters)
+
+
+def _get_tempfile_name(base_path: str, prefix: str):
+    """Calculate the default directory to use for temporary files.
+    This routine should be called exactly once.
+
+    We determine whether a candidate temp dir is usable by
+    trying to create and write to a file in that directory.  If this
+    is successful, the test file is deleted.  To prevent denial of
+    service, the name of the test file must be randomized."""
+    # Method extracted from tempfile Python's module.
+
+    _text_openflags = os.O_RDWR | os.O_CREAT | os.O_EXCL
+    if hasattr(os, "O_NOFOLLOW"):
+        _text_openflags |= os.O_NOFOLLOW
+
+    _bin_openflags = _text_openflags
+    if hasattr(os, "O_BINARY"):
+        _bin_openflags |= os.O_BINARY
+
+    namer = _RandomNameSequence()
+
+    if base_path != os.curdir:
+        base_path = os.path.abspath(base_path)
+    # Try only a few names per directory.
+    for seq in range(100):
+        name = next(namer)
+        filename = os.path.join(base_path, prefix + name)
+        try:
+            fd = os.open(filename, _bin_openflags, 0o600)
+            try:
+                try:
+                    with io.open(fd, "wb", closefd=False) as fp:
+                        fp.write(b"blat")
+                finally:
+                    os.close(fd)
+            finally:
+                os.unlink(filename)
+            return filename
+        except FileExistsError:
+            pass
+        except PermissionError:
+            # This exception is thrown when a directory with the chosen name
+            # already exists on windows.
+            if (
+                os.name == "nt"
+                and os.path.isdir(base_path)
+                and os.access(base_path, os.W_OK)
+            ):
+                continue
+            break  # no point trying more names in this directory
+        except OSError:
+            break  # no point trying more names in this directory
+    raise FileNotFoundError(
+        errno.ENOENT, "No usable temporary file found in " % base_path
+    )
+
+
+class CreateTempFile:
+    def __init__(
+        self,
+        base_path: str,
+        prefix: str,
+        file_content: str,
+        extension: str = ".dat",
+        delete_tempfile: bool = True,
+    ):
+        self.delete_tempfile = delete_tempfile
+        # get temp file name
+        self.temp_file_name = _get_tempfile_name(base_path, prefix) + extension
+        self.temp_content = file_content
+
+    def __enter__(self):
+        with open(self.temp_file_name, "w", encoding="utf-8-sig") as tempfile:
+            tempfile.write(self.temp_content)
+            return tempfile
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.delete_tempfile:
+            os.remove(self.temp_file_name)

psr/runner/__init__.py

@@ -0,0 +1,7 @@
+# PSR Factory. Copyright (C) PSR, Inc - All Rights Reserved
+# Unauthorized copying of this file, via any medium is strictly prohibited
+# Proprietary and confidential
+
+from .version import __version__
+from .runner import *
+