logs-py 4.0.7__py3-none-any.whl

LOGS/Converter/Conversion.py

@@ -0,0 +1,248 @@
+import os
+from enum import Enum
+from time import sleep, time
+from typing import Any, Callable, Dict, List, Optional, cast
+
+from LOGS.Auxiliary.Exceptions import (
+    EntityFetchingException,
+    LOGSException,
+    UnfinishedConversionException,
+)
+from LOGS.Auxiliary.Tools import Tools
+from LOGS.Converter.ConverterParameter import ParameterType
+from LOGS.Entities.ParserLog import ParserLog
+from LOGS.Entity import Entity
+from LOGS.Entity.ConnectedEntity import ConnectedEntity
+from LOGS.Entity.SerializableContent import SerializableClass
+from LOGS.LOGSConnection import ResponseTypes
+
+
+class ConversionState(Enum):
+    Successful = "Successful"
+    Failed = "Failed"
+    Waiting = "Waiting"
+
+
+class ConversionFile(SerializableClass):
+    path: Optional[str] = None
+    size: Optional[int] = None
+
+
+class DatasetConversionZipReadModel(SerializableClass):
+    size: Optional[int] = None
+    id: Optional[str] = None
+    url: Optional[str] = None
+
+
+class ConverterParameterEntry(SerializableClass):
+    id: Optional[str] = None
+    value: Optional[Any] = None
+    type: Optional[ParameterType] = None
+
+
+class ConversionLogModel(ParserLog):
+    pass
+
+
+class Conversion(ConnectedEntity):
+    _datasetId: Optional[int] = None
+    _datasetFormat: Optional[str] = None
+    _exportFormat: Optional[str] = None
+    _files: Optional[List[ConversionFile]] = None
+    _logs: Optional[List[ConversionLogModel]] = None
+    _output: Optional[str] = None
+    _state: Optional[ConversionState] = None
+    _zip: Optional[DatasetConversionZipReadModel] = None
+    _inputParameters: Optional[List[ConverterParameterEntry]] = None
+
+    _payload: Dict[str, Any] = {}
+    _parentEntity: Optional[Entity] = None
+
+    def download(
+        self,
+        directory: Optional[str] = None,
+        fileName: Optional[str] = None,
+        overwrite=False,
+    ):
+        connection, _ = self._getConnectionData()
+
+        if self.state == ConversionState.Waiting:
+            raise UnfinishedConversionException(self)
+
+        if self.state == ConversionState.Failed:
+            raise LOGSException(
+                f"Conversion for dataset {self.datasetId} from format '{self.datasetFormat}' to format '{self.exportFormat}' failed. Check logs for more information."
+            )
+
+        if (
+            self.zip is None
+            or self.zip.size is None
+            or self.zip.id is None
+            or self.zip.url is None
+        ):
+            raise LOGSException(
+                f"Conversion for dataset {self.datasetId} from format '{self.datasetFormat}' to format '{self.exportFormat}' did not result in any data."
+            )
+
+        if not directory:
+            directory = os.curdir
+
+        if not fileName:
+            fileName = f"dataset_{self.datasetId}_{self.zip.id}.zip"
+
+        path = os.path.join(directory, Tools.sanitizeFileName(fileName=fileName))
+
+        if overwrite:
+            if os.path.exists(path) and not os.path.isfile(path):
+                raise LOGSException("Path %a is not a file" % path)
+        else:
+            if os.path.exists(path):
+                raise LOGSException("File %a already exists" % path)
+
+        data, responseError = connection.getUrl(
+            self.zip.url, responseType=ResponseTypes.RAW
+        )
+
+        if responseError:
+            raise LOGSException(
+                f"Could not fetch conversion result for dataset {self.datasetId} from format '{self.datasetFormat}' to format '{self.exportFormat}'.",
+                responseError=responseError,
+            )
+
+        with open(path, mode="wb") as localFile:
+            localFile.write(cast(bytes, data))
+
+        return path
+
+    def _reloadOnWaiting(self, connection, endpoint):
+        if self.state != ConversionState.Waiting:
+            return
+        data, responseError = connection.postEndpoint(endpoint, data=self._payload)
+        if responseError:
+            raise EntityFetchingException(
+                entity=self._parentEntity, responseError=responseError
+            )
+        self.fromDict(data)
+
+    def awaitResult(self):
+        connection, endpoint = self._getConnectionData()
+        if self.state != ConversionState.Waiting:
+            return
+
+        while self.state == ConversionState.Waiting:
+            sleep(0.5)
+            # print("request...")
+
+            self._reloadOnWaiting(connection, endpoint)
+
+            # print("... done", self.state)
+
+    @classmethod
+    def awaitAllResults(
+        cls,
+        conversions: List["Conversion"],
+        timeout=600,
+        stateChangeHook: Optional[Callable[[int], None]] = None,
+    ):
+        l = [c for c in conversions if c is not None]
+
+        connectionList = []
+        for i, c in enumerate(l):
+            connection, endpoint = c._getConnectionData()
+            connectionList.append((c, connection, endpoint))
+
+        count = sum(
+            [1 for c in connectionList if c[0].state != ConversionState.Waiting]
+        )
+        if stateChangeHook:
+            stateChangeHook(len(connectionList) - count)
+
+        start = time()
+        while count < len(connectionList) and time() - start < timeout:
+            oldCount = count
+            count = 0
+            for c in connectionList:
+                if c[0].state != ConversionState.Waiting:
+                    count += 1
+                    continue
+                c[0]._reloadOnWaiting(c[1], c[2])
+            if stateChangeHook and oldCount != count:
+                stateChangeHook(len(connectionList) - count)
+            # print(f"waiting for {len(connectionList) -count} jobs")
+            sleep(0.5)
+
+    @property
+    def datasetId(self) -> Optional[int]:
+        return self._datasetId
+
+    @datasetId.setter
+    def datasetId(self, value):
+        self._datasetId = self.checkAndConvertNullable(value, int, "datasetId")
+
+    @property
+    def datasetFormat(self) -> Optional[str]:
+        return self._datasetFormat
+
+    @datasetFormat.setter
+    def datasetFormat(self, value):
+        self._datasetFormat = self.checkAndConvertNullable(value, str, "datasetFormat")
+
+    @property
+    def exportFormat(self) -> Optional[str]:
+        return self._exportFormat
+
+    @exportFormat.setter
+    def exportFormat(self, value):
+        self._exportFormat = self.checkAndConvertNullable(value, str, "exportFormat")
+
+    @property
+    def files(self) -> Optional[List[ConversionFile]]:
+        return self._files
+
+    @files.setter
+    def files(self, value):
+        self._files = self.checkListAndConvertNullable(value, ConversionFile, "files")
+
+    @property
+    def logs(self) -> Optional[List[ConversionLogModel]]:
+        return self._logs
+
+    @logs.setter
+    def logs(self, value):
+        self._logs = self.checkListAndConvertNullable(value, ConversionLogModel, "logs")
+
+    @property
+    def output(self) -> Optional[str]:
+        return self._output
+
+    @output.setter
+    def output(self, value):
+        self._output = self.checkAndConvertNullable(value, str, "output")
+
+    @property
+    def state(self) -> Optional[ConversionState]:
+        return self._state
+
+    @state.setter
+    def state(self, value):
+        self._state = self.checkAndConvertNullable(value, ConversionState, "state")
+
+    @property
+    def zip(self) -> Optional[DatasetConversionZipReadModel]:
+        return self._zip
+
+    @zip.setter
+    def zip(self, value):
+        self._zip = self.checkAndConvertNullable(
+            value, DatasetConversionZipReadModel, "zip"
+        )
+
+    @property
+    def inputParameters(self) -> Optional[List[ConverterParameterEntry]]:
+        return self._inputParameters
+
+    @inputParameters.setter
+    def inputParameters(self, value):
+        self._inputParameters = self.checkListAndConvertNullable(
+            value, ConverterParameterEntry, "inputParameters"
+        )

LOGS/Converter/Converter.py

@@ -0,0 +1,96 @@
+from typing import Any, Dict, List, Optional
+
+from LOGS.Auxiliary import Tools
+from LOGS.Converter.ConverterParameter import IConverterParameter
+from LOGS.Converter.ExportParameters import ExportParameters
+from LOGS.Entity.SerializableContent import SerializableContent
+
+
+class Converter(SerializableContent):
+    _formatId: Optional[str] = None
+    _exportId: Optional[str] = None
+    _version: Optional[str] = None
+    _name: Optional[str] = None
+    _id: Optional[str] = None
+    _parameters: Optional[List[IConverterParameter]] = None
+
+    def __init__(self, ref=None):
+        t = type(self)
+
+        self._noSerialize += [
+            t.parameters.fget.__name__,  # type: ignore
+        ]
+
+        super().__init__(ref)
+
+    def __str__(self):
+        return Tools.ObjectToString(self)
+
+    @property
+    def formatId(self) -> Optional[str]:
+        return self._formatId
+
+    @formatId.setter
+    def formatId(self, value):
+        self._formatId = self.checkAndConvertNullable(value, str, "formatId")
+
+    @property
+    def exportId(self) -> Optional[str]:
+        return self._exportId
+
+    @exportId.setter
+    def exportId(self, value):
+        self._exportId = self.checkAndConvertNullable(value, str, "exportId")
+
+    @property
+    def version(self) -> Optional[str]:
+        return self._version
+
+    @version.setter
+    def version(self, value):
+        self._version = self.checkAndConvertNullable(value, str, "version")
+
+    @property
+    def name(self) -> Optional[str]:
+        return self._name
+
+    @name.setter
+    def name(self, value):
+        self._name = self.checkAndConvertNullable(value, str, "name")
+
+    @property
+    def id(self) -> Optional[str]:
+        return self._id
+
+    @id.setter
+    def id(self, value):
+        self._id = self.checkAndConvertNullable(value, str, "id")
+
+    @property
+    def requestParameter(self) -> Optional[ExportParameters]:
+        if self.parameters is None:
+            return None
+
+        p: Dict[str, Any] = {p.id: None for p in self.parameters if p.id}
+        p["_parentId"] = self.id
+        return ExportParameters(
+            ref=p,
+            types={
+                p.id: IConverterParameter.TypeMapper(p.type)
+                for p in self.parameters
+                if p.id
+            },
+        )
+
+    @property
+    def parameters(self) -> Optional[List[IConverterParameter]]:
+        return self._parameters
+
+    @parameters.setter
+    def parameters(self, value):
+        self._parameters = self.checkListAndConvertNullable(
+            value,
+            fieldType=IConverterParameter,
+            converter=IConverterParameter.GetParameterFromDict,
+            fieldName="parameters",
+        )

LOGS/Converter/ConverterParameter.py

@@ -0,0 +1,88 @@
+from enum import Enum
+from typing import Optional, Union
+
+from LOGS.Entity.SerializableContent import SerializableClass
+
+
+class ParameterType(Enum):
+    Integer = "Integer"
+    String = "String"
+    Boolean = "Boolean"
+    Float = "Float"
+
+
+class IConverterParameter(SerializableClass):
+    name: Optional[str] = None
+    id: Optional[str] = None
+    description: Optional[str] = None
+    required: Optional[bool] = None
+    hasDefault: Optional[bool] = None
+    identifier: Optional[str] = None
+
+    @classmethod
+    def TypeMapper(cls, t: Union[ParameterType, str]):
+        if t == ParameterType.Integer:
+            return int
+        if t == ParameterType.String:
+            return str
+        if t == ParameterType.Boolean:
+            return bool
+        if t == ParameterType.Float:
+            return float
+        raise Exception("Unknown parameter type '%s'" % t)
+
+    @classmethod
+    def GetParameterFromDict(cls, data: dict):
+        if not isinstance(data, dict) or "type" not in data:
+            raise Exception(f"Export parameter type not specified.")
+
+        type = data["type"]
+        if type not in ParameterType.__members__:
+            raise Exception(
+                f"Unknown export parameter type '{type}' not found in parameter data"
+            )
+
+        type = ParameterType.__members__[type]
+
+        if type == ParameterType.Integer:
+            return ConverterIntegerParameter(data)
+        if type == ParameterType.String:
+            return ConverterStringParameter(data)
+        if type == ParameterType.Boolean:
+            return ConverterBooleanParameter(data)
+        if type == ParameterType.Float:
+            return ConverterFloatParameter(data)
+
+        raise Exception(
+            f"Unknown export parameter type '{type}' not found in parameter data"
+        )
+
+    @property
+    def type(self) -> ParameterType:
+        raise NotImplementedError(
+            f"{type(self).__name__} is an abstract class. Please implement the 'type' property."
+        )
+
+
+class ConverterIntegerParameter(IConverterParameter):
+    @property
+    def type(self) -> ParameterType:
+        return ParameterType.Integer
+
+
+class ConverterStringParameter(IConverterParameter):
+    @property
+    def type(self) -> ParameterType:
+        return ParameterType.String
+
+
+class ConverterBooleanParameter(IConverterParameter):
+    @property
+    def type(self) -> ParameterType:
+        return ParameterType.Boolean
+
+
+class ConverterFloatParameter(IConverterParameter):
+    @property
+    def type(self) -> ParameterType:
+        return ParameterType.Float

LOGS/Converter/DateTimeRange.py

@@ -0,0 +1,58 @@
+from datetime import datetime
+from typing import Any, Dict, Optional, cast
+
+from LOGS.Auxiliary.Tools import Tools
+from LOGS.Entity.SerializableContent import SerializableContent
+
+
+class DateTimeRange(SerializableContent):
+    """Represents a range of two datetime values"""
+
+    _start: Optional[datetime] = None
+    _end: Optional[datetime] = None
+
+    def __init__(
+        self, ref=None, start: Optional[datetime] = None, end: Optional[datetime] = None
+    ):
+        if isinstance(ref, list):
+            ref = {
+                "start": (ref[0] if len(ref) > 0 else None),
+                "end": (ref[1] if len(ref) > 1 else None),
+            }
+        super().__init__(ref=ref)
+        if start is not None:
+            self.start = start
+        if end is not None:
+            self.end = end
+
+    def toDict(self) -> Dict[str, Any]:
+        return cast(
+            Dict[str, Any],
+            [
+                Tools.datetimeToUTCString(self.start),
+                Tools.datetimeToUTCString(self.end),
+            ],
+        )
+
+    def __str__(self) -> str:
+        return f"<DateTimeRange [{self.start}, {self.end}]>"
+
+    @property
+    def start(self) -> Optional[datetime]:
+        return self._start
+
+    @start.setter
+    def start(self, value: Optional[datetime]):
+        self._start = Tools.checkAndConvert(
+            value, datetime, "DateTimeRange.start", allowNone=True
+        )
+
+    @property
+    def end(self) -> Optional[datetime]:
+        return self._end
+
+    @end.setter
+    def end(self, value: Optional[datetime]):
+        self._end = Tools.checkAndConvert(
+            value, datetime, "DateTimeRange.end", allowNone=True
+        )

LOGS/Converter/ExportParameters.py

@@ -0,0 +1,89 @@
+from typing import Any, Dict, Optional
+
+from LOGS.Auxiliary.Tools import Tools
+from LOGS.Entity.SerializableContent import SerializableClass
+
+
+class ExportParameters(SerializableClass):
+
+    def __init__(
+        self,
+        ref=None,
+        types: Optional[Dict[str, type]] = None,
+    ):
+        self._lock = False
+        if types is not None:
+            if not isinstance(types, dict):
+                raise Exception("type must be a dictionary")
+            for k, v in types.items():
+                if not isinstance(v, type):
+                    raise Exception(
+                        f"Invalid type for key '{k}'. (Expected '{type.__name__}' got '{type(v).__name__}')"
+                    )
+        else:
+            types = {}
+
+        self._typeMapping: Dict[str, type] = {}
+        if isinstance(ref, dict):
+            if "_parentId" in ref:
+                self._parentId: Optional[str] = str(ref["_parentId"])
+                del ref["_parentId"]
+            else:
+                self._parentId = None
+
+            self._includeNone = True
+            for k in ref.keys():
+                setattr(self, k, ref[k])
+                if k in types:
+                    self._typeMapping[k] = types[k]
+        super().__init__(ref)
+        self._lock = True
+
+    def fromDict(self, ref) -> None:
+        if getattr(self, "_lock", False):
+            if isinstance(ref, dict):
+                if "parameters" in ref and isinstance(ref["parameters"], list):
+                    for p in ref["parameters"]:
+                        if "id" in p and "value" in p:
+                            setattr(self, p["id"], p["value"])
+                for k, v in ref.items():
+                    setattr(self, k, v)
+        return super().fromDict(ref)
+
+    def toDict(self) -> Dict[str, Any]:
+        l = [
+            {"id": k, "value": getattr(self, k)}
+            for k in self.__dict__
+            if k[0] != "_" and getattr(self, k) is not None
+        ]
+        if l:
+            return {"parameters": l}
+        return {}
+
+    def __repr__(self):
+        fields = [
+            f"{k}{':' + self._typeMapping[k].__name__ if k in self._typeMapping else ''}={v}"
+            for k, v in self.toDict().items()
+        ]
+        return f'{self.identifier}({", ".join(fields)})'
+
+    def __setattr__(self, name: str, value: Any) -> None:
+        if getattr(self, "_lock", False):
+            if name not in self._typeMapping:
+                raise Exception(f"Unknown field '{self.identifier}.{name}'")
+
+            if name in self._typeMapping:
+                value = Tools.checkAndConvert(
+                    value,
+                    self._typeMapping[name],
+                    f"{self.identifier}.{name}",
+                    allowNone=True,
+                )
+        return super().__setattr__(name, value)
+
+    @property
+    def identifier(self):
+        return f"{self.__class__.__name__}{'<' + self._parentId + '>' if self._parentId else ''}"
+
+    def __str__(self):
+        return self.__repr__()

LOGS/Converter/__init__.py

@@ -0,0 +1,13 @@
+"""
+This package is written for the LOGS Repository.
+
+It's objects can be used to write your own dataset parser.
+
+License:
+Permission to use this libraray and all of its contents is
+strictly permitted to the Signals Company.
+"""
+
+from .Conversion import *
+from .Converter import *
+from .ConverterParameter import *

LOGS/Entities/Attachment.py

@@ -0,0 +1,84 @@
+from typing import TYPE_CHECKING, List, Optional, Sequence
+
+from LOGS.Auxiliary.Constants import Constants
+from LOGS.Auxiliary.Decorators import Endpoint, UiEndpoint
+from LOGS.Auxiliary.Exceptions import LOGSException
+from LOGS.Entities.DatasetBase import DatasetBase
+from LOGS.Entities.DatasetModels import ViewableEntityTypes
+from LOGS.Entities.FileEntry import FileEntry
+from LOGS.Interfaces.IEntryRecord import IEntryRecord
+from LOGS.Interfaces.IModificationRecord import IModificationRecord
+from LOGS.Interfaces.IUniqueEntity import IUniqueEntity
+from LOGS.LOGSConnection import LOGSConnection
+
+if TYPE_CHECKING:
+    pass
+
+
+@Endpoint("datasets")
+@UiEndpoint("#data")
+class Attachment(
+    DatasetBase,
+    IUniqueEntity,
+    IEntryRecord,
+    IModificationRecord,
+):
+    _viewableEntityTypes: List[ViewableEntityTypes] = [ViewableEntityTypes.CustomField]
+    _isViewableEntity: bool = True
+
+    def __init__(
+        self,
+        ref=None,
+        id: Optional[int] = None,
+        connection: Optional[LOGSConnection] = None,
+        files: Optional[Sequence[Constants.FILE_TYPE]] = None,
+        pathPrefixToStrip: Optional[str] = None,
+        pathPrefixToAdd: Optional[str] = None,
+    ):
+        from LOGS.Entities.DataFormatMinimal import DataFormatMinimal
+
+        super().__init__(ref=ref, id=id, connection=connection)
+
+        t = type(self)
+        self._noSerialize += [
+            t.parserLogs.fget.__name__,  # type: ignore
+        ]
+        self._format = DataFormatMinimal(
+            id="doc_multi", name="Attachment format", connection=self._getConnection()
+        )
+
+        if files:
+            if not self._format or not self._format.id:
+                raise LOGSException(
+                    "Cannot create %s object from files parameter without a format"
+                    % type(self).__name__
+                )
+
+            self._files = FileEntry.entriesFromFiles(files)
+            if self._files is not None:
+                for file in self._files:
+                    if pathPrefixToStrip and file.path:
+                        file.modifyPathPrefix(pathPrefixToStrip, pathPrefixToAdd)
+
+    def fromDict(self, ref) -> None:
+        if isinstance(ref, dict):
+
+            infoFields = [
+                "parserLogs",
+                "parsingState",
+            ]
+
+            self._noInfo = not all(f in ref for f in infoFields)
+
+            info = {}
+            for field in infoFields:
+                if field in ref:
+                    info[field] = ref[field]
+                    del ref[field]
+
+        super().fromDict(ref=ref)
+        self._setInfo(info)
+
+    def fetchFull(self):
+        self.fetchInfo()
+        self.fetchZipSize()

LOGS/Entities/AttachmentMinimal.py

@@ -0,0 +1,8 @@
+from LOGS.Auxiliary.Decorators import FullModel
+from LOGS.Entities.Attachment import Attachment
+from LOGS.Entity.EntityMinimalWithIntId import EntityMinimalWithIntId
+
+
+@FullModel(Attachment)
+class AttachmentMinimal(EntityMinimalWithIntId[Attachment]):
+    pass