logs-py 4.0.7__py3-none-any.whl

LOGS/Entities/DatasetBase.py

@@ -0,0 +1,196 @@
+import os
+from typing import Any, Dict, List, Optional, cast
+
+from LOGS.Auxiliary.Exceptions import (
+    EntityFetchingException,
+    EntityIncompleteException,
+    LOGSException,
+)
+from LOGS.Auxiliary.Tools import Tools
+from LOGS.Entities.DataFormatMinimal import DataFormatMinimal
+from LOGS.Entities.DatasetInfo import DatasetInfo
+from LOGS.Entities.DatasetRequestParameter import ParsingStates
+from LOGS.Entities.Datatrack import Datatrack
+from LOGS.Entities.FileEntry import FileEntry
+from LOGS.Entities.HierarchyNode import HierarchyNode
+from LOGS.Entities.ParserLog import ParserLog
+from LOGS.Entities.Track import Track
+from LOGS.Entity.EntityWithIntId import IEntityWithIntId
+from LOGS.Interfaces.INamedEntity import INamedEntity
+from LOGS.LOGSConnection import ResponseTypes
+
+
+class DatasetBase(
+    IEntityWithIntId,
+    INamedEntity,
+):
+    # private attributes
+    _files: Optional[List[FileEntry]] = None
+
+    # state of additionally fetched data
+    _noInfo = True
+
+    # fields
+    _path: Optional[str] = None
+
+    # special fetched fields
+    ## Getter and setter implemented in inherited classes
+    _formatVersion: Optional[int] = None
+    _tracks: Optional[List[Track]] = None
+    _datatracks: Optional[List[Datatrack]] = None
+    _tracksHierarchy: Optional[HierarchyNode] = None
+
+    ## Getter and setter implemented in this class
+    _parsingState: Optional[ParsingStates] = None
+    _parserLogs: Optional[List[ParserLog]] = None
+    _zipSize: Optional[int] = None
+    _format: Optional["DataFormatMinimal"] = None
+
+    def fetchZipSize(self):
+        connection, endpoint, id = self._getConnectionData()
+
+        zip, responseError = connection.getEndpoint(
+            endpoint + ["zip_size"], parameters={"ids": [self.id]}
+        )
+        if responseError:
+            raise EntityFetchingException(entity=self, responseError=responseError)
+
+        if isinstance(zip, dict) and "size" in zip:
+            self._zipSize = zip["size"]
+
+    def _getDataDir(self):
+        if self.cacheDir:
+            if not os.path.isdir(self.cacheDir):
+                raise LOGSException(
+                    f"Specified cache directory '{self.cacheDir}' cannot be opened or is not a directory."
+                )
+            return self.cacheDir
+        return None
+
+    def fetchInfo(self):
+        connection, endpoint, id = self._getConnectionData()
+
+        data, responseError = connection.getEndpoint(endpoint + [id, "info"])
+        if responseError:
+            raise EntityFetchingException(entity=self, responseError=responseError)
+
+        dataDir = self._getDataDir()
+        if dataDir and not os.path.exists(dataDir):
+            os.mkdir(dataDir)
+
+        self._setInfo(cast(dict, data))
+        self._noInfo = False
+        if self._datatracks:
+            for datatrack in self._datatracks:
+                datatrack._endpoint = (
+                    endpoint + [str(id), "datatrack"] if endpoint else None
+                )
+
+    def _setInfo(self, data: dict):
+        info = DatasetInfo(data)
+        self._formatVersion = info.formatVersion
+        self._parserLogs = info.parserLogs
+        self._tracks = info.tracks
+        self._datatracks = info.datatracks
+        self._tracksHierarchy = info.tracksHierarchy
+        self._parsingState = info.parsingState
+
+        dataDir = self._getDataDir()
+
+        trackLookup: Dict[str, Datatrack] = {}
+        if self._datatracks:
+            for datatrack in self._datatracks:
+                datatrack._setConnection(self._getConnection())
+                datatrack.cacheDir = dataDir
+                if datatrack.id:
+                    trackLookup[datatrack.id] = datatrack
+
+        if self._tracks:
+            for track in self._tracks:
+                track._setConnection(self._getConnection())
+                track.cacheDir = dataDir
+                if track._dataIds:
+                    track.datatracks = cast(
+                        Any,
+                        {
+                            k: (trackLookup[v] if v in trackLookup else None)
+                            for k, v in track._dataIds.items()
+                        },
+                    )
+
+    def download(
+        self,
+        directory: Optional[str] = None,
+        fileName: Optional[str] = None,
+        overwrite=False,
+    ):
+        connection, endpoint, id = self._getConnectionData()
+
+        if not directory:
+            directory = os.curdir
+
+        if not fileName:
+            fileName = self.name if self.name and self.name != "" else "Dataset"
+            fileName += ".zip"
+
+        path = os.path.join(directory, Tools.sanitizeFileName(fileName=fileName))
+
+        if overwrite:
+            if os.path.exists(path) and not os.path.isfile(path):
+                raise LOGSException("Path %a is not a file" % path)
+        else:
+            if os.path.exists(path):
+                raise LOGSException("File %a already exists" % path)
+
+        data, responseError = connection.getEndpoint(
+            endpoint + [id, "files", "zip"], responseType=ResponseTypes.RAW
+        )
+        if responseError:
+            raise EntityFetchingException(entity=self, responseError=responseError)
+
+        with open(path, mode="wb") as localFile:
+            localFile.write(cast(bytes, data))
+
+        return path
+
+    @property
+    def format(self) -> Optional["DataFormatMinimal"]:
+        return self._format
+
+    @property
+    def path(self) -> Optional[str]:
+        return self._path
+
+    @path.setter
+    def path(self, value):
+        self._path = self.checkAndConvertNullable(value, str, "path")
+
+    @property
+    def parserLogs(self) -> Optional[List[ParserLog]]:
+        if self._noInfo:
+            raise EntityIncompleteException(
+                self,
+                parameterName="parserLogs",
+                functionName=f"{self.fetchInfo.__name__}()",
+            )
+        return self._parserLogs
+
+    @property
+    def parsingState(self) -> Optional[ParsingStates]:
+        return self._parsingState
+
+    @parsingState.setter
+    def parsingState(self, value):
+        self._parsingState = cast(
+            ParsingStates, self.checkAndConvertNullable(value, str, "parsingState")
+        )
+
+    @property
+    def zipSize(self) -> Optional[int]:
+        if self._zipSize is None:
+            raise EntityIncompleteException(
+                self,
+                parameterName="zipSize",
+                functionName=f"{self.fetchZipSize.__name__}()",
+            )
+        return self._zipSize

LOGS/Entities/DatasetCreator.py

@@ -0,0 +1,148 @@
+from typing import Any, Dict, List, Optional, Union
+
+from LOGS.Auxiliary.Decorators import Endpoint
+from LOGS.Auxiliary.Exceptions import EntityCreatingException, LOGSException
+from LOGS.Auxiliary.Tools import Tools
+from LOGS.Entities.Attachment import Attachment
+from LOGS.Entities.Dataset import Dataset
+from LOGS.Entities.DatasetModels import ViewableEntityTypes
+from LOGS.Entities.FileEntry import FileEntry
+from LOGS.Entity.ConnectedEntity import ConnectedEntity
+from LOGS.Entity.EntityConnector import EntityConnector
+from LOGS.LOGSConnection import LOGSConnection, MultipartEntry
+
+
+class DatasetUploadRequest(ConnectedEntity):
+    _typeMapper = {"files": FileEntry}
+    _dataset: Optional[Union[Dataset, Attachment]] = None
+
+    _viewableEntityTypes: Optional[List[ViewableEntityTypes]] = None
+    _isViewableEntity: Optional[bool] = None
+    _files: Optional[List[FileEntry]] = None
+    _filePathsAreAbsolute: Optional[bool] = True
+
+    def __init__(
+        self,
+        ref=None,
+        dataset: Optional[Union[Dataset, Attachment]] = None,
+        connection: Optional[LOGSConnection] = None,
+    ):
+        super().__init__(ref=ref, connection=connection)
+
+        if not dataset:
+            return
+        self._dataset = dataset
+
+        if not isinstance(dataset, (Dataset, Attachment)):
+            raise ValueError(
+                f"Dataset parameter must be of type Dataset or Attachment in the {type(self).__name__} constructor."
+            )
+        self._files = dataset._files
+        self._format = dataset._format
+        if isinstance(dataset, Attachment):
+            self._isViewableEntity = True
+            self._viewableEntityTypes = dataset._viewableEntityTypes
+
+    def toDict(self) -> Dict[str, Any]:
+        result = self._dataset.toDict() if self._dataset else {}
+        result.update(super().toDict())
+        return result
+
+    @property
+    def files(self) -> Optional[List[FileEntry]]:
+        return self._files
+
+    @files.setter
+    def files(self, value):
+        self._files = self.checkListAndConvertNullable(value, FileEntry, "files")
+
+    @property
+    def filePathsAreAbsolute(self) -> Optional[bool]:
+        return self._filePathsAreAbsolute
+
+    @filePathsAreAbsolute.setter
+    def filePathsAreAbsolute(self, value):
+        self._filePathsAreAbsolute = self.checkAndConvertNullable(
+            value, bool, "filePathsAreAbsolute"
+        )
+
+    @property
+    def isViewableEntity(self) -> Optional[bool]:
+        return self._isViewableEntity
+
+    @isViewableEntity.setter
+    def isViewableEntity(self, value):
+        self._isViewableEntity = self.checkAndConvertNullable(
+            value, bool, "isViewableEntity"
+        )
+
+    @property
+    def viewableEntityTypes(self) -> Optional[List[ViewableEntityTypes]]:
+        return self._viewableEntityTypes
+
+    @viewableEntityTypes.setter
+    def viewableEntityTypes(self, value):
+        self._viewableEntityTypes = self.checkListAndConvertNullable(
+            value, ViewableEntityTypes, "viewableEntityTypes"
+        )
+
+
+@Endpoint("datasets")
+class DatasetCreator(EntityConnector):
+    _request: DatasetUploadRequest = DatasetUploadRequest()
+    _formatId: str
+    _files: List[FileEntry]
+
+    def __init__(self, connection: LOGSConnection, dataset: Union[Dataset, Attachment]):
+        self._connection = connection
+
+        if not dataset:
+            raise LOGSException("Cannot not create empty dataset")
+        if not dataset._files:
+            raise LOGSException("Cannot not create dataset without files")
+        if not dataset.format or not dataset.format.id:
+            raise LOGSException("Cannot not create dataset without a format field")
+
+        self._formatId = dataset.format.id
+        self._files = dataset._files
+        dataset._setConnection(self._connection)
+        self._request = self._getDatasetUploadRequest(dataset=dataset)
+
+    def create(self):
+        connection, endpoint = self._getConnectionData()
+
+        multipart = [
+            MultipartEntry(
+                name="Dataset", fileName=None, content=self._request.toDict()
+            )
+        ]
+        multipart.extend(
+            [
+                MultipartEntry(name="files", fileName=file.id, content=file)
+                for file in self._files
+            ]
+        )
+
+        data, responseError = connection.postMultipartEndpoint(
+            endpoint=endpoint + ["create"], data=multipart
+        )
+        if responseError:
+            raise EntityCreatingException(responseError=responseError)
+
+        return Tools.checkAndConvert(data, dict, "dataset creation result")
+
+    def _getDatasetUploadRequest(self, dataset: Union[Dataset, Attachment]):
+        # print("\n".join([f.fullPath for f in fileList]))
+        if not self._files:
+            raise LOGSException("Cannot not create dataset without files")
+        if not self._formatId:
+            raise LOGSException("Cannot not create dataset without a formatId")
+
+        for file in self._files:
+            file.addMtime()
+
+        request = DatasetUploadRequest(
+            dataset=dataset, connection=self._getConnection()
+        )
+
+        return request

LOGS/Entities/DatasetInfo.py

@@ -0,0 +1,147 @@
+from typing import List, Optional, TypeVar, Union, cast
+
+from LOGS.Entities.DatasetRequestParameter import ParsingStates
+from LOGS.Entities.Datatrack import Datatrack
+from LOGS.Entities.DatatrackFormattedTable import DatatrackFormattedTable
+from LOGS.Entities.DatatrackGeneric import DatatrackGeneric
+from LOGS.Entities.DatatrackImage import DatatrackImage
+from LOGS.Entities.DatatrackNumericArray import DatatrackNumericArray
+from LOGS.Entities.DatatrackNumericMatrix import DatatrackNumericMatrix
+from LOGS.Entities.HierarchyNode import HierarchyNode
+from LOGS.Entities.ParserLog import ParserLog
+from LOGS.Entities.Track import Track
+from LOGS.Entities.TrackImage import TrackImage
+from LOGS.Entities.TrackMatrix import TrackMatrix
+from LOGS.Entities.TrackTable import TrackTable
+from LOGS.Entities.TrackXY import TrackXY
+from LOGS.Entities.TrackXYComplex import TrackXYComplex
+from LOGS.Entity.SerializableContent import SerializableContent
+
+TRACKS = Union[Track, TrackXY]
+DATATRACKS = Union[Datatrack, DatatrackNumericArray]
+_T = TypeVar("_T", Track, TrackXY)
+
+
+class DatasetInfo(SerializableContent):
+    _name: Optional[str] = None
+    _type: Optional[str] = None
+    _formatVersion: Optional[int] = None
+    _parsingState: Optional[ParsingStates] = None
+    _parserLogs: Optional[List[ParserLog]] = None
+    _tracks: Optional[List[TRACKS]] = None
+    _datatracks: Optional[List[DATATRACKS]] = None
+    _tracksHierarchy: Optional[HierarchyNode] = None
+
+    @property
+    def name(self) -> Optional[str]:
+        return self._name
+
+    @name.setter
+    def name(self, value):
+        self._name = self.checkAndConvertNullable(value, str, "name")
+
+    @property
+    def type(self) -> Optional[str]:
+        return self._type
+
+    @type.setter
+    def type(self, value):
+        self._type = self.checkAndConvertNullable(value, str, "type")
+
+    @property
+    def parsingState(self) -> Optional[ParsingStates]:
+        return self._parsingState
+
+    @parsingState.setter
+    def parsingState(self, value):
+        self._parsingState = cast(
+            ParsingStates, self.checkAndConvertNullable(value, str, "parsingState")
+        )
+
+    @property
+    def formatVersion(self) -> Optional[int]:
+        return self._formatVersion
+
+    @formatVersion.setter
+    def formatVersion(self, value):
+        self._formatVersion = self.checkAndConvertNullable(value, int, "formatVersion")
+
+    @property
+    def parserLogs(self) -> Optional[List[ParserLog]]:
+        return self._parserLogs
+
+    @parserLogs.setter
+    def parserLogs(self, value):
+        self._parserLogs = self.checkListAndConvertNullable(
+            value, ParserLog, "parserLogs"
+        )
+
+    @classmethod
+    def _trackConverter(cls, value: dict) -> TRACKS:
+        if isinstance(value, dict) and "type" in value:
+            # print("_trackConverter", value["type"])
+            if value["type"] == "XY_real":
+                return TrackXY(value)
+            elif value["type"] == "XY_complex":
+                return TrackXYComplex(value)
+            elif value["type"] == "matrix_real":
+                return TrackMatrix(value)
+            elif value["type"] == "image":
+                return TrackImage(value)
+            elif value["type"] == "table":
+                return TrackTable(value)
+            else:
+                return Track(value)
+        else:
+            return Track(value)
+
+    @classmethod
+    def _datatrackConverter(cls, value: dict) -> DATATRACKS:
+        if isinstance(value, dict) and "type" in value:
+            # print("_datatrackConverter", value["type"])
+            if value["type"] == "numeric_array":
+                return DatatrackNumericArray(value)
+            elif value["type"] == "numeric_matrix":
+                return DatatrackNumericMatrix(value)
+            elif value["type"] == "image":
+                return DatatrackImage(value)
+            elif value["type"] == "formatted_table":
+                return DatatrackFormattedTable(value)
+            else:
+                return DatatrackGeneric(value)
+        else:
+            return DatatrackGeneric(value)
+
+    @property
+    def tracks(self) -> Optional[List[Track]]:
+        return self._tracks
+
+    @tracks.setter
+    def tracks(self, value):
+        self._tracks = self.checkListAndConvertNullable(
+            value, Track, "tracks", converter=self._trackConverter
+        )
+
+    @property
+    def datatracks(self) -> Optional[List[Datatrack]]:
+        return self._datatracks
+
+    @datatracks.setter
+    def datatracks(self, value):
+        if value is None:
+            return None
+
+        if not isinstance(value, list):
+            value = [value]
+
+        self._datatracks = [self._datatrackConverter(v) for v in value]
+
+    @property
+    def tracksHierarchy(self) -> Optional[HierarchyNode]:
+        return self._tracksHierarchy
+
+    @tracksHierarchy.setter
+    def tracksHierarchy(self, value):
+        self._tracksHierarchy = self.checkAndConvertNullable(
+            value, HierarchyNode, "tracksHierarchy"
+        )

LOGS/Entities/DatasetMatchTypes.py

@@ -0,0 +1,157 @@
+import uuid
+from typing import Any, List, Optional
+
+from deprecation import deprecated  # type: ignore
+
+from LOGS.Auxiliary.Tools import Tools
+from LOGS.Entities.FileEntry import FileEntry, FileFragment
+from LOGS.Entity.SerializableContent import SerializableClass
+
+
+class MatchRequest(SerializableClass):
+    _typeMapper = {"files": FileEntry}
+
+    def __init__(self, ref: Any = None):
+        self.files: List[FileEntry] = []
+        self.formatIds: Optional[List[str]] = None
+        super().__init__(ref)
+
+
+class DatasetForSearch(SerializableClass):
+    _typeMapper = {"files": FileEntry}
+
+    def __init__(self, ref: Any = None):
+        self.id: str = ""
+        self.formatId: str = ""
+        self.checkUpdatable: bool = True
+        self.files: List[FileEntry] = []
+        super().__init__(ref)
+        self.id = uuid.uuid4().hex
+
+    @property
+    @deprecated(details="Please use property 'formatId'")
+    def parserId(self):
+        return self.formatId
+
+    @parserId.setter
+    @deprecated(details="Please use property 'formatId'")
+    def paserId(self, value):
+        self.formatId = value
+
+
+class DatasetsUpdatableFiles(SerializableClass):
+    _typeMapper = {"files": FileEntry}
+
+    def __init__(
+        self,
+        ref: Any = None,
+        datasetId: Optional[int] = None,
+        files: Optional[List[FileEntry]] = None,
+    ):
+        self.datasetId: int = 0
+        self.files: List[FileEntry] = []
+        super().__init__(ref)
+        if datasetId is not None:
+            self.datasetId = datasetId
+        if files is not None:
+            self.files = files
+
+
+class DatasetSearchResult(SerializableClass):
+    _typeMapper: dict = {"files": FileEntry}
+
+    def __init__(self, ref: Any = None):
+        self.id: str = ""
+        self.errors: List[str] = []
+        self.logsId: Optional[int] = None
+        self.isUpdateable: Optional[bool] = None
+        self.files: List[FileEntry] = []
+        super().__init__(ref)
+
+
+class DatasetSearchRequest(SerializableClass):
+    _typeMapper = {"datasets": DatasetForSearch}
+
+    def __init__(self, ref: Any = None):
+        self.datasets: List[DatasetForSearch] = []
+        super().__init__(ref)
+
+
+class MatchedDataset(SerializableClass):
+    _typeMapper = {"files": FileEntry}
+
+    def __init__(self, ref: Any = None):
+        self.id: str = ""
+        self.formatId: str = ""
+        self.name: str = ""
+        self.parentMissing: bool = False
+        self.parentPath: str = ""
+        self.files: List[FileEntry] = []
+        super().__init__(ref)
+
+    def __iter__(self):
+        for file in self.files:
+            yield file
+
+    def __str__(self):
+        if len(self.files):
+            return "<%s %a %s>" % (
+                type(self).__name__,
+                self.formatId,
+                Tools.numberPlural("file", len(self.files)),
+            )
+        else:
+            return "<%s>" % (type(self).__name__)
+
+    @property
+    @deprecated(details="Please use property 'formatId'")
+    def parserId(self):
+        return self.formatId
+
+    @parserId.setter
+    @deprecated(details="Please use property 'formatId'")
+    def paserId(self, value):
+        self.formatId = value
+
+
+class DatasetMatch(SerializableClass):
+    _typeMapper = {"datasets": MatchedDataset}
+
+    def __init__(self, ref: Any = None):
+        self.fromatId: str = ""
+        self.parserName: str = ""
+        self.datasets: List[MatchedDataset] = []
+        super().__init__(ref)
+
+    def __iter__(self):
+        for dataset in self.datasets:
+            yield dataset
+
+    def __str__(self):
+        if self.fromatId and len(self.datasets):
+            return "<%s %a(%d)>" % (
+                type(self).__name__,
+                self.fromatId,
+                len(self.datasets),
+            )
+        else:
+            return "<%s>" % (type(self).__name__)
+
+    @property
+    @deprecated(details="Please use property 'formatId'")
+    def parserId(self):
+        return self.formatId
+
+    @parserId.setter
+    @deprecated(details="Please use property 'formatId'")
+    def parserId(self, value):
+        self.formatId = value
+
+
+class MatchResult(SerializableClass):
+    _typeMapper = {"matches": DatasetMatch, "missingFragments": FileFragment}
+
+    def __init__(self, ref: Any = None):
+        self.matches: List[DatasetMatch] = []
+        self.missingFragments: List[FileFragment] = []
+        super().__init__(ref)