logs-py 2.9.6__py3-none-any.whl → 3.0.1__py3-none-any.whl

LOGS/Entities/Dataset.py

@@ -1,4 +1,5 @@
 import os
+from dataclasses import dataclass
 from datetime import datetime
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Union, cast
 
@@ -13,7 +14,6 @@ from LOGS.Auxiliary.Exceptions import (
 )
 from LOGS.Auxiliary.MinimalModelGenerator import (
     BridgeMinimalFromDict,
-    DatasetTypeMinimalFromDict,
     ExperimentMinimalFromDict,
     FormatMinimalFromDict,
     InstrumentMinimalFromDict,
@@ -23,26 +23,33 @@ from LOGS.Auxiliary.MinimalModelGenerator import (
 )
 from LOGS.Auxiliary.ParameterHelper import ParameterHelper
 from LOGS.Auxiliary.Tools import Tools
+from LOGS.Converter import Converter
+from LOGS.Converter.Conversion import Conversion
+from LOGS.Converter.ExportParamters import ExportParamters
 from LOGS.Entities.DatasetInfo import DatasetInfo
+from LOGS.Entities.DatasetModels import DatasetSourceType, ViewableEntityTypes
 from LOGS.Entities.DatasetRelations import DatasetRelations
 from LOGS.Entities.DatasetRequestParameter import ParsingStates
-from LOGS.Entities.DatasetTypeMinimal import DatasetTypeMinimal
 from LOGS.Entities.Datatrack import Datatrack
 from LOGS.Entities.FileEntry import FileEntry
 from LOGS.Entities.HierarchyNode import HierarchyNode
 from LOGS.Entities.ParserLog import ParserLog
 from LOGS.Entities.Track import Track
+from LOGS.Entity.EntityMinimalWithIntId import EntityMinimalWithIntId
 from LOGS.Entity.EntityWithIntId import IEntityWithIntId
 from LOGS.Entity.SerializeableContent import SerializeableContent
+from LOGS.Interfaces.ICreationRecord import ICreationRecord
+from LOGS.Interfaces.IModificationRecord import IModificationRecord
 from LOGS.Interfaces.INamedEntity import INamedEntity
 from LOGS.Interfaces.IOwnedEntity import IOwnedEntity
-from LOGS.Interfaces.IPermissionedEntity import IPermissionedEntity
+from LOGS.Interfaces.IPermissionedEntity import GenericPermissionEntity
 from LOGS.Interfaces.IProjectBased import IProjectBased
 from LOGS.Interfaces.IRelatedEntity import IRelatedEntity
 from LOGS.Interfaces.ISoftDeletable import ISoftDeletable
 from LOGS.Interfaces.ITypedEntity import ITypedEntity
 from LOGS.Interfaces.IUniqueEntity import IUniqueEntity
 from LOGS.LOGSConnection import LOGSConnection, ResponseTypes
+from LOGS.Parameters.ParameterList import ParameterList
 
 if TYPE_CHECKING:
     from LOGS.Entities.BridgeMinimal import BridgeMinimal
@@ -52,7 +59,6 @@ if TYPE_CHECKING:
     from LOGS.Entities.InstrumentMinimal import InstrumentMinimal
     from LOGS.Entities.MethodMinimal import MethodMinimal
     from LOGS.Entities.PersonMinimal import PersonMinimal
-    from LOGS.Entities.ProjectMinimal import ProjectMinimal
     from LOGS.Entities.SampleMinimal import SampleMinimal
 
 
@@ -63,54 +69,69 @@ class ParsedMetadata(SerializeableContent):
     TrackViewerTypes: List[str] = []
 
 
+@dataclass
+class DatasetSource:
+    id: Optional[int] = None
+    type: Optional[DatasetSourceType] = None
+    name: Optional[str] = None
+
+
 @Endpoint("datasets")
 @UiEndpoint("#data")
 class Dataset(
+    IEntityWithIntId,
+    GenericPermissionEntity,
     INamedEntity,
-    IProjectBased,
     IOwnedEntity,
-    IEntityWithIntId,
+    IProjectBased,
     IRelatedEntity[DatasetRelations],
-    ITypedEntity,
     ISoftDeletable,
+    ITypedEntity,
     IUniqueEntity,
-    IPermissionedEntity,
+    ICreationRecord,
+    IModificationRecord,
 ):
     _noInfo = True
     _noParameters = True
-    _relationType = type(DatasetRelations)
-
-    _legacyId: Optional[str]
-    _type: Optional[DatasetTypeMinimal]
-    _format: Optional["FormatMinimal"]
-    _acquisitionDate: Optional[datetime]
-    _path: Optional[str]
-    _sourceBaseDirectory: Optional[str]
-    _sourceRelativeDirectory: Optional[str]
-    _method: Optional["MethodMinimal"]
-    _experiment: Optional["ExperimentMinimal"]
-    _claimed: Optional[bool]
-    _notes: Optional[str]
-    _dateAdded: Optional[datetime]
-    _isViewableEntity: Optional[bool]
-    _other: Optional[str]
-    _instrument: Optional["InstrumentMinimal"]
-    _sample: Optional["SampleMinimal"]
-    _bridge: Optional["BridgeMinimal"]
-    _operators: Optional[List["PersonMinimal"]]
-    _equipments: Optional[List["EquipmentMinimal"]]
-    _projects: Optional[List["ProjectMinimal"]]
-    _parsingState: Optional[ParsingStates]
-    _parsedMetadata: Optional[ParsedMetadata]
-    _parameters: Optional[Dict[str, Any]]
-    _formatVersion: Optional[int]
-    _parserLogs: Optional[List[ParserLog]]
-    _tracks: Optional[List[Track]]
-    _datatracks: Optional[List[Datatrack]]
-    _tracksHierarchy: Optional[HierarchyNode]
-    _files: Optional[List[FileEntry]]
-    _parameterHelper: Optional[ParameterHelper]
-    _zipSize: Optional[int]
+    _noExports = True
+    _noParameterTree = True
+    _relationType = DatasetRelations
+
+    _acquisitionDate: Optional[datetime] = None
+    _automaticName: Optional[str] = None
+    _bridge: Optional["BridgeMinimal"] = None
+    _claimed: Optional[bool] = None
+    _customImport: Optional[EntityMinimalWithIntId] = None
+    _datatracks: Optional[List[Datatrack]] = None
+    _equipments: Optional[List["EquipmentMinimal"]] = None
+    _experiment: Optional["ExperimentMinimal"] = None
+    _exports: Optional[List[Converter]] = None
+    _files: Optional[List[FileEntry]] = None
+    _format: Optional["FormatMinimal"] = None
+    _formatVersion: Optional[int] = None
+    _instrument: Optional["InstrumentMinimal"] = None
+    _isManuallyNamed: Optional[bool] = None
+    _isViewableEntity: Optional[bool] = None
+    _legacyId: Optional[str] = None
+    _method: Optional["MethodMinimal"] = None
+    _notes: Optional[str] = None
+    _operators: Optional[List["PersonMinimal"]] = None
+    _other: Optional[str] = None
+    _parameterHelper: Optional[ParameterHelper] = None
+    _parameters: Optional[Dict[str, Any]] = None
+    _parameterTree: Optional[ParameterList] = None
+    _parsedMetadata: Optional[ParsedMetadata] = None
+    _parserLogs: Optional[List[ParserLog]] = None
+    _parsingState: Optional[ParsingStates] = None
+    _path: Optional[str] = None
+    _sample: Optional["SampleMinimal"] = None
+    _source: Optional[DatasetSource] = None
+    _sourceBaseDirectory: Optional[str] = None
+    _sourceRelativeDirectory: Optional[str] = None
+    _tracks: Optional[List[Track]] = None
+    _tracksHierarchy: Optional[HierarchyNode] = None
+    _viewableEntityType: Optional[ViewableEntityTypes] = None
+    _zipSize: Optional[int] = None
 
     def __init__(
         self,
@@ -120,45 +141,18 @@ class Dataset(
         files: Optional[Sequence[Constants.FILE_TYPE]] = None,
         format: Optional[Union[str, "FormatMinimal"]] = None,
     ):
-        self._legacyId = None
-        self._type = None
-        self._format = None
-        self._acquisitionDate = None
-        self._path = None
-        self._method = None
-        self._experiment = None
-        self._claimed = None
-        self._notes = None
-        self._dateAdded = None
-        self._isViewableEntity = None
-        self._isDeleted = None
-        self._other = None
-        self._instrument = None
-        self._sample = None
-        self._bridge = None
-        self._operators = None
-        self._equipments = None
-        self._projects = None
-        self._parsingState = None
-        self._parsedMetadata = None
-        self._parameters = None
-        self._formatVersion = None
-        self._parserLogs = None
-        self._tracks = None
-        self._datatracks = None
-        self._tracksHierarchy = None
-        self._files = None
-        self._parameterHelper = None
-        self._zipSize = None
-
         super().__init__(ref=ref, id=id, connection=connection)
+
+        t = type(self)
         self._noSerialize += [
-            "parameters",
-            "formatVersion",
-            "parserLogs",
-            "tracks",
-            "datatracks",
-            "tracksHierarchy",
+            t.parameters.fget.__name__,  # type: ignore
+            t.parameterTree.fget.__name__,  # type: ignore
+            t.formatVersion.fget.__name__,  # type: ignore
+            t.parserLogs.fget.__name__,  # type: ignore
+            t.tracks.fget.__name__,  # type: ignore
+            t.datatracks.fget.__name__,  # type: ignore
+            t.tracksHierarchy.fget.__name__,  # type: ignore
+            t.exports.fget.__name__,  # type: ignore
         ]
 
         if isinstance(ref, Dataset):
@@ -183,34 +177,31 @@ class Dataset(
                     ref["parameters"], dict, "parameters"
                 )
                 self._noParameters = False
-                self._noParameters = False
-            if "formatVersion" in ref:
-                self._formatVersion = self.checkAndConvertNullable(
-                    ref["formatVersion"], int, "formatVersion"
-                )
-                self._noInfo = False
-            if "parserLogs" in ref:
-                self._parserLogs = self.checkListAndConvertNullable(
-                    ref["parserLogs"], ParserLog, "parserLogs"
+            if "parameterTree" in ref:
+                self._parameterTree = self.checkAndConvertNullable(
+                    ref["parameterTree"], ParameterList, "parameters"
                 )
-                self._noInfo = False
-            if "tracks" in ref:
-                self._tracks = self.checkListAndConvertNullable(
-                    ref["tracks"], Track, "tracks"
-                )
-                self._noInfo = False
-            if "datatracks" in ref:
-                self._datatracks = self.checkListAndConvertNullable(
-                    ref["datatracks"], Datatrack, "datatracks"
-                )
-                self._noInfo = False
-            if "tracksHierarchy" in ref:
-                self._tracksHierarchy = self.checkAndConvertNullable(
-                    ref["tracksHierarchy"], HierarchyNode, "tracksHierarchy"
-                )
-                self._noInfo = False
+                self._noParameterTree = False
+
+            infoFields = [
+                "formatVersion",
+                "parserLogs",
+                "tracks",
+                "datatracks",
+                "tracksHierarchy",
+                "parsingState",
+            ]
+
+            self._noInfo = not all(f in ref for f in infoFields)
+
+            info = {}
+            for field in infoFields:
+                if field in ref:
+                    info[field] = ref[field]
+                    del ref[field]
 
         super().fromDict(ref=ref)
+        self._setInfo(info)
 
     def fetchZipSize(self):
         connection, endpoint, id = self._getConnectionData()
@@ -243,13 +234,48 @@ class Dataset(
         self._parameterHelper = ParameterHelper(self._parameters)
         self._noParameters = False
 
-    def fetchInfo(self: "Dataset"):
+    def fetchParameterTree(self):
         connection, endpoint, id = self._getConnectionData()
 
-        data, responseError = connection.getEndpoint(endpoint + [id, "info"])
+        parameters, responseError = connection.getEndpoint(
+            endpoint + [id, "parameter_tree"]
+        )
+        if responseError:
+            raise EntityFetchingException(entity=self, responseError=responseError)
+
+        self._parameterTree = self.checkAndConvertNullable(
+            parameters, ParameterList, "parameterTree"
+        )
+
+        self._noParameterTree = False
+
+    def fetchExports(self):
+        connection, endpoint, id = self._getConnectionData()
+
+        exports, responseError = connection.getEndpoint(endpoint + [id, "exports"])
         if responseError:
             raise EntityFetchingException(entity=self, responseError=responseError)
 
+        self.exports = exports
+        self._noExports = False
+
+    def _getDataDir(self):
+        if self.cacheDir:
+            if not os.path.isdir(self.cacheDir):
+                raise LOGSException(
+                    f"Specified cache directory '{self.cacheDir}' cannot be opened or is not a directory."
+                )
+            return self.cacheDir
+        return None
+
+    def clearCache(self):
+        dataDir = self._getDataDir()
+        if dataDir and os.path.exists(dataDir) and self.datatracks:
+            for datatrack in self.datatracks:
+                datatrack.clearCache()
+            os.rmdir(dataDir)
+
+    def _setInfo(self, data: dict):
         info = DatasetInfo(data)
         self._formatVersion = info.formatVersion
         self._parserLogs = info.parserLogs
@@ -258,17 +284,20 @@ class Dataset(
         self._tracksHierarchy = info.tracksHierarchy
         self._parsingState = info.parsingState
 
+        dataDir = self._getDataDir()
+
         trackLookup: Dict[str, Datatrack] = {}
         if self._datatracks:
             for datatrack in self._datatracks:
                 datatrack.connection = self.connection
-                datatrack._endpoint = endpoint + [str(id), "datatrack"]
+                datatrack.cacheDir = dataDir
                 if datatrack.id:
                     trackLookup[datatrack.id] = datatrack
 
         if self._tracks:
             for track in self._tracks:
                 track.connection = self.connection
+                track.cacheDir = dataDir
                 if track._dataIds:
                     track.datatracks = cast(
                         Any,
@@ -277,12 +306,31 @@ class Dataset(
                             for k, v in track._dataIds.items()
                         },
                     )
+
+    def fetchInfo(self):
+        connection, endpoint, id = self._getConnectionData()
+
+        data, responseError = connection.getEndpoint(endpoint + [id, "info"])
+        if responseError:
+            raise EntityFetchingException(entity=self, responseError=responseError)
+
+        dataDir = self._getDataDir()
+        if dataDir and not os.path.exists(dataDir):
+            os.mkdir(dataDir)
+
+        self._setInfo(cast(dict, data))
         self._noInfo = False
+        if self._datatracks:
+            for datatrack in self._datatracks:
+                datatrack._endpoint = (
+                    endpoint + [str(id), "datatrack"] if endpoint else None
+                )
 
     def fetchFull(self):
         self.fetchParameters()
         self.fetchInfo()
         self.fetchZipSize()
+        self.fetchExports()
 
     def download(
         self,
@@ -324,15 +372,54 @@ class Dataset(
             self._parameterHelper = ParameterHelper(self.parameters)
         return self._parameterHelper.get(key, removeUnit)
 
-    @property
-    def type(self) -> Optional[DatasetTypeMinimal]:
-        return self._type
+    def _requestReport(self, exportId: str, parameters: Optional[ExportParamters]):
+        connection, endpoint, id = self._getConnectionData()
+        converterEndpoint: Any = endpoint + [id, "exports", exportId]
+        payload = parameters.toDict() if parameters else {}
+        data, responseError = connection.postEndpoint(converterEndpoint, data=payload)
+        if responseError:
+            raise EntityFetchingException(entity=self, responseError=responseError)
 
-    @type.setter
-    def type(self, value):
-        self._type = DatasetTypeMinimalFromDict(
-            value, "datasetType", connection=self.connection
-        )
+        # TODO: create a report type to wait for the report to be generated
+        # TODO: maybe a class "Conversion" can be created that has a state and also and automatic awaiter function or so
+        conversion = self.checkAndConvert(data, Conversion, f"Conversion_to_{exportId}")
+        conversion.connection = self.connection
+        conversion._endpoint = converterEndpoint
+        conversion._payload = payload
+        conversion._parentEntity = self
+        return conversion
+
+    def exportTo(
+        self, exportId: str, parameters: Optional[Union[ExportParamters, dict]] = None
+    ):
+
+        if self._noExports:
+            self.fetchExports()
+
+        if self.exports is None:
+            raise LOGSException(f"Export id '{exportId}' not found in exports")
+
+        exports = {e.exportId: e for e in self.exports}
+        exports.update({e.id: e for e in self.exports})
+        if exportId not in exports:
+            raise LOGSException(f"Export id '{exportId}' not found in exports")
+
+        export = exports[exportId]
+        p = export.requestParameter
+        if parameters is not None and p is not None:
+            if isinstance(parameters, dict):
+                p.fromDict(parameters)
+            elif isinstance(parameters, ExportParamters):
+                if parameters._parentId is None or parameters._parentId != p._parentId:
+                    raise LOGSException(
+                        f"The passed export parameters is not generated by and valid export format. (Expected class '{p.identifier}')"
+                    )
+            else:
+                raise LOGSException(
+                    f"Invalid parameter type '{type(parameters).__name__}'. (Expected 'dict' or '{ExportParamters.__name__}')"
+                )
+
+        return self._requestReport(exportId, p)
 
     @property
     def format(self) -> Optional["FormatMinimal"]:
@@ -432,6 +519,17 @@ class Dataset(
             )
         return self._parameters
 
+    @property
+    def parameterTree(self) -> Optional[ParameterList]:
+        if self._noParameterTree:
+            raise EntityIncompleteException(
+                self,
+                parameterName="parameterTree",
+                functionName=f"{self.fetchParameterTree.__name__}()",
+                hasFetchFull=False,
+            )
+        return self._parameterTree
+
     @property
     def formatVersion(self) -> Optional[int]:
         if self._noInfo:
@@ -600,28 +698,6 @@ class Dataset(
             value, "PersonMinimal", "operators", connection=self.connection
         )
 
-    @property
-    def projects(self) -> Optional[List["ProjectMinimal"]]:
-        return self._projects
-
-    @projects.setter
-    def projects(self, value):
-        self._projects = MinimalFromList(
-            value, "ProjectMinimal", "projects", connection=self.connection
-        )
-
-    @property
-    def projectIds(self) -> Optional[List[int]]:
-        if self._projects is None:
-            return None
-        return [e.id for e in self._projects]
-
-    @projectIds.setter
-    def projectIds(self, value):
-        self._projects = MinimalFromList(
-            value, "ProjectMinimal", "projects", connection=self.connection
-        )
-
     @property
     def instrument(self) -> Optional["InstrumentMinimal"]:
         return self._instrument
@@ -681,3 +757,65 @@ class Dataset(
         self._isViewableEntity = self.checkAndConvertNullable(
             value, bool, "isViewableEntity"
         )
+
+    @property
+    def exports(self) -> Optional[List[Converter]]:
+        if self._noExports:
+            raise EntityIncompleteException(
+                self,
+                parameterName="exports",
+                functionName=f"{self.fetchExports.__name__}()",
+                hasFetchFull=True,
+            )
+
+        return self._exports
+
+    @exports.setter
+    def exports(self, value):
+        self._exports = self.checkListAndConvertNullable(value, Converter, "exports")
+
+    @property
+    def viewableEntityType(self) -> Optional[ViewableEntityTypes]:
+        return self._viewableEntityType
+
+    @viewableEntityType.setter
+    def viewableEntityType(self, value):
+        self._viewableEntityType = self.checkAndConvertNullable(
+            value, ViewableEntityTypes, "viewableEntityType"
+        )
+
+    @property
+    def customImport(self) -> Optional[EntityMinimalWithIntId]:
+        return self._customImport
+
+    @customImport.setter
+    def customImport(self, value):
+        self._customImport = self.checkAndConvertNullable(
+            value, EntityMinimalWithIntId, "customImport"
+        )
+
+    @property
+    def source(self) -> Optional[DatasetSource]:
+        return self._source
+
+    @source.setter
+    def source(self, value):
+        self._source = self.checkAndConvertNullable(value, DatasetSource, "source")
+
+    @property
+    def isManuallyNamed(self) -> Optional[bool]:
+        return self._isManuallyNamed
+
+    @isManuallyNamed.setter
+    def isManuallyNamed(self, value):
+        self._isManuallyNamed = self.checkAndConvertNullable(
+            value, bool, "isManuallyNamed"
+        )
+
+    @property
+    def automaticName(self) -> Optional[str]:
+        return self._automaticName
+
+    @automaticName.setter
+    def automaticName(self, value):
+        self._automaticName = self.checkAndConvertNullable(value, str, "automaticName")

LOGS/Entities/DatasetCreator.py

@@ -1,7 +1,4 @@
-from enum import Enum
-from typing import TYPE_CHECKING, Any, List, Optional
-
-from deprecation import deprecated
+from typing import TYPE_CHECKING, List, Optional
 
 from LOGS.Auxiliary.Decorators import Endpoint
 from LOGS.Auxiliary.Exceptions import EntityCreatingException, LOGSException
@@ -9,86 +6,40 @@ from LOGS.Auxiliary.Tools import Tools
 from LOGS.Entities.Dataset import Dataset
 from LOGS.Entities.FileEntry import FileEntry
 from LOGS.Entity.EntityConnector import EntityConnector
-from LOGS.Entity.SerializeableContent import SerializeableClass
 from LOGS.LOGSConnection import LOGSConnection, MultipartEntry
 
 if TYPE_CHECKING:
     pass
 
 
-class DatasetSourceType(Enum):
-    ManualUpload = 0
-    SFTPAutoload = 1
-    ClientAutoload = 2
-    APIUpload = 3
+class DatasetUploadRequest(Dataset):
+    _typeMapper = {"files": FileEntry}
 
+    def __init__(self, ref=None):
+        super().__init__(ref)
+        if ref and isinstance(ref, Dataset):
+            self._files = ref._files
 
-class DatasetUploadRequest(SerializeableClass):
-    _typeMapper = {"files": FileEntry}
+    _files: Optional[List[FileEntry]]
+    _filePathsAreAbsolute: Optional[bool] = True
 
-    def __init__(self, ref: Any = None):
-        self.name: Optional[str] = None
-        self.formatId: str = ""
-        self.methodId: Optional[int] = None
-        self.instrumentId: Optional[int] = None
-        self.experimentId: Optional[int] = None
-        self.sampleId: Optional[int] = None
-        self.ownerId: Optional[int] = None
-        self.projectIds: Optional[List[int]] = None
-        self.organizationIds: Optional[List[int]] = None
-        self.operatorIds: Optional[List[int]] = None
-        self.equipmentIds: Optional[List[int]] = None
-        self.autoloadBaseDir: str = ""
-        self.relativeDir: str = ""
-        self.dataSourceId: Optional[str] = None
-        self.isViewableEntity: Optional[bool] = None
-        self.files: List[FileEntry] = []
-        self.filePathsAreAbsolute: Optional[bool] = True
-
-        if isinstance(ref, Dataset) and ref.format:
-            self.name = ref.name
-
-            if ref.name:
-                self.name = ref.name
-            if ref.format:
-                self.formatId = ref.format.id
-            if ref.owner:
-                self.ownerId = ref.owner.id
-            if ref._files:
-                self.files = ref._files
-            if ref.method:
-                self.methodId = ref.method.id
-            if ref.instrument:
-                self.instrumentId = ref.instrument.id
-            if ref.experiment:
-                self.experimentId = ref.experiment.id
-            if ref.sampleId:
-                self.sampleId = ref.sampleId
-            if ref.owner:
-                self.ownerId = ref.owner.id
-            if ref.projects:
-                self.projectIds = [p.id for p in ref.projects]
-            if ref.operators:
-                self.operatorIds = [o.id for o in ref.operators]
-            if ref.equipments:
-                self.equipmentIds = [e.id for e in ref.equipments]
-            if ref.isViewableEntity:
-                self.isViewableEntity = ref.isViewableEntity
-            if ref._files:
-                self.files = ref._files
-            ref = None
+    @property
+    def files(self) -> Optional[List[FileEntry]]:
+        return self._files
 
-        super().__init__(ref)
+    @files.setter
+    def files(self, value):
+        self._files = self.checkListAndConvertNullable(value, FileEntry, "files")
 
     @property
-    @deprecated(details="Please use property 'formatId'")
-    def parserId(self):
-        return self.formatId
-
-    @parserId.setter
-    @deprecated(details="Please use property 'formatId'")
-    def paserId(self, value):
-        self.formatId = value
+    def filePathsAreAbsolute(self) -> Optional[bool]:
+        return self._filePathsAreAbsolute
+
+    @filePathsAreAbsolute.setter
+    def filePathsAreAbsolute(self, value):
+        self._filePathsAreAbsolute = self.checkAndConvertNullable(
+            value, bool, "filePathsAreAbsolute"
+        )
 
 
 @Endpoint("datasets")