logs-py 4.0.7__py3-none-any.whl

LOGS/Entities/DatasetMatching.py

@@ -0,0 +1,196 @@
+from dataclasses import dataclass, field
+from typing import Dict, List, Literal, Optional, Sequence, Union
+
+from LOGS.Auxiliary.Constants import Constants
+from LOGS.Auxiliary.Decorators import Endpoint
+from LOGS.Auxiliary.Exceptions import LOGSException
+from LOGS.Auxiliary.Tools import Tools
+from LOGS.Entities.DatasetMatchTypes import MatchRequest, MatchResult
+from LOGS.Entities.FileEntry import FileEntry
+from LOGS.Entity.EntityConnector import EntityConnector
+from LOGS.LOGSConnection import LOGSConnection
+
+
+@dataclass
+class DirectoryTreeNode:
+    name: str = ""
+    path: str = "/"
+    type: Literal["dir", "file"] = "dir"
+    content: Dict[str, "DirectoryTreeNode"] = field(default_factory=dict)
+    fileCount: int = 0
+
+    def print(self, countOnly=False, indent=""):
+        if self.type == "file":
+            if not countOnly:
+                print("%s%s (f)" % (indent, self.name))
+            return
+        print(
+            "%s%s (%s)%s"
+            % (
+                indent,
+                self.name,
+                Tools.getHumanReadableSize(self.fileCount),
+                ": " + str(len(self.content.keys())) if countOnly else "",
+            )
+        )
+        for entry in self.content.values():
+            entry.print(indent=indent + "..", countOnly=countOnly)
+
+    def splitTreeByFileCount(self, maxCount: int, parentPath="", level=0):
+        forrest: List[DirectoryTreeNode] = []
+
+        path = parentPath + "/" + self.name
+        if path == "/":
+            path = ""
+        # print(
+        #     "." * level + self.name,
+        #     self.fileCount,
+        #     "<",
+        #     maxSize,
+        #     "->",
+        #     self.fileCount <= maxSize,
+        #     "=>",
+        #     path,
+        # )
+        if self.fileCount <= maxCount:
+            self.path = path
+            return [self]
+        else:
+            files = [item for item in self.content.values() if item.type == "file"]
+            if len(files) > 0:
+                forrest.append(
+                    DirectoryTreeNode(
+                        name=self.name,
+                        path=path,
+                        content={f.name: f for f in files},
+                        fileCount=len(files),
+                    )
+                )
+
+            for item in self.content.values():
+                if item.type == "dir":
+                    forrest.extend(
+                        item.splitTreeByFileCount(
+                            maxCount=maxCount, parentPath=path, level=level + 1
+                        )
+                    )
+
+        return forrest
+
+
+@Endpoint("data_formats")
+class DatasetMatching(EntityConnector):
+    _request: MatchRequest = MatchRequest()
+    _formatIds: Optional[List[str]] = None
+    _matchResult: Optional[MatchResult] = None
+    _files: List[List[FileEntry]] = []
+    _maxFileCountInDirectory = 20000
+
+    def __init__(
+        self,
+        connection: LOGSConnection,
+        files: Union[Constants.FILE_TYPE, Sequence[Constants.FILE_TYPE]],
+        formatIds: Optional[List[str]] = None,
+        ignoreReadErrors=False,
+    ):
+        self._connection = connection
+        self._formatIds = formatIds
+        self._files = self.splitFileList(
+            FileEntry.entriesFromFiles(files, ignoreReadErrors)
+        )
+
+    @classmethod
+    def fileListToTree(cls, files: List[FileEntry]):
+        root = DirectoryTreeNode(name="")
+        i = 0
+        for file in files:
+            path = file.path.split("/")
+            rootPath = path.pop(0)
+            fileName = path.pop()
+            # fileSize = file.size if file.size else 0
+            if rootPath != "":
+                continue
+            current = root
+            for i, p in enumerate(path):
+                current.fileCount += 1
+                if p not in current.content:
+                    current.content[p] = DirectoryTreeNode(
+                        name=p, path="/".join(f for f in path[:i])
+                    )
+
+                current = current.content[p]
+            current.fileCount += 1
+            current.content[fileName] = DirectoryTreeNode(
+                name=fileName, path=file.path, type="file", fileCount=1
+            )
+
+        return root
+
+    @classmethod
+    def TreeToFileList(cls, root: DirectoryTreeNode):
+        if root.type == "file":
+            file = FileEntry()
+            file.fullPath = root.path
+            file.id = root.path
+            file.path = root.path
+            return [file]
+
+        files: List[FileEntry] = []
+        for item in root.content.values():
+            files.extend(cls.TreeToFileList(item))
+
+        return files
+
+    def splitFileList(self, files: List[FileEntry]):
+        root = self.fileListToTree(files)
+        forrest = root.splitTreeByFileCount(maxCount=self._maxFileCountInDirectory)
+        return [self.TreeToFileList(tree) for tree in forrest]
+
+    def __iter__(self):
+        for files in self._files:
+            self._request = MatchRequest()
+            self._request.formatIds = self._formatIds
+            self._request.files = files
+
+            self._match()
+            if not self._matchResult:
+                return None
+            for match in self._matchResult.matches:
+                for dataset in match:
+                    yield dataset
+
+    def _match(self):
+        connection, endpoint = self._getConnectionData()
+
+        # print(
+        #     ">>> request",
+        #     len(self._request.files),
+        #     len(str(self._request.toDict()).encode("utf-8")),
+        #     "->",
+        #     len(str(self._request.toDict()).encode("utf-8")) / len(self._request.files),
+        #     "<",
+        #     30000000,
+        # )
+        data, error = connection.postEndpoint(
+            endpoint=endpoint + ["match"], data=self._request.toDict()
+        )
+        if error:
+            raise LOGSException("Could not match dataset files: %a" % error)
+
+        self._matchResult = MatchResult(data)
+
+        if self._matchResult.missingFragments:
+            lookUp = {e.id: e for e in self._request.files}
+            for fileFragment in self._matchResult.missingFragments:
+                if fileFragment.id not in lookUp:
+                    continue
+
+                file = lookUp[fileFragment.id]
+                file.addFragment(fileFragment.fragments)
+
+            data, error = connection.postEndpoint(
+                endpoint=endpoint + ["match"], data=self._request.toDict()
+            )
+            if error:
+                raise LOGSException("Could not match dataset files: %a" % error)
+            self._matchResult = MatchResult(data)

LOGS/Entities/DatasetMinimal.py

@@ -0,0 +1,8 @@
+from LOGS.Auxiliary.Decorators import FullModel
+from LOGS.Entities.Dataset import Dataset
+from LOGS.Entity.EntityMinimalWithIntId import EntityMinimalWithIntId
+
+
+@FullModel(Dataset)
+class DatasetMinimal(EntityMinimalWithIntId[Dataset]):
+    pass

LOGS/Entities/DatasetModels.py

@@ -0,0 +1,33 @@
+from enum import Enum
+from typing import List, Optional
+
+from LOGS.Auxiliary.Tools import Tools
+from LOGS.Entity.SerializableContent import SerializableClass
+
+
+class DatasetSourceType(Enum):
+    ManualUpload = 0
+    SFTPAutoload = 1
+    ClientAutoload = 2
+    APIUpload = 3
+
+
+class ParsedMetadata(SerializableClass):
+    Parameters: bool = False
+    Tracks: bool = False
+    TrackCount: int = False
+    TrackViewerTypes: List[str] = []
+
+
+class DatasetSource(SerializableClass):
+    id: Optional[int] = None
+    type: Optional[DatasetSourceType] = None
+    name: Optional[str] = None
+
+    def __str__(self):
+        return Tools.ObjectToString(self)
+
+
+class ViewableEntityTypes(Enum):
+    ELN = "ELN"
+    CustomField = "CustomField"

LOGS/Entities/DatasetRequestParameter.py

@@ -0,0 +1,92 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any, Dict, List, Optional, Sequence, Type, cast
+
+from typing_extensions import Self
+
+from LOGS.Auxiliary.Constants import Constants
+from LOGS.Entity.EntityRequestParameter import EntityRequestParameter
+from LOGS.Entity.IGenericEntityOrderBy import (
+    IEntryRecordSortingOptions,
+    IGenericEntitySortingOptions,
+    IModificationRecordSortingOptions,
+    INamedEntitySortingOptions,
+    ITypedEntitySortingOptions,
+)
+from LOGS.Interfaces.IEntryRecord import IEntryRecordRequest
+from LOGS.Interfaces.ILockableEntity import ILockableEntityRequest
+from LOGS.Interfaces.IModificationRecord import IModificationRecordRequest
+from LOGS.Interfaces.INamedEntity import INamedEntityRequest
+from LOGS.Interfaces.IOwnedEntity import IOwnedEntityRequest
+from LOGS.Interfaces.IPermissionedEntity import IPermissionedEntityRequest
+from LOGS.Interfaces.IProjectBased import IProjectBasedRequest
+from LOGS.Interfaces.ISignableEntity import ISignableEntityRequest
+from LOGS.Interfaces.ISoftDeletable import ISoftDeletableRequest
+from LOGS.Interfaces.ITypedEntity import ITypedEntityRequest
+from LOGS.Interfaces.IUniqueEntity import IUniqueEntityRequest
+
+
+class ParsingStates(Enum):
+    ParsedSuccessfully = "ParsedSuccessfully"
+    NotParsable = "NotParsable"
+    ParsingFailed = "ParsingFailed"
+    NotYetParsed = "NotYetParsed"
+
+
+class DatasetSortingOptions(
+    IGenericEntitySortingOptions,
+    INamedEntitySortingOptions,
+    IEntryRecordSortingOptions,
+    IModificationRecordSortingOptions,
+    ITypedEntitySortingOptions,
+):
+    CREATION_DATE: Self = cast(Self, "CREATION_DATE")
+    PARSING_STATE: Self = cast(Self, "PARSING_STATE")
+    FORMAT_ID: Self = cast(Self, "FORMAT_ID")
+    OWNER: Self = cast(Self, "OWNER")
+
+
+@dataclass
+class DatasetRequestParameter(
+    EntityRequestParameter[DatasetSortingOptions],
+    IPermissionedEntityRequest,
+    IUniqueEntityRequest,
+    INamedEntityRequest,
+    IOwnedEntityRequest,
+    IProjectBasedRequest,
+    IEntryRecordRequest,
+    IModificationRecordRequest,
+    ISoftDeletableRequest,
+    ILockableEntityRequest,
+    ISignableEntityRequest,
+    ITypedEntityRequest,
+):
+    _orderByType: Type[DatasetSortingOptions] = field(
+        default=DatasetSortingOptions, init=False
+    )
+    includeSource: Optional[bool] = None
+    hasCustomImport: Optional[bool] = None
+    creationDateFrom: Optional[datetime] = None
+    creationDateTo: Optional[datetime] = None
+    autoloadServerIds: Optional[List[int]] = None
+    bridgeIds: Optional[List[int]] = None
+    dataSourceIds: Optional[List[int]] = None
+    excludeUndeleted: Optional[bool] = None
+    files: Optional[Sequence[Constants.FILE_TYPE]] = None
+    formatIds: Optional[List[str]] = None
+    hashes: Optional[List[str]] = None
+    includeUnclaimed: Optional[Optional[bool]] = None
+    isClaimed: Optional[Optional[bool]] = None
+    isReferencedByLabNotebook: Optional[Optional[bool]] = None
+    parameters: Optional[Dict[str, Any]] = None
+    parsingState: Optional[List[ParsingStates]] = None
+    pathContains: Optional[str] = None
+    searchTermIncludeParameters: Optional[bool] = None
+    searchTermIncludePaths: Optional[bool] = None
+
+    # Additional fetch options
+    includeParameters: Optional[bool] = None
+    includeInfo: Optional[bool] = None
+    includeZipSize: Optional[bool] = None
+    includeExports: Optional[bool] = None

LOGS/Entities/DatasetTemplate.py

@@ -0,0 +1,23 @@
+from typing import TYPE_CHECKING, Optional
+
+from LOGS.Auxiliary.Tools import Tools
+from LOGS.Entity.ConnectedEntity import ConnectedEntity
+from LOGS.Interfaces.IOwnedEntity import IOwnedEntity
+from LOGS.Interfaces.IProjectBased import IProjectBased
+from LOGS.Interfaces.ITypedEntity import ITypedEntity
+
+if TYPE_CHECKING:
+    pass
+
+
+class DatasetTemplate(ConnectedEntity, ITypedEntity, IOwnedEntity, IProjectBased):
+    _id: int = 0
+    _datasetTemplates: Optional[list] = None
+
+    @property
+    def id(self) -> int:
+        return self._id
+
+    @id.setter
+    def id(self, value):
+        self._id = Tools.checkAndConvert(value, int, "id")

LOGS/Entities/DatasetUploadParameter.py

@@ -0,0 +1,14 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
+
+@dataclass
+class DatasetUploadParameter:
+    name: Optional[str] = None
+    methodId: Optional[int] = None
+    instrumentId: Optional[int] = None
+    experimentId: Optional[int] = None
+    sampleId: Optional[int] = None
+    projectIds: Optional[List[int]] = None
+    organizationIds: Optional[List[int]] = None
+    operatorIds: Optional[List[int]] = None

LOGS/Entities/Datasets.py

@@ -0,0 +1,142 @@
+import os
+from typing import List, Optional, Sequence, cast
+
+from LOGS.Auxiliary.Constants import Constants
+from LOGS.Auxiliary.Decorators import Endpoint
+from LOGS.Auxiliary.Exceptions import LOGSException
+from LOGS.Auxiliary.Tools import Tools
+from LOGS.Entities.Dataset import Dataset
+from LOGS.Entities.DatasetMatchTypes import (
+    DatasetForSearch,
+    DatasetSearchRequest,
+    DatasetSearchResult,
+)
+from LOGS.Entities.DatasetRequestParameter import DatasetRequestParameter
+from LOGS.Entities.FileEntry import FileEntry
+from LOGS.Entity.EntityIterator import EntityIterator
+from LOGS.LOGSConnection import ResponseTypes
+
+
+@Endpoint("datasets")
+class Datasets(EntityIterator[Dataset, DatasetRequestParameter]):
+    """LOGS connected Dataset iterator"""
+
+    _generatorType = Dataset
+    _parameterType = DatasetRequestParameter
+
+    def download(
+        self,
+        directory: Optional[str] = None,
+        fileName: Optional[str] = None,
+        overwrite=False,
+    ) -> str:
+        connection, endpoint = self._getConnectionData()
+
+        if not directory:
+            directory = os.curdir
+
+        path = os.path.join(
+            directory,
+            Tools.sanitizeFileName(fileName=fileName, defaultName="Dataset.zip"),
+        )
+
+        if overwrite:
+            if os.path.exists(path) and not os.path.isfile(path):
+                raise LOGSException("Path %a is not a file" % path)
+        else:
+            if os.path.exists(path):
+                raise LOGSException("File %a already exists" % path)
+
+        data, error = connection.postEndpoint(
+            endpoint=endpoint + ["zip"],
+            data=self._parameters.toDict(),
+            responseType=ResponseTypes.RAW,
+        )
+        if error:
+            raise LOGSException(
+                "Could not fetch datasets zip file: %a" % error.errorString()
+            )
+
+        with open(path, mode="wb") as localFile:
+            localFile.write(cast(bytes, data))
+
+        return path
+
+    def _getDatasetSearchRequest(
+        self,
+        files: Sequence[Constants.FILE_TYPE],
+        formatIds: List[str],
+        checkUpdatable=True,
+    ):
+        fileList = FileEntry.entriesFromFiles(files)
+        for file in fileList:
+            file.addHash()
+        # print("\n".join([f.fullPath for f in fileList]))
+
+        request = DatasetSearchRequest()
+
+        request.datasets = []
+        for formatId in formatIds:
+            dataset = DatasetForSearch()
+            dataset.checkUpdatable = checkUpdatable
+            dataset.formatId = formatId
+            dataset.files.extend(fileList)
+            request.datasets.append(dataset)
+        return request
+
+    def findDatasetByFiles(
+        self,
+        files: Sequence[Constants.FILE_TYPE],
+        formatIds: List[str],
+        checkUpdatable=True,
+    ):
+        request = self._getDatasetSearchRequest(files, formatIds, checkUpdatable)
+        connection, endpoint = self._getConnectionData()
+
+        data, errors = connection.postEndpoint(
+            endpoint=endpoint + ["find"], data=request.toDict()
+        )
+        if errors:
+            raise LOGSException("Could not find dataset by files: %a" % errors)
+
+        return Tools.checkListAndConvert(data, DatasetSearchResult, "files search")
+
+    def __iter__(self):
+        if self._parameters:
+            parameters = cast(DatasetRequestParameter, self._parameters)
+
+            if parameters.files:
+                if not parameters.formatIds:
+                    typeName = type(parameters).__name__
+                    raise LOGSException(
+                        "%s.formatIds must be defined when %s.files is used."
+                        % (typeName, typeName)
+                    )
+
+                results = self.findDatasetByFiles(
+                    parameters.files, parameters.formatIds, False
+                )
+
+                if len(results) > 0:
+                    if parameters.ids is None:
+                        parameters.ids = []
+
+                    cast(List[int], parameters.ids).extend(
+                        [r.logsId for r in results if r.logsId]
+                    )
+        self._initEntityIterator()
+        return self
+
+    def __next__(self):
+        dataset = super().__next__()
+
+        if self._parameters.includeParameters:
+            dataset.fetchParameters()
+        if self._parameters.includeInfo:
+            dataset.fetchInfo()
+        if self._parameters.includeZipSize:
+            dataset.fetchZipSize()
+        if self._parameters.includeExports:
+            dataset.fetchExports()
+
+        return dataset

LOGS/Entities/Datatrack.py

@@ -0,0 +1,179 @@
+import os
+from typing import Any, List, Literal, Optional, cast
+
+from LOGS.Auxiliary.Exceptions import EntityIncompleteException, LOGSException
+from LOGS.Entity.ConnectedEntity import ConnectedEntity
+from LOGS.LOGSConnection import LOGSConnection, ResponseTypes
+
+_NumberTypeType = Literal["int", "float", "double"]
+_DatatrackType = Literal[
+    "binary", "char", "formatted_table", "image", "numeric_array", "numeric_matrix"
+]
+_CodecType = Literal["char", "jpeg", "points", "generator"]
+
+
+class Datatrack(ConnectedEntity):
+    _type: Optional[_DatatrackType] = None
+    _codec: Optional[_CodecType] = None
+    _id: Optional[str] = None
+    _count: Optional[int] = None
+    _size: Optional[List[int]] = None
+    _min: Optional[List[float]] = None
+    _max: Optional[List[float]] = None
+    _numberType: Optional[_NumberTypeType] = None
+    _data: Optional[Any] = None
+    _incomplete = True
+
+    def __init__(self, ref=None, connection: Optional[LOGSConnection] = None):
+        self._noSerialize += ["data"]
+        super().__init__(ref=ref, connection=connection)
+
+    def _getConnectionData(self):
+        if not self._endpoint:
+            raise NotImplementedError(
+                "Endpoint missing for of entity type %a." % (type(self).__name__)
+            )
+
+        if not self.id:
+            raise LOGSException("%s id is not defined." % type(self).__name__)
+
+        return self._getConnection(), self._endpoint, self.id
+
+    def _fetchDataFromCache(self):
+        cacheFile = self._getCacheFile()
+        if cacheFile is None:
+            raise LOGSException("Cache directory not defined.")
+
+        if not os.path.exists(cacheFile):
+            return None
+
+        with open(cacheFile, "rb") as f:
+            return f.read()
+
+    def _getCacheFile(self):
+        if self.cacheDir is None:
+            return None
+
+        return os.path.join(self.cacheDir, self.cacheId + ".cache")
+
+    def _storeDataInCache(self, data):
+        cacheFile = self._getCacheFile()
+        if cacheFile is None:
+            raise LOGSException("Cache directory not defined.")
+        with open(cacheFile, "wb") as f:
+            f.write(data)
+
+    def clearCache(self):
+        cacheFile = self._getCacheFile()
+        if cacheFile is not None:
+            if os.path.exists(cacheFile):
+                os.remove(cacheFile)
+
+    def _fetchData(self):
+        data = None
+        if self.cacheDir:
+            data = self._fetchDataFromCache()
+
+        if data is None:
+            connection, endpoint, id = self._getConnectionData()
+
+            data, responseError = connection.getEndpoint(
+                endpoint + [id], responseType=ResponseTypes.RAW
+            )
+            if responseError:
+                raise LOGSException(
+                    "Could not fetch %s: %s"
+                    % (type(self).__name__, responseError.errorString()),
+                    responseError=responseError,
+                )
+
+            if self.cacheDir:
+                self._storeDataInCache(data)
+
+        self._data = data
+
+    def fetchFull(self, cacheDir: Optional[str] = None):
+        self._fetchData()
+        self._incomplete = False
+
+    def __iter__(self):
+        if self._incomplete:
+            raise EntityIncompleteException(self)
+        if self._data is not None:
+            for x in self._data:
+                yield x
+
+    @property
+    def type(self) -> Optional[_DatatrackType]:
+        return self._type
+
+    @property
+    def codec(self) -> Optional[_CodecType]:
+        return self._codec
+
+    @codec.setter
+    def codec(self, value):
+        self._codec = cast(Any, self.checkAndConvertNullable(value, str, "codec"))
+
+    @property
+    def id(self) -> Optional[str]:
+        return self._id
+
+    @id.setter
+    def id(self, value):
+        self._id = self.checkAndConvertNullable(value, str, "id")
+
+    @property
+    def count(self) -> Optional[int]:
+        return self._count
+
+    @count.setter
+    def count(self, value):
+        self._count = self.checkAndConvertNullable(value, int, "count")
+
+    @property
+    def size(self) -> Optional[List[int]]:
+        return self._size
+
+    @size.setter
+    def size(self, value):
+        self._size = self.checkListAndConvertNullable(value, int, "size")
+
+    @property
+    def min(self) -> Optional[List[float]]:
+        return self._min
+
+    @min.setter
+    def min(self, value):
+        self._min = self.checkListAndConvertNullable(value, float, "min")
+
+    @property
+    def max(self) -> Optional[List[float]]:
+        return self._max
+
+    @max.setter
+    def max(self, value):
+        self._max = self.checkListAndConvertNullable(value, float, "max")
+
+    @property
+    def numberType(self) -> Optional[_NumberTypeType]:
+        return self._numberType
+
+    @numberType.setter
+    def numberType(self, value):
+        self._numberType = cast(
+            Any, self.checkAndConvertNullable(value, str, "numberType")
+        )
+
+    @property
+    def data(self) -> Optional[Any]:
+        raise NotImplementedError(
+            "Field 'data' of %a class not implemented." % type(self).__name__
+        )
+
+    @property
+    def cacheId(self) -> str:
+        if self._cacheId is None:
+            return f"{self.type}_{self.id}"
+        else:
+            return self._cacheId