etiket-sync-agent-native 0.3.0b1__py3-none-any.whl

etiket_sync_agent_native/__init__.py

@@ -0,0 +1,5 @@
+from .native_sync_class import NativeSync
+from .native_config_class import NativeConfigData
+
+__all__ = ["NativeSync", "NativeConfigData"]
+__version__ = "0.3.0b1"

etiket_sync_agent_native/native_config_class.py

@@ -0,0 +1,23 @@
+import dataclasses
+from typing import Optional
+
+# Keep imports minimal to avoid hard dependencies in a fresh template
+# from etiket_sync_agent.crud.sync_sources import crud_sync_sources, SyncSources
+# from etiket_sync_agent.db import get_db_session_context
+
+@dataclasses.dataclass
+class NativeConfigData:
+    """Template config dataclass for a sync backend.
+
+    Add your configuration fields here and implement the validate method
+    to perform sanity checks and cross-check with existing sources.
+    """
+    # example_field: str | None = None
+
+    async def validate(self, current_sync_source: Optional[object] = None):
+        """Validate configuration.
+
+        Implement your validation logic here.
+        Raise ValueError with a helpful message on invalid config.
+        """
+        return True

etiket_sync_agent_native/native_sync_class.py

@@ -0,0 +1,214 @@
+import typing
+import uuid
+import datetime
+import os
+
+from etiket_client.local.dao.dataset import dao_dataset, DatasetUpdate as DatasetUpdateLocal
+from etiket_client.local.dao.file import dao_file, FileUpdate as FileUpdateLocal
+from etiket_client.local.database import get_db_session_context 
+from etiket_client.local.models.file import FileSelect, FileStatusLocal, FileType
+from etiket_client.local.models.dataset import DatasetRead
+
+from etiket_client.local.exceptions import DatasetNotFoundException
+
+from etiket_client.remote.endpoints.dataset import dataset_read, dataset_create, dataset_update
+from etiket_client.remote.endpoints.file import file_create, file_generate_presigned_upload_link_single
+from etiket_client.remote.upload.single_part import upload_new_file_single
+
+from etiket_client.remote.endpoints.models.dataset import DatasetCreate, DatasetUpdate, DatasetRead as RemoteDatasetRead
+from etiket_client.remote.endpoints.models.file import FileCreate, FileRead
+from etiket_client.remote.endpoints.models.types import FileStatusRem
+
+from etiket_sync_agent.backends.sync_source_abstract import SyncSourceDatabaseBase, ScopeRequirement
+from etiket_sync_agent.schemas import SyncItemSchema, SyncItemCreate
+from etiket_sync_agent.sync.sync_records.manager import SyncRecordManager
+
+from etiket_sync_agent.sync.sync_utilities import md5
+
+from etiket_sync_agent_native.native_config_class import NativeConfigData
+
+# TODO tests :: add sync loop item, that first creates a qcodes dataset and then changes that dataset
+# -- this is important to see that the restrictions in the database are set correctly.
+
+class NativeSync(SyncSourceDatabaseBase):
+    sync_agent_name: typing.ClassVar[str] = "native"
+    scope_requirement: typing.ClassVar[ScopeRequirement] = ScopeRequirement.DISABLED
+    supports_scope_mapping: typing.ClassVar[bool] = False
+    live_sync_implemented: typing.ClassVar[bool] = False
+    config_data_class: typing.ClassVar[typing.Type[NativeConfigData]] = NativeConfigData
+    has_owner: typing.ClassVar[bool] = False
+
+    @staticmethod
+    async def getNewDatasets(config_data: NativeConfigData, last_sync_item: SyncItemSchema | None) -> typing.List[SyncItemCreate]:
+        with get_db_session_context() as session:
+            last_timestamp = None
+            if last_sync_item is not None:
+                last_timestamp = last_sync_item.syncPriority
+            datasets = dao_dataset.get_sync_items(last_timestamp, session)
+            sync_items = []
+            for dataset in datasets:
+                modified_time = dataset.modified
+                for file in dataset.files:
+                    if file.synchronized is False and file.modified > modified_time:
+                        modified_time = file.modified
+                modified_time_timestamp = modified_time.replace(tzinfo=datetime.timezone.utc).timestamp()
+                sync_item = SyncItemCreate(datasetUUID = dataset.uuid,
+                                        dataIdentifier = str(dataset.uuid),
+                                        syncPriority = modified_time_timestamp,
+                                        owner = dataset.last_modified_by)
+                sync_items.append(sync_item)
+        return sync_items
+
+    @staticmethod
+    async def checkLiveDataset(configData: NativeConfigData, syncIdentifier: SyncItemSchema, maxPriority: bool) -> bool:
+        return False
+    
+    @staticmethod
+    async def syncDatasetNormal(configData: NativeConfigData, syncIdentifier: SyncItemSchema, sync_record: SyncRecordManager):
+        # Here manual functions are used.
+        with sync_record.task("Start synchronization of the dataset"):
+            with get_db_session_context() as session:
+                with sync_record.task("Read local dataset"):
+                    dataset_local = dao_dataset.read(syncIdentifier.datasetUUID, session)
+                    sync_record.add_log("Success!")
+                
+                dataset_remote = None
+                with sync_record.task("synchronize metadata to server"):
+                    try :
+                        dataset_remote = dataset_read(dataset_local.uuid)
+                        sync_record.add_log("A remote dataset is already present.")
+                    except DatasetNotFoundException:
+                        sync_record.add_log("No remote dataset found.")
+
+                    if not dataset_remote:
+                        sync_record.add_log("Attempt to create remote dataset.")
+                        dc = DatasetCreate(**dataset_local.model_dump(), scope_uuid=dataset_local.scope.uuid)
+                        dataset_create(dc)
+                        dataset_remote = dataset_read(dataset_local.uuid)
+                        sync_record.add_log("Remote dataset created.")
+                    else:
+                        if dataset_local.modified > dataset_remote.modified:
+                            if needs_metadata_update(dataset_local, dataset_remote):
+                                sync_record.add_log("Updating metadata of remote dataset ...")
+                                du = DatasetUpdate(**dataset_local.model_dump())
+                                dataset_update(dataset_local.uuid, du)
+                                sync_record.add_log("Metadata updated.")
+                                dataset_remote = dataset_read(dataset_local.uuid)
+                            else:
+                                sync_record.add_log("Metadata up to date, no updates needed.")
+                        else:
+                            sync_record.add_log("Metadata up to date, no updates needed.")
+
+                with sync_record.task("Synchronizing files to server"):
+                    files =  dataset_local.files if dataset_local.files is not None else []
+                    for file in files:
+                        if file.status == FileStatusLocal.complete and file.synchronized is False and file.type != FileType.HDF5_CACHE:
+                            with sync_record.add_upload_task(file.name) as file_upload_info:
+                                file_upload_info.filename = file.filename
+                                file_upload_info.file_uuid = file.uuid
+                                file_upload_info.version_id = file.version_id
+                                file_upload_info.file_path = file.local_path
+                                file_upload_info.file_m_time = file.collected
+                                file_upload_info.fileType = file.type
+                                if file.local_path and os.path.exists(file.local_path):
+                                    file_upload_info.size = os.stat(file.local_path).st_size
+
+                                sync_record.add_log(f"Synchronizing file with name {file.name} and version_id {file.version_id}")
+                                fs = FileSelect(uuid=file.uuid, version_id=file.version_id)
+
+                                file_remote = get_remote_file(dataset_remote.files, file.uuid, file.version_id)
+                                if file_remote:
+                                    sync_record.add_log("File record already present on the remote server, updating details.")
+                                    # TODO (later) update details --> this is currently done auto in dataqruiser.
+                                    if file_remote.status == FileStatusRem.secured:
+                                        fu = FileUpdateLocal(synchronized=True)
+                                        dao_file.update(fs, fu, session, update_modified_by=False)
+                                        sync_record.add_log("File already secured, done.")
+                                        continue
+                                    else:
+                                        sync_record.add_log("File record already present on the remote server, but not yet secured, proceeding to upload.")
+                                else:
+                                    sync_record.add_log("Creating file record on the remote server.")
+                                    fc = FileCreate(**file.model_dump(), ds_uuid=dataset_local.uuid)
+                                    file_create(fc)
+                                    sync_record.add_log("File record created.")
+                                
+                                
+                                sync_record.add_log("Starting upload of the file.")
+                                upload_info = file_generate_presigned_upload_link_single(file.uuid, file.version_id)
+                                md5_checksum = md5(file.local_path)
+                                file_upload_info.md5 = md5_checksum.hexdigest()
+                                upload_new_file_single(file.local_path, upload_info, md5_checksum)
+                                sync_record.add_log("Upload finished.")
+
+                                fu = FileUpdateLocal(synchronized=True)
+                                dao_file.update(fs, fu, session, update_modified_by=False)
+                        else:
+                            if file.status != FileStatusLocal.complete:
+                                sync_record.add_log(f"skipping {file.name} (version :: {file.version_id}) as file status is not yet completed (status = {file.status}).")
+                            elif file.type == FileType.HDF5_CACHE:
+                                sync_record.add_log(f"skipping {file.name} (version :: {file.version_id}) as file is marked as cache.")
+                            elif file.synchronized is True:
+                                sync_record.add_log(f"skipping {file.name} (version :: {file.version_id}) as file is already synchronized.")
+                            else:
+                                raise ValueError(f"Error the code should not reach here. -- name : {file.name}, file_uuid : {file.uuid}, version_id : {file.version_id}, status : {file.status}, synced : {file.synchronized}")
+                    
+                    # update the local dataset to mark it as synchronized
+                    ds_local_reread = dao_dataset.read(dataset_local.uuid, session)
+                    if ds_local_reread.modified == dataset_local.modified:
+                        du = DatasetUpdateLocal(synchronized=True)
+                        if needs_metadata_update(ds_local_reread, dataset_remote):
+                            sync_record.add_log("Updating metadata of local dataset from newer remote ...")
+                            du = DatasetUpdateLocal(
+                                name=dataset_remote.name,
+                                description=dataset_remote.description,
+                                keywords=dataset_remote.keywords,
+                                ranking=dataset_remote.ranking,
+                                attributes=dataset_remote.attributes,
+                                synchronized=True,
+                            )
+                        dao_dataset.update(dataset_local.uuid, du, session)
+                        sync_record.add_log("Local metadata updated from remote and marked as synchronized.")
+                    else:
+                        sync_record.add_log("Not marked as synchronized, as the local dataset was modified during the sync.")
+                    
+            sync_record.add_log("Dataset sync is successful!")
+
+
+    @staticmethod
+    async def syncDatasetLive(configData: NativeConfigData, syncIdentifier: SyncItemSchema, sync_record: SyncRecordManager):
+        raise NotImplementedError
+    
+def get_remote_file(files : typing.List[FileRead], file_uuid : uuid.UUID, version_id : int):
+    for file in files:
+        if file.uuid == file_uuid and file.version_id == version_id:
+            return file
+    return None
+
+
+def datasets_metadata_equal(ds_local: DatasetRead, ds_remote: RemoteDatasetRead) -> bool:
+    """Return True only if all relevant metadata fields match."""
+    if ds_local.uuid != ds_remote.uuid:
+        return False
+    if ds_local.alt_uid != ds_remote.alt_uid:
+        return False
+    if ds_local.collected != ds_remote.collected:
+        return False
+    if ds_local.name != ds_remote.name:
+        return False
+    if ds_local.description != ds_remote.description:
+        return False
+    if ds_local.creator != ds_remote.creator:
+        return False
+    if ds_local.ranking != ds_remote.ranking:
+        return False
+    if ds_local.keywords != ds_remote.keywords:
+        return False
+    if ds_local.attributes != ds_remote.attributes:
+        return False
+    return True
+
+
+def needs_metadata_update(ds_local: DatasetRead, ds_remote: RemoteDatasetRead) -> bool:
+    """Return True if any relevant metadata field differs."""
+    return not datasets_metadata_equal(ds_local, ds_remote)

etiket_sync_agent_native/native_sync_config_class.py

@@ -0,0 +1,23 @@
+import dataclasses
+
+from typing import Optional
+
+import etiket_sync_agent_native
+
+from etiket_sync_agent.crud.sync_sources import crud_sync_sources, SyncSources
+from etiket_sync_agent.db import get_db_session_context
+
+@dataclasses.dataclass
+class NativeConfigData:
+    async def validate(self, current_sync_source : Optional[SyncSources] = None):
+        '''
+        There can only be one native sync source, validate this, as no parameters are needed.
+        '''
+        async with get_db_session_context() as session:
+            sync_sources = await crud_sync_sources.list_sync_sources(session)
+            n_sources = 0
+            for sync_source in sync_sources:
+                if sync_source.backend == etiket_sync_agent_native.__name__:
+                    n_sources+=1
+            if n_sources>1:
+                raise ValueError("Native sync source already present. Only one may be added.")

etiket_sync_agent_native-0.3.0b1.dist-info/METADATA

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: etiket_sync_agent_native
+Version: 0.3.0b1
+Summary: Native filesystem backend for eTiKeT sync agent
+Author: QHarbor team
+License-Expression: LicenseRef-Proprietary
+Project-URL: Homepage, https://qharbor.nl
+Project-URL: Documentation, https://docs.qharbor.nl
+Keywords: etiket,sync,backend,native,filesystem
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENCE
+Requires-Dist: etiket_sync_agent>=0.3.0b1
+Requires-Dist: etiket_client
+Dynamic: license-file
+
+# eTiKeT Sync Agent - Native Backend
+
+Backend for synchronizing locally generated datasets (via the qdrive package) with the cloud.
+This is one of the default backends included with the sync agent.
+
+## Installation
+
+This package is a dependency of `etiket_sync_agent` and is installed automatically—no separate installation required.
+
+## Usage
+
+Once installed, the native backend is automatically discovered and available for synchronization. No additional configuration is needed (for more info, see the [QHarbor documentation](https://docs.qharbor.nl)).
+
+## License
+
+Copyright © 2025 QHarbor. All Rights Reserved. See [LICENCE](LICENCE) for details.

etiket_sync_agent_native-0.3.0b1.dist-info/RECORD

@@ -0,0 +1,10 @@
+etiket_sync_agent_native/__init__.py,sha256=z9jRU4lduU4giCemKR8UFfKPKOYq8GlHIJH7Maw-zrA,162
+etiket_sync_agent_native/native_config_class.py,sha256=rG7FGES9G3ohE9_L-g7LwzTAiUeBMH-YG7ta6EbctXg,805
+etiket_sync_agent_native/native_sync_class.py,sha256=-aIqZIkZLrGDPbbaKb-YyX_hILGYVBiTZkc5pWfKBqE,12757
+etiket_sync_agent_native/native_sync_config_class.py,sha256=gaqqUR25Aa4EbzQJj6_uPopoVhUZHc_VhzArhv3ZDl4,902
+etiket_sync_agent_native-0.3.0b1.dist-info/licenses/LICENCE,sha256=tdZwE43Th9efUgN8-4UpMUyh0kYQ4Dk678agSuhpnc0,2357
+etiket_sync_agent_native-0.3.0b1.dist-info/METADATA,sha256=65SObPNaD8B2zqQP35BG-9AZ8-5P7vyromom_DEUIlg,1874
+etiket_sync_agent_native-0.3.0b1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+etiket_sync_agent_native-0.3.0b1.dist-info/entry_points.txt,sha256=nW66Tgz1B6Ggoco8Bn2KAwypT2RdZ5KP2XpCgMwCB0s,85
+etiket_sync_agent_native-0.3.0b1.dist-info/top_level.txt,sha256=7JV6r6h6u4iLUFG_xy6hbkugbyiGG3kJ8BpnCWRXl9E,25
+etiket_sync_agent_native-0.3.0b1.dist-info/RECORD,,

etiket_sync_agent_native-0.3.0b1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

etiket_sync_agent_native-0.3.0b1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[etiket_sync_agent.backends]
+native_sync_agent = etiket_sync_agent_native:NativeSync

etiket_sync_agent_native-0.3.0b1.dist-info/licenses/LICENCE

@@ -0,0 +1,34 @@
+All Rights Reserved License
+Copyright ©️ 2024-2026 QHarbor B.V. All Rights Reserved.
+
+Agreement to Terms
+By accessing, downloading, installing, or viewing this Software (including its source code, binaries, or application files), you acknowledge and agree to the terms outlined below.
+
+Terms and Conditions
+This software and its source code (the "Software") are the exclusive property of QHarbor B.V. and are protected by copyright and other intellectual property laws.
+
+License and Testing Exemptions
+Commercial License: If you have entered into a separate commercial license agreement with QHarbor B.V., the terms of that agreement shall supersede the restrictions listed below.
+Testing Permission: If you have obtained written permission for testing/evaluation from QHarbor B.V., you are permitted to install and use the Software for evaluation purposes. However, this permission strictly excludes the right to modify, alter, create derivative works, or reverse engineer the Software.
+
+Prohibited Actions
+Unless explicitly authorized by the exemptions above, you are NOT permitted to:
+	•	Copy, reproduce, or duplicate the Software in any form (except as reasonably necessary for viewing or authorized installation)
+	•	Modify, alter, or create derivative works based on the Software
+	•	Distribute, publish, or share the Software with others
+	•	Reverse engineer, decompile, or disassemble the Software
+	•	Use the Software for any commercial or non-commercial purposes
+	•	Transfer, sell, lease, or sublicense the Software
+	•	Remove or alter any copyright notices or proprietary markings
+
+Viewing Only 
+For those without a commercial license or written testing permission, the Software is made available for viewing and reference purposes only. Any access to view the source code or application does not grant any rights to use, copy, or modify the Software.
+
+No Implied Rights
+No rights are granted by implication, estoppel, or otherwise. All rights not expressly granted are reserved by the copyright holder.
+
+Governing Law 
+This license and any disputes arising from it shall be governed by the laws of the Netherlands.
+
+Disclaimer 
+This Software is provided "AS IS" without warranty of any kind. The copyright holder disclaims all warranties and shall not be liable for any damages arising from the use or inability to use this Software.

etiket_sync_agent_native-0.3.0b1.dist-info/top_level.txt

@@ -0,0 +1 @@
+etiket_sync_agent_native