etiket-sync-agent-folderbase 0.3.0b1__py3-none-any.whl

etiket_sync_agent_folderbase/__init__.py

@@ -0,0 +1,5 @@
+from etiket_sync_agent_folderbase.folderbase_sync_class import FolderBaseSync
+from etiket_sync_agent_folderbase.folderbase_config_class import FolderBaseConfigData
+
+__all__ = ["FolderBaseSync", "FolderBaseConfigData"]
+__version__ = "0.3.0b1"

etiket_sync_agent_folderbase/dataset_info.py

@@ -0,0 +1,105 @@
+from etiket_sync_agent.converters.converter_abstract import FileConverter
+from etiket_sync_agent.sync.manifests.v2.definitions import QH_DATASET_INFO_FILE
+
+from typing import Union, Optional, List, Dict, Type
+
+import yaml
+import datetime
+import pathlib
+import warnings
+
+def generate_dataset_info(
+    path: Union[str, pathlib.Path],
+    dataset_name: Optional[str] = None,
+    creation: Optional[datetime.datetime] = None,
+    description: Optional[str] = None,
+    attributes: Optional[Dict[str, Union[str, int, float]]] = None,
+    tags: Optional[List[str]] = None,
+    converters: Optional[List[Type[FileConverter]]] = None,
+    skip: Optional[List[str]] = None,
+    **kwargs) -> None:
+    """
+    Generate a dataset info YAML file for a specified path.
+
+    This function creates a `dataset_info.yml` file containing metadata about the dataset,
+    including its name, creation time, description, attributes, tags, converters, and
+    patterns of files or folders to skip during synchronization.
+
+    Args:
+        path (Union[str, Path]): The path to the dataset where the `dataset_info.yml` will be created.
+            If the path does not yet exists, it will be created.
+        dataset_name (Optional[str], optional): The name of the dataset. Defaults to `None`, which uses the directory name.
+        creation (Optional[datetime.datetime], optional): The creation time of the dataset. 
+            If provided, this should be a `datetime` object.
+        description (Optional[str], optional): A brief description of the dataset.
+        attributes (Optional[Dict[str, Union[str, int, float]]], optional): 
+            A dictionary of attributes to be added to the dataset.
+        tags (Optional[List[str]], optional): 
+            A list of tags associated with the dataset.
+        converters (Optional[List[Type[FileConverter]]], optional): 
+            A list of converter classes used for processing files within the dataset. Each class should be a subclass of `FileConverter`.
+        skip (Optional[List[str]], optional): 
+            A list of file or folder patterns to skip during synchronization (e.g., `["*.json", "text.txt"]`).
+        **kwargs: Additional keyword arguments for backwards compatibility.
+            - keywords (deprecated): Use `tags` instead.
+    """
+    # Handle deprecated 'keywords' argument
+    if 'keywords' in kwargs:
+        warnings.warn(
+            "The 'keywords' argument is deprecated and will be removed in a future version. "
+            "Please use 'tags' instead.",
+            DeprecationWarning,
+            stacklevel=2
+        )
+        if tags is None:
+            tags = kwargs.pop('keywords')
+        else:
+            kwargs.pop('keywords')  # Ignore if tags is already provided
+    
+    # Warn about any unexpected kwargs
+    if kwargs:
+        unexpected = ', '.join(kwargs.keys())
+        warnings.warn(f"Unexpected keyword arguments: {unexpected}", UserWarning, stacklevel=2)
+    
+    dataset_info_dict = {}
+    dataset_info_dict['version'] = "0.1"
+    
+    if dataset_name:
+        dataset_info_dict['dataset_name'] = dataset_name
+    if creation: # format in ISO format
+        dataset_info_dict['creation'] = creation.strftime('%Y-%m-%dT%H:%M:%S')
+    if description:
+        dataset_info_dict['description'] = description
+    if attributes:
+        # check if the depth of the attributes is 1
+        if all(isinstance(v, (str, int, float)) for v in attributes.values()):
+            dataset_info_dict['attributes'] = attributes
+        else:
+            raise ValueError(
+                'Attributes must be a dictionary with a depth of 1, '
+                'and values of type str, int, or float.')
+    if tags:
+        dataset_info_dict['tags'] = tags
+    if converters:
+        if all(issubclass(c, FileConverter) for c in converters):
+            dataset_info_dict['converters'] = { f"{c.input_type}_to_{c.output_type}_converter" : 
+                                                    {'module': c.__module__, 'class': c.__name__}
+                                                    for c in converters}
+        else:
+            raise ValueError('Converters must be a list of subclasses of FileConverter.')
+    if skip:
+        dataset_info_dict['skip'] = skip
+    
+    dataset_path = pathlib.Path(path) if isinstance(path, str) else path
+
+    if not dataset_path.exists():
+        dataset_path.mkdir(parents=True)
+    if not dataset_path.is_dir():
+        raise NotADirectoryError(f"The specified path is not a directory: {dataset_path}")
+
+    output_file = dataset_path / QH_DATASET_INFO_FILE
+    try:
+        with output_file.open('w', encoding="utf-8") as f:
+            yaml.dump(dataset_info_dict, f, sort_keys=False)
+    except IOError as e:
+        raise IOError("Failed to write dataset info file.") from e

etiket_sync_agent_folderbase/folderbase_config_class.py

@@ -0,0 +1,69 @@
+import pathlib
+import dataclasses
+
+from typing import Optional
+
+from etiket_sync_agent.crud.sync_sources import crud_sync_sources, SyncSources
+from etiket_sync_agent.db import get_db_session_context
+from etiket_sync_agent.backends import BACKENDS
+
+
+@dataclasses.dataclass
+class FolderBaseConfigData:
+    root_directory: pathlib.Path
+    is_server_folder : bool
+    
+    def __post_init__(self):
+        # ensure the path is of the type pathlib.Path (str is converted to Path) and expand ~
+        self.root_directory = pathlib.Path(self.root_directory).expanduser()
+    
+    async def validate(self, current_sync_source : Optional[SyncSources] = None):
+        """
+        Validates the file base configuration.
+
+        Checks:
+        1. If the root_directory exists and is a directory.
+        2. If the root_directory conflicts with an existing folderbase sync source
+            (i.e., it's identical, a subdirectory, or a parent directory).
+
+        Raises:
+            ValueError: If any validation check fails.
+
+        Returns:
+            True if all checks pass.
+        """        
+        try:
+            abs_root_dir = self.root_directory.expanduser().resolve(strict=True)
+        except FileNotFoundError:
+            raise ValueError(f"The specified root directory does not exist: {self.root_directory}")
+
+        # check if the root directory exists and is a directory.
+        if not abs_root_dir.is_dir():
+            raise ValueError(f"The specified path is not a directory: {abs_root_dir}")
+
+        # check if the folder is not yet added/is part of a folder that is already added.
+        async with get_db_session_context() as session:
+            sync_sources = await crud_sync_sources.list_sync_sources(session)
+            for sync_source in sync_sources:
+                if sync_source.backend == BACKENDS.FOLDERBASE and (current_sync_source is None or sync_source.id != current_sync_source.id):
+                    # Assuming config_data stores the path as a string and is always present/valid.
+                    existing_path_str = sync_source.config_data['root_directory']
+                    existing_path = pathlib.Path(existing_path_str).expanduser().resolve()
+                    
+                    # Check for conflicts (using samefile for case-insensitive filesystem support)
+                    try:
+                        is_same = abs_root_dir.samefile(existing_path)
+                    except FileNotFoundError:
+                        is_same = False  # existing path was deleted
+
+                    if is_same:
+                        raise ValueError(f"The directory '{abs_root_dir}' is already added as sync source '{sync_source.name}'.")
+                    # Check if the new path is inside an existing path
+                    if abs_root_dir.is_relative_to(existing_path):
+                        raise ValueError(f"The directory '{abs_root_dir}' is inside the directory '{existing_path}' added by sync source '{sync_source.name}'.")
+                    # Check if an existing path is inside the new path
+                    if existing_path.is_relative_to(abs_root_dir):
+                        raise ValueError(f"The directory '{existing_path}' added by sync source '{sync_source.name}' is inside the specified directory '{abs_root_dir}'.")
+
+        return True
+        

etiket_sync_agent_folderbase/folderbase_sync_class.py

@@ -0,0 +1,271 @@
+import fnmatch
+import os
+import pathlib
+import re
+import logging
+import yaml
+import xarray
+import traceback
+
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List
+
+from etiket_sync_agent.converters.converter_abstract import FileConverterHelper
+from etiket_sync_agent_folderbase.folderbase_config_class import FolderBaseConfigData
+from etiket_sync_agent_folderbase.local_sync_record import LocalSyncRecord
+from etiket_sync_agent.exceptions.sync import NoConvertorException
+from etiket_sync_agent.sync.manifests.v2.definitions import QH_DATASET_INFO_FILE, QH_MANIFEST_FILE
+from etiket_sync_agent.backends.sync_source_abstract import SyncSourceFileBase, ScopeRequirement
+from etiket_sync_agent.sync.sync_records.manager import SyncRecordManager
+from etiket_sync_agent.sync.sync_utilities import dataset_info, file_info, FileType, sync_utilities
+from etiket_sync_agent.schemas import SyncItemSchema
+from etiket_sync_agent.converters.converters import get_converter
+
+logger = logging.getLogger(__name__)
+
+# in the config file, the converter should be named as ExtensionA_to_ExtensionB_converter
+converter_naming_scheme = re.compile(r'^(\w+)_to_(\w+)_converter$')
+
+class FolderBaseSync(SyncSourceFileBase):
+    sync_agent_name = "FolderBase"
+    config_data_class = FolderBaseConfigData
+    scope_requirement = ScopeRequirement.REQUIRED
+    supports_scope_mapping = False
+    live_sync_implemented = False
+    has_owner = True
+    level = -1
+    is_single_file = False
+
+    @staticmethod
+    def rootPath(configData: FolderBaseConfigData) -> Path:
+        return Path(configData.root_directory)
+
+    @staticmethod
+    async def checkLiveDataset(configData: FolderBaseConfigData, syncIdentifier: SyncItemSchema, maxPriority: bool) -> bool:
+        return False
+
+    @staticmethod
+    async def syncDatasetNormal(configData: FolderBaseConfigData, syncIdentifier: SyncItemSchema, sync_record: SyncRecordManager):
+        dataset_path = Path(configData.root_directory) / syncIdentifier.dataIdentifier
+
+        with sync_record.task("Read dataset info file and manifest file."):
+            if not (dataset_path / QH_DATASET_INFO_FILE).exists():
+                raise FileNotFoundError(f"Dataset info file not found for dataset: {syncIdentifier.dataIdentifier}")
+            with open(dataset_path / QH_DATASET_INFO_FILE, 'r', encoding="utf-8") as f:
+                sync_info = yaml.safe_load(f)
+
+            local_sync_record = LocalSyncRecord(dataset_path, syncIdentifier, sync_record)
+
+        try:
+            with sync_record.task("Create or update dataset."):
+                await create_or_update_dataset(configData, syncIdentifier, sync_info, sync_record)
+
+            with sync_record.task("Check files."):
+                file_converters = get_file_converters(sync_info, sync_record)
+                skip = sync_info.get('skip', [])
+                skip_folders = ["*.zarr"]
+
+                async def walk_dataset(current_folder : Path):
+                    for dir_entry in os.scandir(current_folder):
+                        dir_path = Path(dir_entry)
+                        name = dir_path.name
+                        if name.startswith('.') or name in [QH_DATASET_INFO_FILE, QH_MANIFEST_FILE]:
+                            continue
+                        rel_path = dir_path.relative_to(dataset_path).as_posix()
+                        if dir_entry.is_dir(follow_symlinks=False):
+                            if dir_path.suffix != "" and not any(fnmatch.fnmatch(rel_path, pattern) for pattern in skip): # must have a suffix (e.g. .zarr)
+                                await upload_folder(dir_path, dataset_path, syncIdentifier, sync_record, file_converters)
+
+                            if any(fnmatch.fnmatch(rel_path, pattern) for pattern in skip_folders + skip):
+                                sync_record.add_log(f"Folder {rel_path} is skipped, as per the skip list.")
+                                continue
+                            await walk_dataset(dir_path)
+                        else:
+                            if any(fnmatch.fnmatch(rel_path, pattern) for pattern in skip):
+                                sync_record.add_log(f"File {rel_path} is skipped, as per the skip list.")
+                                continue
+                            await upload_file(dir_path, dataset_path, syncIdentifier, sync_record, file_converters)
+
+                await walk_dataset(dataset_path)
+        finally:
+            await local_sync_record.apply_pending_updates()
+            local_sync_record.write()
+
+    @staticmethod
+    async def syncDatasetLive(configData: FolderBaseConfigData, syncIdentifier: SyncItemSchema, sync_record: SyncRecordManager):
+        raise NotImplementedError("Live sync is not implemented for FolderBaseSync")
+
+async def create_or_update_dataset(configData : FolderBaseConfigData, syncIdentifier : SyncItemSchema, sync_info : dict, sync_record: SyncRecordManager):
+    """
+    Create or update the dataset based on synchronization information.
+
+    This function gathers the dataset name, attributes, creation time, keywords, and identifiers
+    and creates/updates the dataset as needed in the remote server.
+
+    Args:
+        configData (FileBaseData): Configuration data containing the root directory.
+        syncIdentifier (sync_item): Synchronization identifier with dataset UUIDs and identifiers.
+        sync_info (dict): Dictionary containing synchronization information from the dataset info file.
+    """
+    dataset_path = pathlib.Path(configData.root_directory) / syncIdentifier.dataIdentifier
+
+    name = sync_info.get("dataset_name", dataset_path.name)
+    attributes = sync_info.get('attributes', {})
+    description = sync_info.get('description', None)
+    # get creation time of first file in the folder
+    created = get_created_time(dataset_path)
+
+    if description is None:
+        description = f"Dataset {name} from FileBaseGeneric"
+    else:
+        description += f"\n\nDataset source path: {dataset_path}"
+
+    if 'created' in sync_info:
+        try :
+            # get collected, and if not present, get created
+            created_time_str = sync_info.get('collected', None)
+            if created_time_str is None:
+                created_time_str = sync_info.get('created', None)
+            created = datetime.strptime(created_time_str, '%Y-%m-%dT%H:%M:%S')
+        except Exception:
+            pass
+
+    tags = sync_info.get('tags', None)
+    if tags is None:
+        tags = sync_info.get('keywords', [])
+    ds_info =  dataset_info(name = name, datasetUUID = syncIdentifier.datasetUUID,
+                alt_uid = None, scopeUUID = syncIdentifier.scopeUUID,
+                description=description,
+                created = created, keywords = list(tags),
+                attributes = attributes)
+
+    await sync_utilities.create_or_update_dataset(False, syncIdentifier, ds_info, sync_record)
+
+def get_created_time(dataset_path : pathlib.Path) -> datetime:
+    """
+    Get the earliest modification time among all files in the dataset.
+
+    This function traverses all files within the dataset directory (excluding specific manifest files and hidden files (UNIX))
+    to find the earliest modification time, which is considered the creation time of the dataset.
+
+    Args:
+        dataset_path (Path): Path to the dataset directory.
+
+    Returns:
+        datetime: The earliest modification time of the dataset files.
+    """
+    min_time = datetime.timestamp(datetime.now())
+
+    for root, _, files in os.walk(dataset_path):
+        for file in files:
+            file_path = pathlib.Path(root) / file
+            if file_path.name in [QH_DATASET_INFO_FILE, QH_MANIFEST_FILE] or file_path.name.startswith('.'):
+                continue
+            if file_path.stat().st_mtime < min_time:
+                min_time = file_path.stat().st_mtime
+    return datetime.fromtimestamp(min_time)
+
+def process_name(dataset_path : Path, current_path : Path) -> str:
+    # if root is /A/B and current is /A/B/C/D.txt then return C/D.txt
+    return current_path.relative_to(dataset_path).as_posix()
+
+def generate_converted_file_name(dataset_path : Path, current_path : Path, new_extension : str) -> str:
+    # get processed name
+    relative_path = current_path.relative_to(dataset_path)
+    if relative_path.suffix:
+        new_name = relative_path.with_suffix(f'.{new_extension}')
+    else:
+        new_name = relative_path.parent / f"{relative_path.name}.{new_extension}"
+    return pathlib.PurePosixPath(new_name).as_posix()
+
+async def upload_file(file_path : Path, dataset_path : pathlib.Path,
+                syncIdentifier : SyncItemSchema, sync_record: SyncRecordManager,
+                file_converters : Dict[str, List[FileConverterHelper]]):
+    file = file_path.name
+    name = process_name(dataset_path, file_path)
+    if file in [QH_DATASET_INFO_FILE, QH_MANIFEST_FILE] or file.startswith('.'):
+        return
+
+    f_type = get_file_type(file_path)
+
+    f_info = file_info(name = name, fileName = file,
+        created = datetime.fromtimestamp(file_path.stat().st_mtime),
+        fileType = f_type, file_generator = "")
+    await sync_utilities.upload_file(file_path, syncIdentifier, f_info, sync_record)
+
+    # check if there is a converter for the file
+    extension = file_path.suffix.lstrip('.')
+    if extension in file_converters:
+        for converterHelper in file_converters[extension]:
+            converted_file_name = generate_converted_file_name(dataset_path, file_path, converterHelper.converter.output_type)
+            with sync_record.add_upload_task(converted_file_name) as file_upload_info:
+                with sync_record.define_converter(file_upload_info, converterHelper, file_path) as converted_file_path:
+                    f_type = get_file_type(converted_file_path)
+                    f_info = file_info(name = converted_file_name, fileName = converted_file_path.name,
+                                created = datetime.now(),
+                                fileType = f_type, file_generator = "")
+                    await sync_utilities.upload_file(converted_file_path, syncIdentifier, f_info, sync_record)
+
+async def upload_folder(folder_path : str, dataset_path : pathlib.Path, syncIdentifier : SyncItemSchema, sync_record: SyncRecordManager,
+                file_converters : Dict[str, List[FileConverterHelper]]):
+    '''
+    Function that uploads the contents of a folder as a file. Currently, this is only needed for zarr files.
+    '''
+    folder_path = pathlib.Path(folder_path)
+    extension = folder_path.suffix[1:]
+
+    if extension in file_converters:
+        for converterHelper in file_converters[extension]:
+            converted_file_name = generate_converted_file_name(dataset_path, folder_path, converterHelper.converter.output_type)
+            with sync_record.add_upload_task(converted_file_name) as file_upload_info:
+                with sync_record.define_converter(file_upload_info, converterHelper, folder_path) as converted_file_path:
+                    f_type = get_file_type(converted_file_path)
+                    f_info = file_info(name = converted_file_name, fileName = converted_file_path.name,
+                                created = datetime.now(),
+                                fileType = f_type, file_generator = "")
+
+                    await sync_utilities.upload_file(converted_file_path, syncIdentifier, f_info, sync_record)
+
+def get_file_type(file_path : Path) -> FileType:
+    f_type = FileType.UNKNOWN
+    if file_path.name.endswith(".json"):
+        f_type = FileType.JSON
+    if file_path.name.endswith(".txt"):
+        f_type = FileType.TEXT
+    if file_path.name.endswith(".hdf5") or file_path.name.endswith(".h5") or file_path.name.endswith(".nc"):
+        try:
+            xr_ds = xarray.open_dataset(file_path, engine='h5netcdf', invalid_netcdf=True)
+            xr_ds.close()
+            f_type = FileType.HDF5_NETCDF
+        except Exception:
+            f_type = FileType.HDF5
+    return f_type
+
+def get_file_converters(sync_info : dict, sync_record: SyncRecordManager) -> Dict[str, List[FileConverterHelper]]:
+    converters_to_load = sync_info.get('converters', {})
+    if len(converters_to_load) == 0:
+        sync_record.add_log("No file converters found in the dataset info file.")
+        return {}
+    with sync_record.task("Loading file converters."):
+        converters = {}
+
+        for key in converters_to_load.keys():
+            if converter_naming_scheme.match(key):
+                try:
+                    module_name = converters_to_load[key]['module']
+                    class_name = converters_to_load[key]['class']
+                    converter = get_converter(f"{module_name}:{class_name}")
+                    converter_helper = FileConverterHelper(key, converter)
+                    extension = converter_helper.converter.input_type.lower()
+                    converters.setdefault(extension, []).append(converter_helper)
+                    sync_record.add_log(f"Loaded converter {key}")
+                except KeyError as e:
+                    logger.exception("Converter %s is missing module or class information.", key)
+                    stacktrace = traceback.format_exc()
+                    sync_record.add_error(f"Converter {key} is missing module or class information", e, stacktrace)
+                except NoConvertorException as e:
+                    logger.exception("Failed to load converter %s: %s", key, str(e))
+                    stacktrace = traceback.format_exc()
+                    sync_record.add_error(f"Failed to load converter {key}: {str(e)}", e, stacktrace)
+        return converters

etiket_sync_agent_folderbase/local_sync_record.py

@@ -0,0 +1,91 @@
+import yaml
+import uuid
+import datetime
+
+from etiket_sync_agent.schemas import SyncItemSchema
+from etiket_sync_agent.sync.sync_records.manager import SyncRecordManager
+from etiket_sync_agent.sync.manifests.v2.definitions import QH_MANIFEST_FILE
+from etiket_sync_agent.crud.sync_items import crud_sync_items
+from etiket_sync_agent.db import get_db_session_context
+from pathlib import Path
+
+
+
+class LocalSyncRecord:
+    def __init__(self, dataset_path : Path, syncIdentifier : SyncItemSchema, sync_record: SyncRecordManager):
+        """
+        Small wrapper around the sync record manager to write the local record to a file (with the main function to ensure that the uuid is not swapped of the dataset in the folders.)
+
+        Args:
+            dataset_path (Path): The path to the dataset directory.
+            syncIdentifier (SyncItemSchema): The sync item object.
+        """
+        self.sync_record = sync_record
+        self.sync_time = datetime.datetime.now()
+        self.local_record_path = dataset_path / QH_MANIFEST_FILE
+        self.local_record = generate_empty_manifest(dataset_path,
+                                                syncIdentifier.datasetUUID,
+                                                syncIdentifier.scopeUUID)
+        self._pending_dataset_uuid_update = None
+        self._syncIdentifier = syncIdentifier
+
+        if self.local_record_path.exists():
+            with open(self.local_record_path, 'r', encoding="utf-8") as f:
+                local_record = yaml.safe_load(f)
+            # if the scope uuid is the same as the updated scope uuid, update the manifest
+            manifest_dataset_path = local_record.get('dataset_sync_path', None)
+            manifest_dataset_uuid = local_record.get('dataset_uuid', None)
+            manifest_scope_uuid = local_record.get('scope_uuid', None)
+
+            if manifest_scope_uuid == str(syncIdentifier.scopeUUID):
+                self.local_record = local_record
+
+                if Path(manifest_dataset_path) == dataset_path:
+                    if manifest_dataset_uuid is not None:
+                        if manifest_dataset_uuid != str(syncIdentifier.datasetUUID):
+                            self._pending_dataset_uuid_update = uuid.UUID(manifest_dataset_uuid)
+                            # update the dataset uuid
+                            self.local_record['dataset_uuid'] = manifest_dataset_uuid
+                else:
+                    self.local_record['dataset_sync_path'] = str(dataset_path)
+                    self.local_record['dataset_uuid'] = str(syncIdentifier.datasetUUID)
+
+    async def apply_pending_updates(self):
+        """Apply any pending async CRUD updates (e.g. dataset UUID correction)."""
+        if self._pending_dataset_uuid_update is not None:
+            async with get_db_session_context() as session:
+                await crud_sync_items.update_sync_item(
+                    session, self._syncIdentifier.id,
+                    dataset_uuid=self._pending_dataset_uuid_update)
+            self._syncIdentifier.datasetUUID = self._pending_dataset_uuid_update
+            self._pending_dataset_uuid_update = None
+
+    def write(self):
+        with open(self.local_record_path, 'w', encoding="utf-8") as f:
+            self.local_record['files'] = self.sync_record.to_dict()['files']
+            self.local_record['sync_time'] = self.sync_time.isoformat()
+            yaml.dump(self.local_record, f)
+
+def generate_empty_manifest(root_path: Path, dataset_uuid: uuid.UUID, scope_uuid: uuid.UUID) -> dict:
+    """
+    Generate an empty manifest dictionary.
+
+    This function creates a new manifest dictionary with default values, including version,
+    dataset UUID, dataset synchronization path, synchronization time, and empty files and errors entries.
+
+    Args:
+        root_path (Path): The root path of the dataset.
+        dataset_uuid (uuid.UUID): The unique identifier of the dataset.
+        scope_uuid (uuid.UUID): The unique identifier of the scope.
+
+    Returns:
+        dict: A dictionary representing an empty manifest.
+    """
+    return {
+        'version': 0.1,
+        'dataset_uuid': str(dataset_uuid),
+        'scope_uuid': str(scope_uuid),
+        'dataset_sync_path': str(root_path),
+        'sync_time': datetime.datetime.now().isoformat(),
+        'files': {}
+    }

etiket_sync_agent_folderbase-0.3.0b1.dist-info/METADATA

@@ -0,0 +1,224 @@
+Metadata-Version: 2.4
+Name: etiket_sync_agent_folderbase
+Version: 0.3.0b1
+Summary: Folder-based backend for eTiKeT sync agent
+Author: QHarbor team
+License-Expression: LicenseRef-Proprietary
+Project-URL: Homepage, https://qharbor.nl
+Project-URL: Documentation, https://docs.qharbor.nl
+Keywords: etiket,sync,backend,folderbase,filesystem
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENCE
+Requires-Dist: etiket_sync_agent>=0.3.0b1
+Requires-Dist: xarray
+Requires-Dist: pyyaml
+Dynamic: license-file
+
+# eTiKeT Sync Agent - FolderBase Backend
+
+Backend for synchronizing folder-based datasets with the eTiKeT platform. This backend scans directories for datasets marked with a `_QH_dataset_info.yaml` file and syncs their contents to the cloud.
+
+## How It Works
+
+The FolderBase backend continuously watches a specified folder, automatically detects new and existing datasets, and uploads them to QHarbor. Note that it synchronizes to the server, not from the server.
+
+A folder is recognized as a **dataset** when it contains a `_QH_dataset_info.yaml` file. This file specifies the minimum amount of information needed to create a dataset. Every other file in the folder (and subdirectories) is considered a data file and will be added to the dataset.
+
+### Example Folder Structure
+
+```
+main_folder/
+├── 20240101/
+│   ├── 20240101-211245-165-731d85-experiment_1/
+│   │   ├── _QH_dataset_info.yaml
+│   │   ├── 01-01-2024_01-01-01.json
+│   │   └── 01-01-2024_01-01-01.hdf5
+├── 20240102/
+│   ├── 20240102-220655-268-455d85-experiment_2/
+│   │   ├── _QH_dataset_info.yaml
+│   │   ├── 02-01-2024_02-02-02.json
+│   │   ├── 02-01-2024_02-02-02.hdf5
+│   │   └── analysis/
+│   │       ├── 02-01-2024_02-02-02_analysis.json
+│   │       └── 02-01-2024_02-02-02_analysis.hdf5
+└── some_other_folder/
+    ├── _QH_dataset_info.yaml
+    └── 01-01-2024_01-01-01.json
+```
+
+If a file is added to any of these folders or a new dataset folder is created, the sync agent will automatically detect and upload it.
+
+---
+
+## Installation
+
+```bash
+pip install etiket_sync_agent_folderbase
+```
+
+The package is automatically discovered by `etiket_sync_agent` through the entry-point system.
+
+---
+
+## Configuration
+
+The FolderBase backend requires a `FolderBaseConfigData` configuration:
+
+| Field | Type | Required | Description |
+|-------|------|----------|-------------|
+| `root_directory` | `Path` or `str` | Yes | Root directory to watch for datasets. Supports `~` expansion. |
+| `is_server_folder` | `bool` | Yes | Whether this is a network/server folder (e.g., on a university network drive) |
+
+Please use our flutter GUI or the etiket_sdk to add this sync source.
+
+---
+
+## The `_QH_dataset_info.yaml` File
+
+When performing measurements, we recommend programmatically creating the `_QH_dataset_info.yaml` file in the dataset folder.
+
+### Minimal Example
+
+```yaml
+version: 0.1
+```
+
+### Full Field Reference
+
+| Field | Required | Type | Description |
+|-------|----------|------|-------------|
+| `version` | Yes | `str` | File format version (currently `0.1`) |
+| `dataset_name` | No | `str` | Name of the dataset. Default: folder name |
+| `created` | No | `str` | Creation date in format `YYYY-MM-DDTHH:MM:SS`. Default: earliest file modification time |
+| `collected` | No | `str` | Collection date (alternative to `created`) |
+| `description` | No | `str` | Description of the dataset |
+| `attributes` | No | `dict` | Key-value pairs (values must be `str` or `number`) |
+| `tags` | No | `list` | Tags for the dataset |
+| `skip` | No | `list` | Glob patterns for files/folders to exclude (e.g., `["*.json", "raw_data/*"]`) |
+| `converters` | No | `dict` | File converters to apply (see below) |
+
+### Complete Example
+
+```yaml
+version: 0.1
+dataset_name: 'my_dataset_name'
+description: "Description of the experiment I want to do."
+attributes:
+  initials: 'QH'
+  set_up: 'XLD001'
+  sample: 'my_sample'
+tags: ['rabi', 'test']
+skip: ['*.json', 'raw_data/*']
+converters:
+  csv_to_hdf5_converter:
+    module: etiket_sync_agent_qh_converters
+    class: CSVToHDF5Converter
+```
+
+> ⚠️ **Note**: The YAML file must use **spaces** for indentation, not tabs. Using tabs will cause parsing errors and synchronization will fail.
+
+---
+
+## File Converters
+
+You can specify converters to automatically transform files during sync. The naming convention is `{input}_to_{output}_converter`.
+
+### Converter Syntax
+
+```yaml
+converters:
+  txt_to_csv_converter:
+    module: my_library.location.to.module
+    class: MyConverterClass
+```
+
+### Available Converters
+
+The `etiket_sync_agent_qh_converters` package provides built-in converters:
+- `zarr` → HDF5
+- CSV → HDF5
+- And more...
+
+To create custom converters, implement a class that inherits from `FileConverter` and provides the `convert` method. The converter can be installed with the `etiket_sdk` package. For more information on creating converters, see the `etiket_sync_agent` package documentation.
+
+---
+
+## Programmatic Dataset Creation
+
+You can programmatically create the `_QH_dataset_info.yaml` file using the `generate_dataset_info` function:
+
+```python
+from datetime import datetime
+from etiket_sync_agent_folderbase import generate_dataset_info
+from etiket_sync_agent_qh_converters import CSVToHDF5Converter
+
+path = "my_path/test/"
+generate_dataset_info(
+    path,
+    dataset_name="my_dataset_name",
+    creation=datetime.now(),
+    description="Description of the experiment I want to do.",
+    attributes={"sample": "my_sample"},
+    tags=["rabi", "test"],
+    converters=[CSVToHDF5Converter],
+    skip=["*.json", "raw_data/*"]
+)
+```
+
+> **Note**: This function is also re-exported by the `qdrive` package as `qdrive.dataset.generate_dataset_info`.
+
+See [`dataset_info.py`](etiket_sync_agent_folderbase/dataset_info.py) for the full function signature and documentation.
+
+---
+
+## What Gets Synchronized
+
+| Source | eTiKeT Field | Description |
+|--------|--------------|-------------|
+| `dataset_name` or folder name | `name` | Name of the dataset |
+| `description` | `description` | Dataset description (appended with source path) |
+| `created`/`collected` or earliest file mtime | `collected` | Dataset creation time |
+| `tags` | `tags` | Searchable tags |
+| `attributes` | `attributes` | Key-value metadata |
+| All files (except skipped) | Data files | Uploaded with detected file type |
+
+### Supported File Types
+
+Any file type is supported. For zarr files (which are actually folders), use a converter from `etiket_sync_agent_qh_converters` to convert them to HDF5.
+
+---
+
+## Features
+
+- **Directory-based dataset discovery**: Automatically finds datasets by `_QH_dataset_info.yaml` presence
+- **YAML-based configuration**: Simple declarative dataset metadata
+- **File converter support**: Transform files during sync (e.g., zarr → HDF5, CSV → HDF5)
+- **Skip patterns**: Exclude files/folders using glob patterns
+- **Automatic file type detection**: Detects JSON, text, HDF5/NetCDF files
+- **Subdirectory support**: Syncs all files recursively within dataset folders
+
+## Requirements
+
+- Python >= 3.10
+- xarray
+- h5netcdf
+- PyYAML
+
+## License
+
+Copyright © 2025 QHarbor. All Rights Reserved. See [LICENCE](LICENCE) for details.

etiket_sync_agent_folderbase-0.3.0b1.dist-info/RECORD

@@ -0,0 +1,11 @@
+etiket_sync_agent_folderbase/__init__.py,sha256=LwS_Se8MXrmsKxvseQcIaUIH4AMA9xVYYZ4SrAmuw1Q,242
+etiket_sync_agent_folderbase/dataset_info.py,sha256=ku15iZPOsI_LKKsiHaHCuzGd2E9CKeCwHI7PpCv4fjk,4796
+etiket_sync_agent_folderbase/folderbase_config_class.py,sha256=2_kmE38GVXbVVcBjjuURrM83s0IqY-NewIy9SR4n4qM,3328
+etiket_sync_agent_folderbase/folderbase_sync_class.py,sha256=GESKAjsN4asBDETnyduZpOBnDAduuxWbudtaynAk9cY,13655
+etiket_sync_agent_folderbase/local_sync_record.py,sha256=gv-la6QBYv4yLlZ4zFcAFNCLhHq9ZFF3CEMX0gEdX0Y,4317
+etiket_sync_agent_folderbase-0.3.0b1.dist-info/licenses/LICENCE,sha256=tdZwE43Th9efUgN8-4UpMUyh0kYQ4Dk678agSuhpnc0,2357
+etiket_sync_agent_folderbase-0.3.0b1.dist-info/METADATA,sha256=tzwl5DhOrBc5uiFKgeMSLNTs8JbT5PQ95qULEsu1qms,8030
+etiket_sync_agent_folderbase-0.3.0b1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+etiket_sync_agent_folderbase-0.3.0b1.dist-info/entry_points.txt,sha256=TABh3YgDWyzd6ce1r_1z2V3YY9345KiZxGsEHgqLPKo,97
+etiket_sync_agent_folderbase-0.3.0b1.dist-info/top_level.txt,sha256=_gGRYxwsTiNsFcopQ-1b4Cygybhzct16Cr7a6CtdvlQ,29
+etiket_sync_agent_folderbase-0.3.0b1.dist-info/RECORD,,

etiket_sync_agent_folderbase-0.3.0b1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

etiket_sync_agent_folderbase-0.3.0b1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[etiket_sync_agent.backends]
+folderbase_sync_agent = etiket_sync_agent_folderbase:FolderBaseSync

etiket_sync_agent_folderbase-0.3.0b1.dist-info/licenses/LICENCE

@@ -0,0 +1,34 @@
+All Rights Reserved License
+Copyright ©️ 2024-2026 QHarbor B.V. All Rights Reserved.
+
+Agreement to Terms
+By accessing, downloading, installing, or viewing this Software (including its source code, binaries, or application files), you acknowledge and agree to the terms outlined below.
+
+Terms and Conditions
+This software and its source code (the "Software") are the exclusive property of QHarbor B.V. and are protected by copyright and other intellectual property laws.
+
+License and Testing Exemptions
+Commercial License: If you have entered into a separate commercial license agreement with QHarbor B.V., the terms of that agreement shall supersede the restrictions listed below.
+Testing Permission: If you have obtained written permission for testing/evaluation from QHarbor B.V., you are permitted to install and use the Software for evaluation purposes. However, this permission strictly excludes the right to modify, alter, create derivative works, or reverse engineer the Software.
+
+Prohibited Actions
+Unless explicitly authorized by the exemptions above, you are NOT permitted to:
+	•	Copy, reproduce, or duplicate the Software in any form (except as reasonably necessary for viewing or authorized installation)
+	•	Modify, alter, or create derivative works based on the Software
+	•	Distribute, publish, or share the Software with others
+	•	Reverse engineer, decompile, or disassemble the Software
+	•	Use the Software for any commercial or non-commercial purposes
+	•	Transfer, sell, lease, or sublicense the Software
+	•	Remove or alter any copyright notices or proprietary markings
+
+Viewing Only 
+For those without a commercial license or written testing permission, the Software is made available for viewing and reference purposes only. Any access to view the source code or application does not grant any rights to use, copy, or modify the Software.
+
+No Implied Rights
+No rights are granted by implication, estoppel, or otherwise. All rights not expressly granted are reserved by the copyright holder.
+
+Governing Law 
+This license and any disputes arising from it shall be governed by the laws of the Netherlands.
+
+Disclaimer 
+This Software is provided "AS IS" without warranty of any kind. The copyright holder disclaims all warranties and shall not be liable for any damages arising from the use or inability to use this Software.

etiket_sync_agent_folderbase-0.3.0b1.dist-info/top_level.txt

@@ -0,0 +1 @@
+etiket_sync_agent_folderbase