lamindb_setup 0.72.2__py2.py3-none-any.whl → 0.73.1__py2.py3-none-any.whl

lamindb_setup/__init__.py

@@ -34,7 +34,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.72.2"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.73.1"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

lamindb_setup/_init_instance.py

@@ -16,7 +16,7 @@ from ._silence_loggers import silence_loggers
 from .core import InstanceSettings
 from .core._settings import settings
 from .core._settings_storage import StorageSettings, init_storage
-from .core.upath import convert_pathlike
+from .core.upath import UPath
 
 if TYPE_CHECKING:
     from pydantic import PostgresDsn
@@ -56,6 +56,7 @@ def register_storage_in_instance(ssettings: StorageSettings):
         "region": ssettings.region,
         "instance_uid": instance_uid,
         "created_by_id": current_user_id(),
+        "run": None,
     }
     if ssettings._uid is not None:
         defaults["uid"] = ssettings._uid
@@ -350,7 +351,7 @@ def infer_instance_name(
         return str(db).split("/")[-1]
     if storage == "create-s3":
         raise ValueError("pass name to init if storage = 'create-s3'")
-    storage_path = convert_pathlike(storage)
+    storage_path = UPath(storage)
     if storage_path.name != "":
         name = storage_path.name
     else:

lamindb_setup/_migrate.py

@@ -68,6 +68,8 @@ class migrate:
     @classmethod
     def deploy(cls) -> None:
         """Deploy a migration."""
+        from ._schema_metadata import update_schema_in_hub
+
         if _check_instance_setup():
             raise RuntimeError("Restart Python session to migrate or use CLI!")
         from lamindb_setup.core._hub_client import call_with_fallback_auth
@@ -104,6 +106,9 @@ class migrate:
         # this populates the hub
         if instance_is_on_hub:
             logger.important(f"updating lamindb version in hub: {lamindb.__version__}")
+            # TODO: integrate update of instance table within update_schema_in_hub & below
+            if settings.instance.dialect != "sqlite":
+                update_schema_in_hub()
             call_with_fallback_auth(
                 update_instance,
                 instance_id=settings.instance._id.hex,

lamindb_setup/_schema_metadata.py

@@ -0,0 +1,479 @@
+from __future__ import annotations
+
+import hashlib
+import importlib
+import json
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Dict
+from uuid import UUID
+
+import sqlparse
+from django.contrib.postgres.expressions import ArraySubquery
+from django.db.models import (
+    Field,
+    ForeignObjectRel,
+    ManyToManyField,
+    ManyToManyRel,
+    OuterRef,
+    QuerySet,
+    Subquery,
+)
+from django.db.models.functions import JSONObject
+from sqlparse.sql import Identifier, IdentifierList
+from sqlparse.tokens import DML, Keyword
+
+from lamindb_setup import settings
+from lamindb_setup._init_instance import get_schema_module_name
+from lamindb_setup.core._hub_client import call_with_fallback_auth
+
+if TYPE_CHECKING:
+    from lnschema_core.models import Registry
+    from supabase import Client
+
+
+def update_schema_in_hub() -> tuple[bool, UUID, dict]:
+    return call_with_fallback_auth(_synchronize_schema)
+
+
+def _synchronize_schema(client: Client) -> tuple[bool, UUID, dict]:
+    schema_metadata = SchemaMetadata()
+    schema_metadata_dict = schema_metadata.to_json()
+    schema_uuid = _dict_to_uuid(schema_metadata_dict)
+    schema = _get_schema_by_id(schema_uuid, client)
+
+    is_new = schema is None
+    if is_new:
+        module_set_info = schema_metadata._get_module_set_info()
+        module_ids = "-".join(str(module_info["id"]) for module_info in module_set_info)
+        schema = (
+            client.table("schema")
+            .insert(
+                {
+                    "id": schema_uuid.hex,
+                    "module_ids": module_ids,
+                    "module_set_info": module_set_info,
+                    "json": schema_metadata_dict,
+                }
+            )
+            .execute()
+            .data[0]
+        )
+
+    instance_response = (
+        client.table("instance")
+        .update({"schema_id": schema_uuid.hex})
+        .eq("id", settings.instance._id.hex)
+        .execute()
+    )
+    assert (
+        len(instance_response.data) == 1
+    ), f"schema of instance {settings.instance._id.hex} could not be updated with schema {schema_uuid.hex}"
+
+    return is_new, schema_uuid, schema
+
+
+def get_schema_by_id(id: UUID):
+    return call_with_fallback_auth(_get_schema_by_id, id=id)
+
+
+def _get_schema_by_id(id: UUID, client: Client):
+    response = client.table("schema").select("*").eq("id", id.hex).execute()
+    if len(response.data) == 0:
+        return None
+    return response.data[0]
+
+
+def _dict_to_uuid(dict: dict):
+    encoded = json.dumps(dict, sort_keys=True).encode("utf-8")
+    hash = hashlib.md5(encoded).digest()
+    uuid = UUID(bytes=hash[:16])
+    return uuid
+
+
+class SchemaMetadata:
+    def __init__(self) -> None:
+        self.included_modules = ["core"] + list(settings.instance.schema)
+        self.modules = self._get_modules_metadata()
+
+    def to_dict(
+        self, include_django_objects: bool = True, include_select_terms: bool = True
+    ):
+        return {
+            module_name: {
+                model_name: model.to_dict(include_django_objects, include_select_terms)
+                for model_name, model in module.items()
+            }
+            for module_name, module in self.modules.items()
+        }
+
+    def to_json(self, include_select_terms: bool = True):
+        return self.to_dict(
+            include_django_objects=False, include_select_terms=include_select_terms
+        )
+
+    def _get_modules_metadata(self):
+        return {
+            module_name: {
+                model._meta.model_name: ModelMetadata(
+                    model, module_name, self.included_modules
+                )
+                for model in self._get_schema_module(
+                    module_name
+                ).models.__dict__.values()
+                if model.__class__.__name__ == "ModelBase"
+                and model.__name__ not in ["Registry", "ORM"]
+                and not model._meta.abstract
+                and model.__get_schema_name__() == module_name
+            }
+            for module_name in self.included_modules
+        }
+
+    def _get_module_set_info(self):
+        # TODO: rely on schemamodule table for this
+        module_set_info = []
+        for module_name in self.included_modules:
+            module = self._get_schema_module(module_name)
+            module_set_info.append(
+                {"id": 0, "name": module_name, "version": module.__version__}
+            )
+        return module_set_info
+
+    @staticmethod
+    def _get_schema_module(module_name):
+        return importlib.import_module(get_schema_module_name(module_name))
+
+
+@dataclass
+class FieldMetadata:
+    schema_name: str
+    model_name: str
+    field_name: str
+    type: str
+    is_link_table: bool
+    column: str | None = None
+    relation_type: str | None = None
+    related_schema_name: str | None = None
+    related_model_name: str | None = None
+    related_field_name: str | None = None
+    through: dict | None = None
+
+
+class ModelRelations:
+    def __init__(self, fields: list[ForeignObjectRel]) -> None:
+        self.many_to_one = {}
+        self.one_to_many = {}
+        self.many_to_many = {}
+        self.one_to_one = {}
+
+        for field in fields:
+            if field.many_to_one:
+                self.many_to_one.update({field.name: field})
+            elif field.one_to_many:
+                self.one_to_many.update({field.name: field})
+            elif field.many_to_many:
+                self.many_to_many.update({field.name: field})
+            elif field.one_to_one:
+                self.one_to_one.update({field.name: field})
+
+        self.all = {
+            **self.many_to_one,
+            **self.one_to_many,
+            **self.many_to_many,
+            **self.one_to_one,
+        }
+
+
+class ModelMetadata:
+    def __init__(self, model, module_name: str, included_modules: list[str]) -> None:
+        self.model = model
+        self.class_name = model.__name__
+        self.module_name = module_name
+        self.model_name = model._meta.model_name
+        self.table_name = model._meta.db_table
+        self.included_modules = included_modules
+        self.fields, self.relations = self._get_fields_metadata(self.model)
+
+    def to_dict(
+        self, include_django_objects: bool = True, include_select_terms: bool = True
+    ):
+        _dict = {
+            "fields": self.fields.copy(),
+            "class_name": self.class_name,
+            "table_name": self.table_name,
+        }
+
+        select_terms = self.select_terms if include_select_terms else []
+
+        for field_name in self.fields.keys():
+            _dict["fields"][field_name] = _dict["fields"][field_name].__dict__
+            if field_name in select_terms:
+                _dict["fields"][field_name].update(
+                    {"select_term": select_terms[field_name]}
+                )
+
+        if include_django_objects:
+            _dict.update({"model": self.model})
+
+        return _dict
+
+    @property
+    def select_terms(self):
+        return (
+            DjangoQueryBuilder(self.module_name, self.model_name)
+            .add_all_sub_queries()
+            .extract_select_terms()
+        )
+
+    def _get_fields_metadata(self, model):
+        related_fields = []
+        fields_metadata: dict[str, FieldMetadata] = {}
+
+        for field in model._meta.get_fields():
+            field_metadata = self._get_field_metadata(model, field)
+            if field_metadata.related_schema_name is None:
+                fields_metadata.update({field.name: field_metadata})
+
+            if (
+                field_metadata.related_schema_name in self.included_modules
+                and field_metadata.schema_name in self.included_modules
+            ):
+                related_fields.append(field)
+
+        model_relations_metadata = ModelRelations(related_fields)
+
+        related_fields_metadata = self._get_related_fields_metadata(
+            model, model_relations_metadata
+        )
+
+        fields_metadata = {**fields_metadata, **related_fields_metadata}
+
+        return fields_metadata, model_relations_metadata
+
+    def _get_related_fields_metadata(
+        self, model, model_relations_metadata: ModelRelations
+    ):
+        related_fields: dict[str, FieldMetadata] = {}
+
+        # Many to one (foreign key defined in the model)
+        for link_field_name, link_field in model_relations_metadata.many_to_one.items():
+            related_fields.update(
+                {f"{link_field_name}": self._get_field_metadata(model, link_field)}
+            )
+            for field in link_field.related_model._meta.fields:
+                related_fields.update(
+                    {
+                        f"{link_field_name}__{field.name}": self._get_field_metadata(
+                            model, field
+                        )
+                    }
+                )
+
+        # One to many (foreign key defined in the related model)
+        for relation_name, relation in model_relations_metadata.one_to_many.items():
+            # exclude self reference as it is already included in the many to one
+            if relation.related_model == model:
+                continue
+            related_fields.update(
+                {f"{relation_name}": self._get_field_metadata(model, relation.field)}
+            )
+
+        # One to one
+        for link_field_name, link_field in model_relations_metadata.one_to_one.items():
+            related_fields.update(
+                {f"{link_field_name}": self._get_field_metadata(model, link_field)}
+            )
+            for field in link_field.related_model._meta.fields:
+                related_fields.update(
+                    {
+                        f"{link_field_name}__{field.name}": self._get_field_metadata(
+                            model, field
+                        )
+                    }
+                )
+
+        # Many to many
+        for (
+            link_field_name,
+            link_field,
+        ) in model_relations_metadata.many_to_many.items():
+            related_fields.update(
+                {f"{link_field_name}": self._get_field_metadata(model, link_field)}
+            )
+
+        return related_fields
+
+    def _get_field_metadata(self, model, field: Field):
+        from lnschema_core.models import LinkORM
+
+        internal_type = field.get_internal_type()
+        model_name = field.model._meta.model_name
+        relation_type = self._get_relation_type(model, field)
+        if field.related_model is None:
+            schema_name = field.model.__get_schema_name__()
+            related_model_name = None
+            related_schema_name = None
+            related_field_name = None
+            field_name = field.name
+        else:
+            related_model_name = field.related_model._meta.model_name
+            related_schema_name = field.related_model.__get_schema_name__()
+            schema_name = field.model.__get_schema_name__()
+            related_field_name = field.remote_field.name
+            field_name = field.name
+
+        if relation_type in ["one-to-many"]:
+            # For a one-to-many relation, the field belong
+            # to the other model as a foreign key.
+            # To make usage similar to other relation types
+            # we need to invert model and related model.
+            schema_name, related_schema_name = related_schema_name, schema_name
+            model_name, related_model_name = related_model_name, model_name
+            field_name, related_field_name = related_field_name, field_name
+            pass
+
+        column = None
+        if relation_type not in ["many-to-many", "one-to-one", "one-to-many"]:
+            column = field.column
+
+        through = None
+        if relation_type == "many-to-many":
+            through = self._get_through(model, field)
+
+        return FieldMetadata(
+            schema_name,
+            model_name,
+            field_name,
+            internal_type,
+            issubclass(field.model, LinkORM),
+            column,
+            relation_type,
+            related_schema_name,
+            related_model_name,
+            related_field_name,
+            through,
+        )
+
+    @staticmethod
+    def _get_through(model, field_or_rel: ManyToManyField | ManyToManyRel):
+        table_name = model._meta.db_table
+        related_table_name = field_or_rel.related_model._meta.db_table
+
+        if isinstance(field_or_rel, ManyToManyField):
+            return {
+                "link_table_name": field_or_rel.remote_field.through._meta.db_table,
+                table_name: field_or_rel.m2m_column_name(),
+                related_table_name: field_or_rel.m2m_reverse_name(),
+            }
+
+        if isinstance(field_or_rel, ManyToManyRel):
+            return {
+                "link_table_name": field_or_rel.through._meta.db_table,
+                table_name: field_or_rel.field.m2m_column_name(),
+                related_table_name: field_or_rel.field.m2m_reverse_name(),
+            }
+
+    @staticmethod
+    def _get_relation_type(model, field: Field):
+        if field.many_to_one:
+            # defined in the model
+            if model == field.model:
+                return "many-to-one"
+            # defined in the related model
+            else:
+                return "one-to-many"
+        elif field.one_to_many:
+            return "one-to-many"
+        elif field.many_to_many:
+            return "many-to-many"
+        elif field.one_to_one:
+            return "one-to-one"
+        else:
+            return None
+
+
+class DjangoQueryBuilder:
+    def __init__(
+        self, module_name: str, model_name: str, query_set: QuerySet | None = None
+    ) -> None:
+        self.schema_metadata = SchemaMetadata()
+        self.module_name = module_name
+        self.model_name = model_name
+        self.model_metadata = self.schema_metadata.modules[module_name][model_name]
+        self.query_set = query_set if query_set else self.model_metadata.model.objects
+
+    def add_all_sub_queries(self):
+        all_fields = self.model_metadata.fields
+        included_relations = [
+            field_name
+            for field_name, field in all_fields.items()
+            if field.relation_type is not None
+        ]
+        self.add_sub_queries(included_relations)
+        return self
+
+    def add_sub_queries(self, included_relations: list[str]):
+        sub_queries = {
+            f"annotated_{relation_name}": self._get_sub_query(
+                self.model_metadata.fields[relation_name]
+            )
+            for relation_name in included_relations
+        }
+        self.query_set = self.query_set.annotate(**sub_queries)
+        return self
+
+    def extract_select_terms(self):
+        parsed = sqlparse.parse(self.sql_query)
+        select_found = False
+        select_terms = {}
+
+        def get_name(identifier):
+            name = identifier.get_name()
+            return name if name is not None else str(identifier).split(".")
+
+        for token in parsed[0].tokens:
+            if token.ttype is DML and token.value.upper() == "SELECT":
+                select_found = True
+            elif select_found and isinstance(token, IdentifierList):
+                for identifier in token.get_identifiers():
+                    select_terms[get_name(identifier)] = str(identifier)
+            elif select_found and isinstance(token, Identifier):
+                select_terms[get_name(token)] = str(token)
+            elif token.ttype is Keyword:
+                if token.value.upper() in ["FROM", "WHERE", "GROUP BY", "ORDER BY"]:
+                    break
+
+        return select_terms
+
+    def _get_sub_query(self, field_metadata: FieldMetadata):
+        module_name = field_metadata.related_schema_name
+        model_name = field_metadata.related_model_name
+        field_name = field_metadata.related_field_name
+        model_metadata = self.schema_metadata.modules[module_name][model_name]
+        query_set = model_metadata.model.objects.get_queryset()
+        select = {
+            field_name: field_name
+            for field_name in model_metadata.fields.keys()
+            if model_metadata.fields[field_name].relation_type is None
+            and "__" not in field_name
+        }
+
+        if field_metadata.relation_type in ["many-to-many", "one-to-many"]:
+            return ArraySubquery(
+                Subquery(
+                    query_set.filter(**{field_name: OuterRef("pk")}).values(
+                        data=JSONObject(**select)
+                    )[:5]
+                )
+            )
+        if field_metadata.relation_type in ["many-to-one", "one-to-one"]:
+            return Subquery(
+                query_set.filter(**{field_name: OuterRef("pk")}).values(
+                    data=JSONObject(**select)
+                )[:5]
+            )
+
+    @property
+    def sql_query(self):
+        sql_template, params = self.query_set.query.sql_with_params()
+        sql_query = sql_template % tuple(f"'{p}'" for p in params)
+        return sql_query.replace("annotated_", "")

lamindb_setup/core/_aws_credentials.py

@@ -67,8 +67,8 @@ class AWSCredentialsManager:
     def _path_inject_options(self, path: S3Path, credentials: dict) -> S3Path:
         if credentials == {}:
             # credentials were specified manually for the path
-            if "anon" in path._kwargs:
-                anon = path._kwargs["anon"]
+            if "anon" in path.storage_options:
+                anon = path.storage_options["anon"]
             elif path.fs.key is not None and path.fs.secret is not None:
                 anon = False
             else:
@@ -77,8 +77,8 @@ class AWSCredentialsManager:
         else:
             connection_options = credentials
 
-        if "cache_regions" in path._kwargs:
-            cache_regions = path._kwargs["cache_regions"]
+        if "cache_regions" in path.storage_options:
+            cache_regions = path.storage_options["cache_regions"]
         else:
             cache_regions = True
 

lamindb_setup/core/_settings_instance.py

@@ -18,7 +18,7 @@ from .cloud_sqlite_locker import (
     EXPIRATION_TIME,
     InstanceLockedException,
 )
-from .upath import LocalPathClasses, UPath, convert_pathlike
+from .upath import LocalPathClasses, UPath
 
 if TYPE_CHECKING:
     from uuid import UUID
@@ -205,7 +205,7 @@ class InstanceSettings:
                 )
             if response != "y":
                 return None
-        local_root = convert_pathlike(local_root)
+        local_root = UPath(local_root)
         assert isinstance(local_root, LocalPathClasses)
         self._storage_local = init_storage(local_root, self._id, register_hub=True)  # type: ignore
         register_storage_in_instance(self._storage_local)  # type: ignore

lamindb_setup/core/_settings_storage.py

@@ -17,7 +17,6 @@ from ._settings_store import system_storage_settings_file
 from .upath import (
     LocalPathClasses,
     UPath,
-    convert_pathlike,
     create_path,
 )
 
@@ -37,40 +36,43 @@ def base62(n_char: int) -> str:
     return id
 
 
-def get_storage_region(storage_root: UPathStr) -> str | None:
-    storage_root_str = str(storage_root)
-    if storage_root_str.startswith("s3://"):
-        import botocore.session as session
+def get_storage_region(path: UPathStr) -> str | None:
+    path_str = str(path)
+    if path_str.startswith("s3://"):
+        import botocore.session
         from botocore.config import Config
-        from botocore.exceptions import NoCredentialsError
+        from botocore.exceptions import ClientError
 
         # strip the prefix and any suffixes of the bucket name
-        bucket = storage_root_str.replace("s3://", "").split("/")[0]
-        s3_session = session.get_session()
-        s3_client = s3_session.create_client("s3")
+        bucket = path_str.replace("s3://", "").split("/")[0]
+        session = botocore.session.get_session()
+        credentials = session.get_credentials()
+        if credentials is None or credentials.access_key is None:
+            config = Config(signature_version=botocore.session.UNSIGNED)
+        else:
+            config = None
+        s3_client = session.create_client("s3", config=config)
         try:
             response = s3_client.head_bucket(Bucket=bucket)
-        except NoCredentialsError:  # deal with anonymous access
-            s3_client = s3_session.create_client(
-                "s3", config=Config(signature_version=session.UNSIGNED)
-            )
-            response = s3_client.head_bucket(Bucket=bucket)
-        storage_region = response["ResponseMetadata"].get("HTTPHeaders", {})[
-            "x-amz-bucket-region"
-        ]
-        # if we want to except botcore.exceptions.ClientError to reformat an
-        # error message, this is how to do test for the "NoSuchBucket" error:
-        #     exc.response["Error"]["Code"] == "NoSuchBucket"
+        except ClientError as exc:
+            response = getattr(exc, "response", {})
+            if response.get("Error", {}).get("Code") == "404":
+                raise exc
+        region = (
+            response.get("ResponseMetadata", {})
+            .get("HTTPHeaders", {})
+            .get("x-amz-bucket-region")
+        )
     else:
-        storage_region = None
-    return storage_region
+        region = None
+    return region
 
 
 def mark_storage_root(root: UPathStr, uid: str):
     # we need to touch a 0-byte object in folder-like storage location on S3 to avoid
     # permission errors from leveraging s3fs on an empty hosted storage location
     # for consistency, we write this file everywhere
-    root_upath = convert_pathlike(root)
+    root_upath = UPath(root)
     mark_upath = root_upath / IS_INITIALIZED_KEY
     mark_upath.write_text(uid)
 
@@ -156,7 +158,7 @@ class StorageSettings:
     ):
         self._uid = uid
         self._uuid_ = uuid
-        self._root_init = convert_pathlike(root)
+        self._root_init = UPath(root)
         if isinstance(self._root_init, LocalPathClasses):  # local paths
             try:
                 (self._root_init / ".lamindb").mkdir(parents=True, exist_ok=True)

lamindb_setup/core/_setup_bionty_sources.py

@@ -53,6 +53,7 @@ def write_bionty_sources(isettings: InstanceSettings) -> None:
             kwargs["species"] = kwargs.pop("organism")
         elif hasattr(PublicSource, "organism") and "species" in kwargs:
             kwargs["organism"] = kwargs.pop("species")
+        kwargs["run"] = None  # can't yet access tracking information
         record = PublicSource(**kwargs)
         all_records.append(record)
 

lamindb_setup/core/upath.py

@@ -59,6 +59,7 @@ VALID_SUFFIXES = {
 VALID_COMPOSITE_SUFFIXES = {
     ".anndata.zarr",
     ".spatialdata.zarr",
+    ".ome.zarr",
 }
 
 TRAILING_SEP = (os.sep, os.altsep) if os.altsep is not None else os.sep
@@ -155,7 +156,10 @@ def create_mapper(
 
 
 def print_hook(size: int, value: int, objectname: str, action: str):
-    progress_in_percent = (value / size) * 100
+    if size == 0:
+        progress_in_percent = 100.0
+    else:
+        progress_in_percent = (value / size) * 100
     out = f"... {action} {objectname}:" f" {min(progress_in_percent, 100):4.1f}%"
     if "NBPRJ_TEST_NBPATH" not in os.environ:
         end = "\n" if progress_in_percent >= 100 else "\r"
@@ -238,50 +242,70 @@ class ChildProgressCallback(fsspec.callbacks.Callback):
         self.parent.update_relative_value(inc)
 
     def relative_update(self, inc=1):
-        self.parent_update(inc / self.size)
+        if self.size != 0:
+            self.parent_update(inc / self.size)
+        else:
+            self.parent_update(1)
 
 
-def download_to(self, path: UPathStr, print_progress: bool = False, **kwargs):
-    """Download to a path."""
+def download_to(self, local_path: UPathStr, print_progress: bool = True, **kwargs):
+    """Download from self (a destination in the cloud) to the local path."""
+    if "recursive" not in kwargs:
+        kwargs["recursive"] = True
     if print_progress and "callback" not in kwargs:
         callback = ProgressCallback(
-            PurePosixPath(path).name, "downloading", adjust_size=True
+            PurePosixPath(local_path).name, "downloading", adjust_size=True
         )
         kwargs["callback"] = callback
 
-    self.fs.download(str(self), str(path), **kwargs)
+    self.fs.download(str(self), str(local_path), **kwargs)
 
 
 def upload_from(
     self,
-    path: UPathStr,
-    dir_inplace: bool = False,
-    print_progress: bool = False,
+    local_path: UPathStr,
+    create_folder: bool | None = None,
+    print_progress: bool = True,
     **kwargs,
-):
-    """Upload from a local path."""
-    path = Path(path)
-    path_is_dir = path.is_dir()
-    if not path_is_dir:
-        dir_inplace = False
+) -> UPath:
+    """Upload from the local path to `self` (a destination in the cloud).
+
+    If the local path is a directory, recursively upload its contents.
+
+    Args:
+        local_path: A local path of a file or directory.
+        create_folder: Only applies if `local_path` is a directory and then
+            defaults to `True`. If `True`, make a new folder in the destination
+            using the directory name of `local_path`. If `False`, upload the
+            contents of the directory to to the root-level of the destination.
+        print_progress: Print progress.
+
+    Returns:
+        The destination path.
+    """
+    local_path = Path(local_path)
+    local_path_is_dir = local_path.is_dir()
+    if create_folder is None:
+        create_folder = local_path_is_dir
+    if create_folder and not local_path_is_dir:
+        raise ValueError("create_folder can only be True if local_path is a directory")
 
     if print_progress and "callback" not in kwargs:
-        callback = ProgressCallback(path.name, "uploading")
+        callback = ProgressCallback(local_path.name, "uploading")
         kwargs["callback"] = callback
 
-    if dir_inplace:
-        source = [f for f in path.rglob("*") if f.is_file()]
-        destination = [str(self / f.relative_to(path)) for f in source]
+    if local_path_is_dir and not create_folder:
+        source = [f for f in local_path.rglob("*") if f.is_file()]
+        destination = [str(self / f.relative_to(local_path)) for f in source]
         source = [str(f) for f in source]  # type: ignore
     else:
-        source = str(path)  # type: ignore
+        source = str(local_path)  # type: ignore
         destination = str(self)  # type: ignore
-    # this weird thing is to avoid s3fs triggering create_bucket in upload
-    # if dirs are present
-    # it allows to avoid permission error
-    if self.protocol != "s3" or not path_is_dir or dir_inplace:
-        cleanup_cache = False
-    else:
+
+    # the below lines are to avoid s3fs triggering create_bucket in upload if
+    # dirs are present it allows to avoid permission error
+    # would be easier to just
+    if self.protocol == "s3" and local_path_is_dir and create_folder:
         bucket = self._url.netloc
         if bucket not in self.fs.dircache:
             self.fs.dircache[bucket] = [{}]
@@ -290,14 +314,21 @@ def upload_from(
             cleanup_cache = True
         else:
             cleanup_cache = False
+    else:
+        cleanup_cache = False
 
-    self.fs.upload(source, destination, **kwargs)
+    self.fs.upload(source, destination, recursive=create_folder, **kwargs)
 
     if cleanup_cache:
         # normally this is invalidated after the upload but still better to check
         if bucket in self.fs.dircache:
             del self.fs.dircache[bucket]
 
+    if local_path_is_dir and create_folder:
+        return self / local_path.name
+    else:
+        return self
+
 
 def synchronize(
     self,
@@ -305,14 +336,14 @@ def synchronize(
     error_no_origin: bool = True,
     print_progress: bool = False,
     callback: fsspec.callbacks.Callback | None = None,
-    **kwargs,
+    timestamp: float | None = None,
 ):
     """Sync to a local destination path."""
     # optimize the number of network requests
-    if "timestamp" in kwargs:
+    if timestamp is not None:
         is_dir = False
         exists = True
-        cloud_mts = kwargs.pop("timestamp")
+        cloud_mts = timestamp
     else:
         # perform only one network request to check existence, type and timestamp
         try:
@@ -378,8 +409,8 @@ def synchronize(
                 origin = f"{self.protocol}://{file}"
                 destination = objectpath / file_key
                 child = callback.branched(origin, destination.as_posix())
-                UPath(origin, **self._kwargs).synchronize(
-                    destination, timestamp=timestamp, callback=child, **kwargs
+                UPath(origin, **self.storage_options).synchronize(
+                    destination, callback=child, timestamp=timestamp
                 )
                 child.close()
             if destination_exists:
@@ -400,15 +431,16 @@ def synchronize(
     callback = ProgressCallback.requires_progress(
         callback, print_progress, objectpath.name, "synchronizing"
     )
-    kwargs["callback"] = callback
     if objectpath.exists():
-        local_mts = objectpath.stat().st_mtime  # type: ignore
-        need_synchronize = cloud_mts > local_mts
+        local_mts_obj = objectpath.stat().st_mtime  # type: ignore
+        need_synchronize = cloud_mts > local_mts_obj
     else:
         objectpath.parent.mkdir(parents=True, exist_ok=True)
         need_synchronize = True
     if need_synchronize:
-        self.download_to(objectpath, **kwargs)
+        self.download_to(
+            objectpath, recursive=False, print_progress=False, callback=callback
+        )
         os.utime(objectpath, times=(cloud_mts, cloud_mts))
     else:
         # nothing happens if parent_update is not defined
@@ -477,7 +509,7 @@ def compute_file_tree(
             if child_path.is_dir():
                 if include_dirs and child_path not in include_dirs:
                     continue
-                yield prefix + pointer + child_path.name
+                yield prefix + pointer + child_path.name + "/"
                 n_directories += 1
                 n_files_per_dir_and_type = defaultdict(lambda: 0)
                 extension = branch if pointer == tee else space
@@ -657,22 +689,8 @@ Args:
 """
 
 
-def convert_pathlike(pathlike: UPathStr) -> UPath:
-    """Convert pathlike to Path or UPath inheriting options from root."""
-    if isinstance(pathlike, (str, UPath)):
-        path = UPath(pathlike)
-    elif isinstance(pathlike, Path):
-        path = UPath(str(pathlike))  # UPath applied on Path gives Path back
-    else:
-        raise ValueError("pathlike should be of type UPathStr")
-    # remove trailing slash
-    if path._parts and path._parts[-1] == "":
-        path._parts = path._parts[:-1]
-    return path
-
-
 def create_path(path: UPath, access_token: str | None = None) -> UPath:
-    path = convert_pathlike(path)
+    path = UPath(path)
     # test whether we have an AWS S3 path
     if not isinstance(path, S3Path):
         return path
@@ -681,15 +699,16 @@ def create_path(path: UPath, access_token: str | None = None) -> UPath:
 
 def get_stat_file_cloud(stat: dict) -> tuple[int, str, str]:
     size = stat["size"]
+    etag = stat["ETag"]
     # small files
-    if "-" not in stat["ETag"]:
+    if "-" not in etag:
         # only store hash for non-multipart uploads
         # we can't rapidly validate multi-part uploaded files client-side
         # we can add more logic later down-the-road
-        hash = b16_to_b64(stat["ETag"])
+        hash = b16_to_b64(etag)
         hash_type = "md5"
     else:
-        stripped_etag, suffix = stat["ETag"].split("-")
+        stripped_etag, suffix = etag.split("-")
         suffix = suffix.strip('"')
         hash = f"{b16_to_b64(stripped_etag)}-{suffix}"
         hash_type = "md5-n"  # this is the S3 chunk-hashing strategy
@@ -721,7 +740,7 @@ class InstanceNotEmpty(Exception):
 def check_storage_is_empty(
     root: UPathStr, *, raise_error: bool = True, account_for_sqlite_file: bool = False
 ) -> int:
-    root_upath = convert_pathlike(root)
+    root_upath = UPath(root)
     root_string = root_upath.as_posix()  # type: ignore
     # we currently touch a 0-byte file in the root of a hosted storage location
     # ({storage_root}/.lamindb/_is_initialized) during storage initialization

{lamindb_setup-0.72.2.dist-info → lamindb_setup-0.73.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.72.2
+Version: 0.73.1
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <laminlabs@gmail.com>
 Description-Content-Type: text/markdown
@@ -11,7 +11,7 @@ Requires-Dist: dj_database_url>=1.3.0,<3.0.0
 Requires-Dist: pydantic[dotenv]<2.0.0
 Requires-Dist: appdirs<2.0.0
 Requires-Dist: requests
-Requires-Dist: universal_pathlib==0.1.4
+Requires-Dist: universal_pathlib==0.2.2
 Requires-Dist: botocore<2.0.0
 Requires-Dist: supabase==2.2.1
 Requires-Dist: urllib3<2 ; extra == "aws"

{lamindb_setup-0.72.2.dist-info → lamindb_setup-0.73.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-lamindb_setup/__init__.py,sha256=p_LDSrN8hygUj_CeoWH3ZILNhJi_GQVghFfu7pSmt5w,1542
+lamindb_setup/__init__.py,sha256=XYsA4Sk8k2514_kS3XNzBJs1ZXhDkUcrd1rQTzwsan4,1542
 lamindb_setup/_cache.py,sha256=wA7mbysANwe8hPNbjDo9bOmXJ0xIyaS5iyxIpxSWji4,846
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=cNEL9Q4yPpmEkGKHH8JgullWl1VUZwALJ4RHn9wZypY,2613
@@ -8,15 +8,16 @@ lamindb_setup/_delete.py,sha256=Y8KSFYgY0UHAvjd7cCL6hZ_XiLeJwx50BguVATcj_Xo,5524
 lamindb_setup/_django.py,sha256=EoyWvFzH0i9wxjy4JZhcoXCTckztP_Mrl6FbYQnMmLE,1534
 lamindb_setup/_exportdb.py,sha256=uTIZjKKTB7arzEr1j0O6lONiT2pRBKeOFdLvOV8ZwzE,2120
 lamindb_setup/_importdb.py,sha256=yYYShzUajTsR-cTW4CZ-UNDWZY2uE5PAgNbp-wn8Ogc,1874
-lamindb_setup/_init_instance.py,sha256=lR-6txbf3Z2O7ki-DMZWFg36QNiUZ_B5lc6JXjScdus,11897
-lamindb_setup/_migrate.py,sha256=4nBTFg5-BK4A2gH-D3_tcFf8EtvMnIo5Mq0e_C6_9-U,8815
+lamindb_setup/_init_instance.py,sha256=Cji3h2kCj8Meukkm8Btl8Lu4Jio4NiWLlzAAKukEzYY,11896
+lamindb_setup/_migrate.py,sha256=49xPkwU-QQjpq0xSUepiVfvRsMhmjfbRlhd72YIq8o8,9059
 lamindb_setup/_register_instance.py,sha256=Jeu0wyvJVSVQ_n-A_7yn7xOZIP0ncJD92DRABqzPIjA,940
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
+lamindb_setup/_schema_metadata.py,sha256=G3yXJ46OkVGqHxccCgjvQnnBNKg1uhwxXB1CddlkwYw,16882
 lamindb_setup/_set_managed_storage.py,sha256=mNZrANn-9rwZ0oGWxxg0wS0T0VOQCWyo4nSSyNAE15Q,1419
 lamindb_setup/_setup_user.py,sha256=6Oc7Rke-yRQSZbuntdUAz8QbJ6UuPzYHI9FnYlf_q-A,3670
 lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
 lamindb_setup/core/__init__.py,sha256=dV9S-rQpNK9JcBn4hiEmiLnmNqfpPFJD9pqagMCaIew,416
-lamindb_setup/core/_aws_credentials.py,sha256=nK04-lNYz6MYDgD6Z56peoxCULZ82uFDRMzjwsPh25U,5293
+lamindb_setup/core/_aws_credentials.py,sha256=uKMQO9q42Hnepz8aj3RxwLKDWUJx8pNOYrFnnNh5X40,5325
 lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
 lamindb_setup/core/_deprecated.py,sha256=3qxUI1dnDlSeR0BYrv7ucjqRBEojbqotPgpShXs4KF8,2520
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
@@ -25,20 +26,20 @@ lamindb_setup/core/_hub_core.py,sha256=RGjTqf1owuWmkXAYy0EPaoHAaJ-0T0hAidkqa3cId
 lamindb_setup/core/_hub_crud.py,sha256=b1XF7AJpM9Q-ttm9nPG-r3OTRWHQaGzAGIyvmb83NTo,4859
 lamindb_setup/core/_hub_utils.py,sha256=b_M1LkdCjiMWm1EOlSb9GuPdLijwVgQDtATTpeZuXI0,1875
 lamindb_setup/core/_settings.py,sha256=jjZ_AxRXB3Y3UP6m04BAw_dhFbJbdg2-nZWmEv2LNZ8,3141
-lamindb_setup/core/_settings_instance.py,sha256=kda3kqUtwh-XZMDmbdFIp8RGGFw0Az8T8g2SKEU65mo,16949
+lamindb_setup/core/_settings_instance.py,sha256=O9TtGijSJCXMREePegrxlQmHzKJFbgpC8yrlOe4BwJo,16920
 lamindb_setup/core/_settings_load.py,sha256=NGgCDpN85j1EqoKlrYFIlZBMlBJm33gx2-wc96CP_ZQ,3922
 lamindb_setup/core/_settings_save.py,sha256=d1A-Ex-7H08mb8l7I0Oe0j0GilrfaDuprh_NMxhQAsQ,2704
-lamindb_setup/core/_settings_storage.py,sha256=7f0jt1zcSltpOYDPQ5CVvbBon_d7aneKTte935-2REY,13236
+lamindb_setup/core/_settings_storage.py,sha256=k4XyJR6_KpUpQuBYZp4mEdABiT91gTTfbK7tAVwqZCA,13093
 lamindb_setup/core/_settings_store.py,sha256=dagS5c7wAMRnuZTRfCU4sKaIOyF_HwAP5Fnnn8vphno,2084
 lamindb_setup/core/_settings_user.py,sha256=P2lC4WDRAFfT-Xq3MlXJ-wMKIHCoGNhMTQfRGIAyUNQ,1344
-lamindb_setup/core/_setup_bionty_sources.py,sha256=OgPpZxN2_Wffy-ogEBz_97c_k8d2bD-DDVt89-u9GLY,3002
+lamindb_setup/core/_setup_bionty_sources.py,sha256=h_pBANsSGK6ujAFsG21mtADHVJoMLKDR4eGgRP4Fgls,3072
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=NIBNAGq7TTRrip9OzMdiQKj8QOuwhL9esyM0aehUqBA,6893
 lamindb_setup/core/django.py,sha256=QUQm3zt5QIiD8uv6o9vbSm_bshqiSWzKSkgD3z2eJCg,3542
 lamindb_setup/core/exceptions.py,sha256=eoI7AXgATgDVzgArtN7CUvpaMUC067vsBg5LHCsWzDM,305
 lamindb_setup/core/hashing.py,sha256=7r96h5JBzuwfOR_gNNqTyWNPKMuiOUfBYwn6sCbZkf8,2269
 lamindb_setup/core/types.py,sha256=bcYnZ0uM_2NXKJCl94Mmc-uYrQlRUUVKG3sK2N-F-N4,532
-lamindb_setup/core/upath.py,sha256=QnAiaOZgT1TLUaX0PEs9dSJ0E4ZDD431hCfKrJIbmqQ,26339
-lamindb_setup-0.72.2.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-0.72.2.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
-lamindb_setup-0.72.2.dist-info/METADATA,sha256=oeLIvcLxoZcHsOT7yDplKrq89sW5elwq2TV9CpZWY6M,1620
-lamindb_setup-0.72.2.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=dwudkTVsXuyjS-2xR16WomcWtXJAEfRZ0ZzFq8_EDhE,27157
+lamindb_setup-0.73.1.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-0.73.1.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
+lamindb_setup-0.73.1.dist-info/METADATA,sha256=7iNq1IHqO4W4cHuOXPr_4wG7SxruZvn8MTBHZ_fRNM0,1620
+lamindb_setup-0.73.1.dist-info/RECORD,,