lamindb_setup 1.10.1__py3-none-any.whl → 1.11.0__py3-none-any.whl

lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Modules & settings:
 
 """
 
-__version__ = "1.10.1"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.11.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 

lamindb_setup/_connect_instance.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import importlib
 import os
 import sys
+import types
 from typing import TYPE_CHECKING, Any
 from uuid import UUID
 
@@ -184,24 +185,26 @@ def _connect_instance(
 def reset_django_module_variables():
     # This function updates all module-level references to Django classes
     # But it will fail to update function level references
+    # This is not a problem unless for the function that calls ln.connect() itself
     # So, if a user has
+    #
     # def my_function():
     #     import lamindb as ln
-    #     ...
+    #     ln.connect(...)
     #
-    # Then it will **not** work and the `ln` variable will become stale and hold a reference
+    # Then it will **not** work and the `ln` variable becomes stale and hold a reference
     # to the old classes
-    # There doesn't seem to be an easy way to fix this problem
-
-    logger.important_hint("resetting django module variables")
-
-    import types
+    # Other functions that dynamically import are no problem because the variables
+    # are automatically refreshed when the function runs the next time after ln.connect() was called
+    logger.debug("resetting django module variables")
 
+    # django.apps needs to be a local import to refresh variables
     from django.apps import apps
 
     app_names = {app.name for app in apps.get_app_configs()}
-
-    for name, module in sys.modules.items():
+    # always copy before iterations over sys.modules
+    # see https://docs.python.org/3/library/sys.html#sys.modules
+    for name, module in sys.modules.copy().items():
         if (
             module is not None
             and (not name.startswith("__") or name == "__main__")

lamindb_setup/_init_instance.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import importlib
 import os
 import uuid
+from pathlib import Path
 from typing import TYPE_CHECKING, Literal
 from uuid import UUID
 

lamindb_setup/_migrate.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import requests  # type: ignore
 from django.db import connection
 from django.db.migrations.loader import MigrationLoader
 from lamin_utils import logger
@@ -142,6 +143,13 @@ class migrate:
             logger.important(f"updating lamindb version in hub: {lamindb.__version__}")
             if settings.instance.dialect != "sqlite":
                 update_schema_in_hub()
+                logger.warning(
+                    "clearing instance cache in hub; if this fails, re-run with latest lamindb version"
+                )
+                requests.delete(
+                    f"{settings.instance.api_url}/cache/instances/{settings.instance._id.hex}",
+                    headers={"Authorization": f"Bearer {settings.user.access_token}"},
+                )
             call_with_fallback_auth(
                 update_instance,
                 instance_id=settings.instance._id.hex,

lamindb_setup/_schema_metadata.py

@@ -404,23 +404,27 @@ class _SchemaHandler:
         return self.to_dict(include_django_objects=False)
 
     def _get_modules_metadata(self):
+        from django.apps import apps
         from lamindb.models import Registry, SQLRecord
 
-        all_models = {
-            module_name: {
-                model._meta.model_name: _ModelHandler(
-                    model, module_name, self.included_modules
-                )
-                for model in self._get_schema_module(
-                    module_name
-                ).models.__dict__.values()
-                if model.__class__ is Registry
+        all_models = {module_name: {} for module_name in self.included_modules}
+
+        # Iterate through all registered Django models
+        for model in apps.get_models():
+            # Check if model meets the criteria
+            if (
+                model.__class__ is Registry
                 and model is not SQLRecord
                 and not model._meta.abstract
-                and model.__get_module_name__() == module_name
-            }
-            for module_name in self.included_modules
-        }
+            ):
+                module_name = model.__get_module_name__()
+                # Only include if module is in our included list
+                if module_name in self.included_modules:
+                    model_name = model._meta.model_name
+                    all_models[module_name][model_name] = _ModelHandler(
+                        model, module_name, self.included_modules
+                    )
+
         assert all_models
         return all_models
 

lamindb_setup/core/_hub_core.py

@@ -661,7 +661,12 @@ def _sign_in_hub(email: str, password: str, handle: str | None, client: Client):
             "password": password,
         }
     )
-    data = client.table("account").select("*").eq("id", auth.user.id).execute().data
+    # normally public.account.id is equal to auth.user.id
+    # but it might be not the case in the future
+    # this is why we check public.account.user_id that references auth.user.id
+    data = (
+        client.table("account").select("*").eq("user_id", auth.user.id).execute().data
+    )
     if data:  # sync data from hub to local cache in case it was updated on the hub
         user = data[0]
         user_uuid = UUID(user["id"])
@@ -710,8 +715,9 @@ def _sign_in_hub_api_key(api_key: str, client: Client):
     # probably need more info here to avoid additional queries
     # like handle, uid etc
     account_id = jwt.decode(access_token, options={"verify_signature": False})["sub"]
+
     client.postgrest.auth(access_token)
-    # normally public.account.id is equal to auth.user.id
+
     data = client.table("account").select("*").eq("id", account_id).execute().data
     if data:
         user = data[0]

lamindb_setup/core/_settings.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import os
 import sys
 import warnings
+from pathlib import Path
 from typing import TYPE_CHECKING
 
 from lamin_utils import logger
@@ -23,8 +24,6 @@ from ._settings_store import (
 from .upath import LocalPathClasses, UPath
 
 if TYPE_CHECKING:
-    from pathlib import Path
-
     from lamindb.models import Branch, Space
 
     from lamindb_setup.core import InstanceSettings, StorageSettings, UserSettings
@@ -60,6 +59,7 @@ class SetupSettings:
 
     _auto_connect_path: Path = settings_dir / "auto_connect"
     _private_django_api_path: Path = settings_dir / "private_django_api"
+    _work_dir: Path = settings_dir / "work_dir.txt"
 
     _cache_dir: Path | None = None
 
@@ -70,6 +70,25 @@ class SetupSettings:
     def _instance_settings_path(self) -> Path:
         return current_instance_settings_file()
 
+    @property
+    def work_dir(self) -> Path | None:
+        """Get or set the current working directory.
+
+        If setting it to `None`, the working directory is unset
+        """
+        if not self._work_dir.exists():
+            return None
+        return Path(self._work_dir.read_text())
+
+    @work_dir.setter
+    def work_dir(self, value: str | Path | None) -> None:
+        if value is None:
+            if self._work_dir.exists():
+                self._work_dir.unlink()
+        else:
+            value_str = Path(value).expanduser().resolve().as_posix()
+            self._work_dir.write_text(value_str)
+
     @property
     def settings_dir(self) -> Path:
         """The directory that holds locally persisted settings."""
@@ -317,6 +336,7 @@ class SetupSettings:
         repr += "\nConfig:\n"
         repr += f" - private Django API: {self.private_django_api}\n"
         repr += "Local directories:\n"
+        repr += f" - working directory: {self.work_dir}\n"
         repr += f" - cache: {self.cache_dir.as_posix()}\n"
         repr += f" - user settings: {settings_dir.as_posix()}\n"
         repr += f" - system settings: {system_settings_dir.as_posix()}\n"

lamindb_setup/core/django.py

@@ -168,7 +168,10 @@ def setup_django(
                 ssl_require = False
             else:
                 ssl_require = not is_local_db_url(instance_db)
-            options = {"connect_timeout": os.getenv("PGCONNECT_TIMEOUT", 20)}
+            options = {
+                "connect_timeout": os.getenv("PGCONNECT_TIMEOUT", 20),
+                "gssencmode": "disable",
+            }
         else:
             ssl_require = False
             options = {}

lamindb_setup/core/upath.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import math
 import os
 import warnings
 from collections import defaultdict
@@ -353,6 +354,15 @@ def upload_from(
             destination = fsspec.utils.other_paths(
                 files, self.as_posix(), exists=False, flatten=False
             )
+    elif self.protocol == "s3" and "chunksize" not in kwargs:
+        size = local_path.stat().st_size
+        MiB = 1024**2
+        DEFAULT_CHUNKSIZE = 50 * MiB  # so in s3fs
+        if size / DEFAULT_CHUNKSIZE > 10000:  # should be no more than 10k parts for s3
+            raw = math.ceil(size / 10000)
+            step = 5 * MiB
+            rounded = math.ceil(raw / step) * step
+            kwargs["chunksize"] = rounded
 
     # the below lines are to avoid s3fs triggering create_bucket in upload if
     # dirs are present, it allows to avoid the permission error

{lamindb_setup-1.10.1.dist-info → lamindb_setup-1.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.10.1
+Version: 1.11.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10

{lamindb_setup-1.10.1.dist-info → lamindb_setup-1.11.0.dist-info}/RECORD

@@ -1,19 +1,19 @@
-lamindb_setup/__init__.py,sha256=f4yuL2G9SCvHZrn-uJT2LGRITfCtR6_nD63gAW-eaBI,2783
+lamindb_setup/__init__.py,sha256=iSKoEHzHwBMbBer5p_FuUPHJndK2eX3pfc6avRWqSaY,2783
 lamindb_setup/_cache.py,sha256=pGvDNVHGx4HWr_6w5ajqEJOdysmaGc6F221qFnXkT-k,2747
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=ToKMxsUq8dQBQh8baOrNVlSb1iC8h4zTg5dV8wMu0W4,6760
-lamindb_setup/_connect_instance.py,sha256=MPImCVAV0sZFX5s7muiR5lT2nqU5Z_j7LLo7apMZEZY,17143
+lamindb_setup/_connect_instance.py,sha256=BWQplcUNZ-hCLDwB1xUPtBrvcmLAl9-5-I5yfKc2tXE,17536
 lamindb_setup/_delete.py,sha256=4kS-_nQrV5xMvZE3BGCNEEGCboyGmqqDMXlckF0GxSk,5780
 lamindb_setup/_disconnect.py,sha256=FT8EpCm5XXDdhDH7QtAnkO3KPatq2HqT9VXGNjgJDbk,1232
 lamindb_setup/_django.py,sha256=uIQflpkp8l3axyPaKURlk3kacgpElVP5KOKmFxYSMGk,1454
 lamindb_setup/_entry_points.py,sha256=sKwXPX9xjOotoAjvgkU5LBwjjHLWVkh0ZGdiSsrch9k,522
 lamindb_setup/_exportdb.py,sha256=QLjoH4dEwqa01A12naKaDPglCCzl2_VLKWFfJRE_uSg,2113
 lamindb_setup/_importdb.py,sha256=fKv9ev5OOj_-bmzC8XZ1GxOcjIjI486yrHSHDWQrJeI,1874
-lamindb_setup/_init_instance.py,sha256=0DhUGJ6FERxVg30F4XhCfsXaIy3AP1UecQzyjEm-FLw,14965
-lamindb_setup/_migrate.py,sha256=oaqFcONqclTBXjxr4OWCJkSIH08nmC69xgVzdRq0N8U,10375
+lamindb_setup/_init_instance.py,sha256=5FRECcB_wuQ9pFkQctooyI6fJV0ItpVaBOHlL5EjfIk,14990
+lamindb_setup/_migrate.py,sha256=O84O2QLzLrXOK6ogeK4Nxtt-M2oLPUuZ8lKYo2pCcgU,10795
 lamindb_setup/_register_instance.py,sha256=RdUZxZWHLdbvdNZWpF8e0UWROb_T0cStWbzc5yUw34I,1047
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
-lamindb_setup/_schema_metadata.py,sha256=yMSuLZc-7iWUV7tETNuKmcDeMW0wtUr5ypKxsKC-m7k,14898
+lamindb_setup/_schema_metadata.py,sha256=VWO03tYBoU_rZF2eI2a5HSMPzpDS4kCE0Ul3fpvnSTA,15132
 lamindb_setup/_set_managed_storage.py,sha256=y5YQASsWNYVWUYeLgh3N2YBETYP7mBtbpxe3X_Vgb5I,2699
 lamindb_setup/_setup_user.py,sha256=DapdzT3u0f5LN5W9W9A6PWw-n8ejcJciQtHN9b5lidA,5889
 lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
@@ -26,11 +26,11 @@ lamindb_setup/core/_aws_storage.py,sha256=ofPTHXvF97I9eUCHxj5RPpUUqAcNV0VsPWpyKM
 lamindb_setup/core/_deprecated.py,sha256=M3vpM4fZPOncxY2qsXQAPeaEph28xWdv7tYaueaUyAA,2554
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
 lamindb_setup/core/_hub_client.py,sha256=Pl3sEG2rasdJp4Rh5HNY0VsPVls-nfY0OxD5QzrYF9A,8678
-lamindb_setup/core/_hub_core.py,sha256=r1tpfDjzgv7phESjq_dqYk23fyvm1M4KaVI-snP3NLA,26477
+lamindb_setup/core/_hub_core.py,sha256=6VNeREgFGjJ6nMNBlhbylV28oHZeoT0BlXL6dXzn9LU,26628
 lamindb_setup/core/_hub_crud.py,sha256=j6516H82kLjFUNPqFGUINbDw9YbofMgjxadGzYb0OS4,6362
 lamindb_setup/core/_hub_utils.py,sha256=6dyDGyzYFgVfR_lE3VN3CP1jGp98gxPtr-T91PAP05U,2687
 lamindb_setup/core/_private_django_api.py,sha256=By63l3vIEtK1pq246FhHq3tslxsaTJGKm5VakYluWp4,2656
-lamindb_setup/core/_settings.py,sha256=Xsn4Z-Z-gGM6q89f0SiWMaSDhIbTurwEwJLRsz6vspM,13312
+lamindb_setup/core/_settings.py,sha256=s2wB0vMeR0D36YCtwMSbyfYj6R05cQc0JYO1SH89bqc,14033
 lamindb_setup/core/_settings_instance.py,sha256=P2O2RWOSx2OUuU2nJBaD8FV6EIZYvUGGqTeVabIBKsA,23325
 lamindb_setup/core/_settings_load.py,sha256=j20cy3J56ZBHLDfB2A8oKjekNetMNsy0_W3eWD36pWI,5161
 lamindb_setup/core/_settings_save.py,sha256=XZx-vow7BT6y3JpRBB2UOJp2vwc7jOGea4wSgOPqjPU,3262
@@ -39,12 +39,12 @@ lamindb_setup/core/_settings_store.py,sha256=QmeWIGdIyq7UmjfHiEB_0xRD8hY-8-ZR2Wn
 lamindb_setup/core/_settings_user.py,sha256=gFfyMf-738onbh1Mf4wsmLlenQJPtjQfpUgKnOlqc2o,1453
 lamindb_setup/core/_setup_bionty_sources.py,sha256=ox3X-SHiHa2lNPSWjwZhINypbLacX6kGwH6hVVrSFZc,1505
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=H_CTUCjURFXwD1cCtV_Jn0_60iztZTkaesLLXIBgIxc,7204
-lamindb_setup/core/django.py,sha256=Er0ikZmWHx5rPkzbuWLNkq2XTL9yTcEL16iX1yz9R60,10470
+lamindb_setup/core/django.py,sha256=kV8W3WZy5Rkhn4rDJv2GNoq8JYvX_8dLBHhDRZdSgwE,10542
 lamindb_setup/core/exceptions.py,sha256=qjMzqy_uzPA7mCOdnoWnS_fdA6OWbdZGftz-YYplrY0,84
 lamindb_setup/core/hashing.py,sha256=Y8Uc5uSGTfU6L2R_gb5w8DdHhGRog7RnkK-e9FEMjPY,3680
 lamindb_setup/core/types.py,sha256=T7NwspfRHgIIpYsXDcApks8jkOlGeGRW-YbVLB7jNIo,67
-lamindb_setup/core/upath.py,sha256=-Wxct7lYOLVzUGv3ynqq0zLcRSAGWbvs-NqrZL0Aqy4,35579
-lamindb_setup-1.10.1.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-1.10.1.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-lamindb_setup-1.10.1.dist-info/METADATA,sha256=O4pmYo1z2JRRyPtjeySj76t-9cwPzsmM-vmjFiBlkPI,1804
-lamindb_setup-1.10.1.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=5mrAeAyYWXn1zJlwQmBM-NfXhpi2S-YsbEkXo_UESk0,36017
+lamindb_setup-1.11.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-1.11.0.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
+lamindb_setup-1.11.0.dist-info/METADATA,sha256=gvtfSb-3uOq8Vl0buE_koqO1zGu2B22kmcBNjmrChbQ,1804
+lamindb_setup-1.11.0.dist-info/RECORD,,