lamindb_setup 0.74.1__tar.gz → 0.74.3__tar.gz

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/.github/workflows/build.yml

@@ -43,7 +43,7 @@ jobs:
             python-version: "3.11"
           - lamin_env: "staging"
             python-version: "3.10" # test on 3.9
-    timeout-minutes: 6
+    timeout-minutes: 7
     steps:
       - uses: aws-actions/configure-aws-credentials@v2
         with:

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/.pre-commit-config.yaml

@@ -44,9 +44,3 @@ repos:
           (?x)(
               tests/hub-local/conftest.py
           )
-  - repo: https://github.com/pycqa/pydocstyle
-    rev: 6.1.1
-    hooks:
-      - id: pydocstyle
-        args: # google style + __init__, see http://www.pydocstyle.org/en/stable/error_codes.html
-          - --ignore=D100,D101,D102,D103,D104,D106,D107,D203,D204,D213,D215,D400,D401,D403,D404,D406,D407,D408,D409,D412,D413,D417

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.74.1
+Version: 0.74.3
 Summary: Setup & configure LaminDB.
-Author-email: Lamin Labs <laminlabs@gmail.com>
+Author-email: Lamin Labs <open-source@lamin.ai>
 Description-Content-Type: text/markdown
 Requires-Dist: lnschema_core>=0.51.0
 Requires-Dist: lamin_utils>=0.3.3
 Requires-Dist: django>4.2,<5.3.0
 Requires-Dist: dj_database_url>=1.3.0,<3.0.0
-Requires-Dist: pydantic[dotenv]<2.0.0
+Requires-Dist: pydantic-settings
 Requires-Dist: appdirs<2.0.0
 Requires-Dist: requests
 Requires-Dist: universal_pathlib==0.2.2

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/docs/hub-cloud/04-test-bionty.ipynb

@@ -49,7 +49,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from lnschema_bionty import PublicSource"
+    "from lnschema_bionty import Source"
    ]
   },
   {
@@ -58,7 +58,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sources_df = pd.DataFrame(PublicSource.objects.all().values())\n",
+    "sources_df = pd.DataFrame(Source.objects.all().values())\n",
     "sources_df.head()"
    ]
   },
@@ -124,7 +124,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "sources_df = pd.DataFrame(PublicSource.objects.all().values())\n",
+    "sources_df = pd.DataFrame(Source.objects.all().values())\n",
     "sources_df.head()"
    ]
   },

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/lamindb_setup/__init__.py

@@ -34,7 +34,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.74.1"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.74.3"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/lamindb_setup/_connect_instance.py

@@ -74,7 +74,8 @@ def update_db_using_local(
             # read from a cached settings file in case the hub result is only
             # read level or inexistent
             elif settings_file.exists() and (
-                "read" in db_dsn_hub.db.user or db_dsn_hub.db.user is None
+                db_dsn_hub.db.user is None
+                or (db_dsn_hub.db.user is not None and "read" in db_dsn_hub.db.user)
             ):
                 isettings = load_instance_settings(settings_file)
                 db_dsn_local = LaminDsnModel(db=isettings.db)
@@ -101,7 +102,7 @@ def update_db_using_local(
             scheme=db_dsn_hub.db.scheme,
             user=db_dsn_local.db.user,
             password=db_dsn_local.db.password,
-            host=db_dsn_hub.db.host,
+            host=db_dsn_hub.db.host,  # type: ignore
             port=db_dsn_hub.db.port,
             database=db_dsn_hub.db.database,
         )

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/lamindb_setup/_exportdb.py

@@ -29,7 +29,7 @@ MODELS = {
     #     "ExperimentalFactor": False,
     #     "DevelopmentalStage": False,
     #     "Ethnicity": False,
-    #     "PublicSource": False,
+    #     "Source": False,
     # },
     # "wetlab": {
     #     "ExperimentType": False,

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/lamindb_setup/_init_instance.py

@@ -245,7 +245,10 @@ def init(
         if instance_state == "connected":
             settings.auto_connect = True  # we can also debate this switch here
             return None
-        ssettings = init_storage(storage, instance_id=instance_id)
+        # cannot past instance_id here because instance does not yet exist!
+        # the instance_id field of the storage table is populated at the end of
+        # init_instance_hub
+        ssettings = init_storage(storage)
         isettings = InstanceSettings(
             id=instance_id,  # type: ignore
             owner=settings.user.handle,

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/lamindb_setup/core/_hub_client.py

@@ -5,7 +5,7 @@ from urllib.request import urlretrieve
 
 from gotrue.errors import AuthUnknownError
 from lamin_utils import logger
-from pydantic import BaseSettings
+from pydantic_settings import BaseSettings
 from supabase import Client, create_client  # type: ignore
 from supabase.lib.client_options import ClientOptions
 
@@ -59,9 +59,11 @@ class Environment:
 
 # runs ~0.5s
 def connect_hub(
-    fallback_env: bool = False, client_options: ClientOptions = ClientOptions()
+    fallback_env: bool = False, client_options: ClientOptions | None = None
 ) -> Client:
     env = Environment(fallback=fallback_env)
+    if client_options is None:
+        client_options = ClientOptions(auto_refresh_token=False)
     return create_client(env.supabase_api_url, env.supabase_anon_key, client_options)
 
 

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/lamindb_setup/core/_hub_core.py

@@ -93,20 +93,21 @@ def _select_storage(
         return False
     else:
         existing_storage = response.data[0]
-        if ssettings._instance_id is not None:
-            # consider storage settings that are meant to be managed by an instance
-            if UUID(existing_storage["instance_id"]) != ssettings._instance_id:
-                # everything is alright if the instance_id matches
-                # we're probably just switching storage locations
-                # below can be turned into a warning and then delegate the error
-                # to a unique constraint violation below
-                raise ValueError(
-                    f"Storage root {root} is already managed by instance {existing_storage['instance_id']}."
-                )
-        else:
-            # if the request is agnostic of the instance, that's alright,
-            # we'll update the instance_id with what's stored in the hub
-            ssettings._instance_id = UUID(existing_storage["instance_id"])
+        if existing_storage["instance_id"] is not None:
+            if ssettings._instance_id is not None:
+                # consider storage settings that are meant to be managed by an instance
+                if UUID(existing_storage["instance_id"]) != ssettings._instance_id:
+                    # everything is alright if the instance_id matches
+                    # we're probably just switching storage locations
+                    # below can be turned into a warning and then delegate the error
+                    # to a unique constraint violation below
+                    raise ValueError(
+                        f"Storage root {root} is already managed by instance {existing_storage['instance_id']}."
+                    )
+            else:
+                # if the request is agnostic of the instance, that's alright,
+                # we'll update the instance_id with what's stored in the hub
+                ssettings._instance_id = UUID(existing_storage["instance_id"])
         ssettings._uuid_ = UUID(existing_storage["id"])
         if update_uid:
             ssettings._uid = existing_storage["lnid"]
@@ -145,11 +146,14 @@ def _init_storage(ssettings: StorageSettings, client: Client) -> None:
         id = uuid.uuid5(uuid.NAMESPACE_URL, root)
     else:
         id = uuid.uuid4()
-    if ssettings._instance_id is None:
+    if ssettings._instance_id is None and settings._instance_exists:
         logger.warning(
             f"will manage storage location {ssettings.root_as_str} with instance {settings.instance.slug}"
         )
         ssettings._instance_id = settings.instance._id
+    instance_id_hex = (
+        None if ssettings._instance_id is None else ssettings._instance_id.hex
+    )
     fields = {
         "id": id.hex,
         "lnid": ssettings.uid,
@@ -157,7 +161,7 @@ def _init_storage(ssettings: StorageSettings, client: Client) -> None:
         "root": root,
         "region": ssettings.region,
         "type": ssettings.type,
-        "instance_id": ssettings._instance_id.hex,
+        "instance_id": instance_id_hex,
         # the empty string is important as we want the user flow to be through LaminHub
         # if this errors with unique constraint error, the user has to update
         # the description in LaminHub

lamindb_setup-0.74.3/lamindb_setup/core/_hub_utils.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+from typing import Any, ClassVar, Optional
+from urllib.parse import urlparse, urlunparse
+
+from pydantic import BaseModel, Field, GetCoreSchemaHandler
+from pydantic_core import CoreSchema, core_schema
+
+
+def validate_schema_arg(schema: str | None = None) -> str:
+    if schema is None or schema == "":
+        return ""
+    # currently no actual validation, can add back if we see a need
+    # the following just strips white spaces
+    to_be_validated = [s.strip() for s in schema.split(",")]
+    return ",".join(to_be_validated)
+
+
+def validate_db_arg(db: str | None) -> None:
+    if db is not None:
+        LaminDsnModel(db=db)
+
+
+class LaminDsn(str):
+    allowed_schemes: ClassVar[set[str]] = {
+        "postgresql",
+        # future enabled schemes
+        # "snowflake",
+        # "bigquery"
+    }
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: GetCoreSchemaHandler
+    ) -> CoreSchema:
+        return core_schema.no_info_after_validator_function(
+            cls.validate,
+            core_schema.str_schema(),
+            serialization=core_schema.plain_serializer_function_ser_schema(str),
+        )
+
+    @classmethod
+    def validate(cls, v: Any) -> LaminDsn:
+        if isinstance(v, str):
+            parsed = urlparse(v)
+            if parsed.scheme not in cls.allowed_schemes:
+                raise ValueError(f"Invalid scheme: {parsed.scheme}")
+            return cls(v)
+        elif isinstance(v, cls):
+            return v
+        else:
+            raise ValueError(f"Invalid value for LaminDsn: {v}")
+
+    @property
+    def user(self) -> str | None:
+        return urlparse(self).username
+
+    @property
+    def password(self) -> str | None:
+        return urlparse(self).password
+
+    @property
+    def host(self) -> str | None:
+        return urlparse(self).hostname
+
+    @property
+    def port(self) -> int | None:
+        return urlparse(self).port
+
+    @property
+    def database(self) -> str:
+        return urlparse(self).path.lstrip("/")
+
+    @property
+    def scheme(self) -> str:
+        return urlparse(self).scheme
+
+    @classmethod
+    def build(
+        cls,
+        *,
+        scheme: str,
+        user: str | None = None,
+        password: str | None = None,
+        host: str,
+        port: int | None = None,
+        database: str | None = None,
+        query: str | None = None,
+        fragment: str | None = None,
+    ) -> LaminDsn:
+        netloc = host
+        if port is not None:
+            netloc = f"{netloc}:{port}"
+        if user is not None:
+            auth = user
+            if password is not None:
+                auth = f"{auth}:{password}"
+            netloc = f"{auth}@{netloc}"
+
+        path = f"/{database}" if database else ""
+
+        url = urlunparse((scheme, netloc, path, "", query or "", fragment or ""))
+        return cls(url)
+
+
+class LaminDsnModel(BaseModel):
+    db: LaminDsn = Field(..., description="The database DSN")
+
+    model_config = {"arbitrary_types_allowed": True}

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/lamindb_setup/core/_settings_load.py

@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Optional
 from uuid import UUID, uuid4
 
 from lamin_utils import logger
-from pydantic.error_wrappers import ValidationError
+from pydantic import ValidationError
 
 from ._settings_instance import InstanceSettings
 from ._settings_storage import StorageSettings

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/lamindb_setup/core/_settings_save.py

@@ -44,6 +44,8 @@ def save_settings(
             if type == Optional[bool]:
                 type = bool
             if "__" not in store_key:
+                if store_key == "model_config":
+                    continue
                 if store_key == "storage_root":
                     value = settings.storage.root_as_str
                 elif store_key == "storage_region":

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/lamindb_setup/core/_settings_store.py

@@ -2,7 +2,7 @@ import os
 from pathlib import Path
 from typing import Optional
 
-from pydantic import BaseSettings
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 if "LAMIN_SETTINGS_DIR" in os.environ:
     # Needed when running with AWS Lambda, as only tmp/ directory has a write access
@@ -58,10 +58,7 @@ class InstanceSettingsStore(BaseSettings):
     id: str
     git_repo: Optional[str]
     keep_artifacts_local: Optional[bool]
-
-    class Config:
-        env_prefix = "lamindb_instance_"
-        env_file = ".env"
+    model_config = SettingsConfigDict(env_prefix="lamindb_instance_", env_file=".env")
 
 
 class UserSettingsStore(BaseSettings):
@@ -72,7 +69,4 @@ class UserSettingsStore(BaseSettings):
     uuid: str
     handle: str
     name: str
-
-    class Config:
-        env_prefix = "lamin_user_"
-        env_file = ".env"
+    model_config = SettingsConfigDict(env_prefix="lamin_user_", env_file=".env")

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/lamindb_setup/core/_setup_bionty_sources.py

@@ -9,14 +9,14 @@ if TYPE_CHECKING:
 
 
 def write_bionty_sources(isettings: InstanceSettings) -> None:
-    """Write bionty sources to PublicSource table."""
+    """Write bionty sources to Source table."""
     if "bionty" not in isettings.schema:
         return None
     import shutil
 
     import bionty_base
     from bionty_base.dev._handle_sources import parse_sources_yaml
-    from lnschema_bionty.models import PublicSource
+    from lnschema_bionty.models import Source
 
     shutil.copy(
         bionty_base.settings.current_sources, bionty_base.settings.lamindb_sources
@@ -32,16 +32,11 @@ def write_bionty_sources(isettings: InstanceSettings) -> None:
             .set_index(["entity", key])
         )
 
-    try:
-        currently_used = _get_currently_used("organism")
-        key = "organism"
-    except KeyError:
-        currently_used = _get_currently_used("species")
-        key = "species"
+    currently_used = _get_currently_used("organism")
 
     all_records = []
     for kwargs in all_sources_dict:
-        act = currently_used.loc[(kwargs["entity"], kwargs[key])].to_dict()
+        act = currently_used.loc[(kwargs["entity"], kwargs["organism"])].to_dict()
         if (act["source"] == kwargs["source"]) and (
             act["version"] == kwargs["version"]
         ):
@@ -49,15 +44,13 @@ def write_bionty_sources(isettings: InstanceSettings) -> None:
 
         # when the database is not yet migrated but setup is updated
         # won't need this once lamindb is released with the new pin
-        if hasattr(PublicSource, "species") and "organism" in kwargs:
-            kwargs["species"] = kwargs.pop("organism")
-        elif hasattr(PublicSource, "organism") and "species" in kwargs:
-            kwargs["organism"] = kwargs.pop("species")
         kwargs["run"] = None  # can't yet access tracking information
-        record = PublicSource(**kwargs)
+        kwargs["in_db"] = False
+        # kwargs["name"] = kwargs.pop("source")
+        record = Source(**kwargs)
         all_records.append(record)
 
-    PublicSource.objects.bulk_create(all_records, ignore_conflicts=True)
+    Source.objects.bulk_create(all_records, ignore_conflicts=True)
 
 
 def load_bionty_sources(isettings: InstanceSettings):
@@ -68,11 +61,11 @@ def load_bionty_sources(isettings: InstanceSettings):
     import bionty_base
     from bionty_base.dev._handle_sources import parse_currently_used_sources
     from bionty_base.dev._io import write_yaml
-    from lnschema_bionty.models import PublicSource
+    from lnschema_bionty.models import Source
 
     try:
         # need try except because of integer primary key migration
-        active_records = PublicSource.objects.filter(currently_used=True).all().values()
+        active_records = Source.objects.filter(currently_used=True).all().values()
         write_yaml(
             parse_currently_used_sources(active_records),
             bionty_base.settings.lamindb_sources,

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/noxfile.py

@@ -45,10 +45,14 @@ uv pip install --system --no-deps {no_deps_packages}
         cmds = ""
         cmds += schema_deps.strip()
     elif group == "hub-local":
-        cmds = schema_deps + """uv pip install --system -e ./laminhub/rest-hub"""
+        cmds = schema_deps.strip()
     # current package
     cmds += """\nuv pip install --system -e '.[aws,dev]' lamin-cli"""
 
+    # above downgrades django, I don't understand why, try this
+    if group == "hub-local":
+        cmds += "\nuv pip install --system -e ./laminhub/rest-hub"
+
     [run(session, line) for line in cmds.splitlines()]
 
 

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "lamindb_setup"
-authors = [{name = "Lamin Labs", email = "laminlabs@gmail.com"}]
+authors = [{name = "Lamin Labs", email = "open-source@lamin.ai"}]
 readme = "README.md"
 dynamic = ["version", "description"]
 dependencies = [
@@ -13,7 +13,7 @@ dependencies = [
     # External dependencies
     "django>4.2,<5.3.0",
     "dj_database_url>=1.3.0,<3.0.0",
-    "pydantic[dotenv]<2.0.0",
+    "pydantic-settings",
     "appdirs<2.0.0",
     "requests",
     "universal_pathlib==0.2.2",  # is still experimental, need pinning

{lamindb_setup-0.74.1 → lamindb_setup-0.74.3}/tests/hub-local/test_all.py

@@ -118,8 +118,9 @@ def create_myinstance(create_testadmin1_session):  # -> Dict
         id=instance_id,
         owner=usettings.handle,
         name="myinstance",
+        # cannot yet pass instance_id here as it does not yet exist
         storage=init_storage_base(
-            "s3://lamindb-ci/myinstance", instance_id=instance_id
+            "s3://lamindb-ci/myinstance",
         ),
         db=db_str,
     )

lamindb_setup-0.74.1/lamindb_setup/core/_hub_utils.py

@@ -1,76 +0,0 @@
-from __future__ import annotations
-
-from typing import Optional
-
-from pydantic import BaseModel, validator
-from pydantic.networks import MultiHostDsn
-
-
-def validate_schema_arg(schema: str | None = None) -> str:
-    if schema is None or schema == "":
-        return ""
-    # currently no actual validation, can add back if we see a need
-    # the following just strips white spaces
-    to_be_validated = [s.strip() for s in schema.split(",")]
-    return ",".join(to_be_validated)
-
-
-def validate_db_arg(db: str | None) -> None:
-    if db is not None:
-        LaminDsnModel(db=db)
-
-
-class LaminDsn(MultiHostDsn):
-    """Custom DSN Type for Lamin.
-
-    This class allows us to customize the allowed schemes for databases
-    and also handles the parsing and building of DSN strings with the
-    database name instead of URL path.
-    """
-
-    allowed_schemes = {
-        "postgresql",
-        # future enabled schemes
-        # "snowflake",
-        # "bigquery"
-    }
-    user_required = True
-    __slots__ = ()
-
-    @property
-    def database(self):
-        return self.path[1:]
-
-    @classmethod
-    def build(
-        cls,
-        *,
-        scheme: str,
-        user: str | None = None,
-        password: str | None = None,
-        host: str,
-        port: str | None = None,
-        database: str | None = None,
-        query: str | None = None,
-        fragment: str | None = None,
-        **_kwargs: str,
-    ) -> str:
-        return super().build(
-            scheme=scheme,
-            user=user,
-            password=password,
-            host=host,
-            port=str(port),
-            path=f"/{database}",
-            query=query,
-            fragment=fragment,
-        )
-
-
-class LaminDsnModel(BaseModel):
-    db: LaminDsn
-
-    @validator("db")
-    def check_db_name(cls, v):
-        assert v.path and len(v.path) > 1, "database must be provided"
-        return v