esgpull 0.6.3__py3-none-any.whl

esgpull/migrations/versions/0.4.0_update_tables.py

@@ -0,0 +1,25 @@
+"""update tables
+
+Revision ID: 0.4.0
+Revises: 0.3.8
+Create Date: 2023-05-02 11:52:41.283723
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.4.0"
+down_revision = "0.3.8"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.5.0_update_tables.py

@@ -0,0 +1,26 @@
+"""update tables
+
+Revision ID: 0.5.0
+Revises: 0.4.0
+Create Date: 2023-05-05 17:10:57.525968
+
+"""
+
+
+# revision identifiers, used by Alembic.
+revision = "0.5.0"
+down_revision = "0.4.0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.5.1_update_tables.py

@@ -0,0 +1,26 @@
+"""update tables
+
+Revision ID: 0.5.1
+Revises: 0.5.0
+Create Date: 2023-05-17 18:09:56.258545
+
+"""
+
+
+# revision identifiers, used by Alembic.
+revision = "0.5.1"
+down_revision = "0.5.0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.5.2_update_tables.py

@@ -0,0 +1,25 @@
+"""update tables
+
+Revision ID: 0.5.2
+Revises: 0.5.1
+Create Date: 2023-06-06 14:13:19.695120
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.5.2"
+down_revision = "0.5.1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.5.3_update_tables.py

@@ -0,0 +1,26 @@
+"""update tables
+
+Revision ID: 0.5.3
+Revises: 0.5.2
+Create Date: 2023-06-13 10:27:02.691266
+
+"""
+
+
+# revision identifiers, used by Alembic.
+revision = "0.5.3"
+down_revision = "0.5.2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.5.4_update_tables.py

@@ -0,0 +1,25 @@
+"""update tables
+
+Revision ID: 0.5.4
+Revises: 0.5.3
+Create Date: 2023-06-13 11:43:15.867306
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.5.4"
+down_revision = "0.5.3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.5.5_update_tables.py

@@ -0,0 +1,25 @@
+"""update tables
+
+Revision ID: 0.5.5
+Revises: 0.5.4
+Create Date: 2023-06-23 15:21:08.251368
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.5.5"
+down_revision = "0.5.4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.6.0_update_tables.py

@@ -0,0 +1,25 @@
+"""update tables
+
+Revision ID: 0.6.0
+Revises: 0.5.5
+Create Date: 2023-07-28 15:01:40.702583
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.6.0"
+down_revision = "0.5.5"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.6.1_update_tables.py

@@ -0,0 +1,25 @@
+"""update tables
+
+Revision ID: 0.6.1
+Revises: 0.6.0
+Create Date: 2023-07-28 18:40:20.911726
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.6.1"
+down_revision = "0.6.0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.6.2_update_tables.py

@@ -0,0 +1,25 @@
+"""update tables
+
+Revision ID: 0.6.2
+Revises: 0.6.1
+Create Date: 2023-08-18 17:11:15.605131
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.6.2"
+down_revision = "0.6.1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.6.3_update_tables.py

@@ -0,0 +1,25 @@
+"""update tables
+
+Revision ID: 0.6.3
+Revises: 0.6.2
+Create Date: 2024-06-13 11:48:48.097943
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.6.3"
+down_revision = "0.6.2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/models/__init__.py

@@ -0,0 +1,31 @@
+from typing import TypeVar
+
+from esgpull.models.base import Base
+from esgpull.models.dataset import Dataset
+from esgpull.models.facet import Facet
+from esgpull.models.file import FastFile, FileStatus
+from esgpull.models.options import Option, Options
+from esgpull.models.query import File, LegacyQuery, Query, QueryDict
+from esgpull.models.selection import Selection
+from esgpull.models.synda_file import SyndaFile
+from esgpull.models.tag import Tag
+
+Table = TypeVar("Table", bound=Base)
+
+__all__ = [
+    "Base",
+    "Dataset",
+    "Facet",
+    "FastFile",
+    "File",
+    "FileStatus",
+    "LegacyQuery",
+    "Option",
+    "Options",
+    "Query",
+    "QueryDict",
+    "Selection",
+    "SyndaFile",
+    "Table",
+    "Tag",
+]

esgpull/models/base.py

@@ -0,0 +1,50 @@
+from collections.abc import Mapping
+from dataclasses import Field
+from hashlib import sha1
+from typing import Any, ClassVar, TypeVar, cast
+
+import sqlalchemy as sa
+from sqlalchemy.orm import (
+    DeclarativeBase,
+    InstanceState,
+    Mapped,
+    MappedAsDataclass,
+    mapped_column,
+)
+
+T = TypeVar("T")
+Sha = sa.String(40)
+
+
+class Base(MappedAsDataclass, DeclarativeBase):
+    __dataclass_fields__: ClassVar[dict[str, Field]]
+    __sql_attrs__ = ("id", "sha", "_sa_instance_state", "__dataclass_fields__")
+
+    sha: Mapped[str] = mapped_column(
+        Sha,
+        init=False,
+        repr=False,
+        primary_key=True,
+    )
+
+    @property
+    def _names(self) -> tuple[str, ...]:
+        result: tuple[str, ...] = ()
+        for name in self.__dataclass_fields__:
+            if name in self.__sql_attrs__:
+                continue
+            result += (name,)
+        return result
+
+    def _as_bytes(self) -> bytes:
+        raise NotImplementedError
+
+    def compute_sha(self) -> None:
+        self.sha = sha1(self._as_bytes()).hexdigest()
+
+    @property
+    def state(self) -> InstanceState:
+        return cast(InstanceState, sa.inspect(self))
+
+    def asdict(self) -> Mapping[str, Any]:
+        raise NotImplementedError

esgpull/models/dataset.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+
+from esgpull.models.utils import find_int, find_str
+
+
+@dataclass
+class Dataset:
+    dataset_id: str
+    master_id: str
+    version: str
+    data_node: str
+    size: int
+    number_of_files: int
+
+    @classmethod
+    def serialize(cls, source: dict) -> Dataset:
+        dataset_id = find_str(source["instance_id"]).partition("|")[0]
+        master_id, version = dataset_id.rsplit(".", 1)
+        data_node = find_str(source["data_node"])
+        size = find_int(source["size"])
+        number_of_files = find_int(source["number_of_files"])
+        return cls(
+            dataset_id=dataset_id,
+            master_id=master_id,
+            version=version,
+            data_node=data_node,
+            size=size,
+            number_of_files=number_of_files,
+        )
+
+    def asdict(self) -> dict:
+        return asdict(self)

esgpull/models/facet.py

@@ -0,0 +1,18 @@
+import sqlalchemy as sa
+from sqlalchemy.orm import Mapped, mapped_column
+
+from esgpull.models.base import Base
+
+
+class Facet(Base):
+    __tablename__ = "facet"
+
+    name: Mapped[str] = mapped_column(sa.String(64))
+    value: Mapped[str] = mapped_column(sa.String(255))
+
+    def _as_bytes(self) -> bytes:
+        self_tuple = (self.name, self.value)
+        return str(self_tuple).encode()
+
+    def __hash__(self) -> int:
+        return hash(self._as_bytes())

esgpull/models/file.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+from typing_extensions import NotRequired, TypedDict
+
+from esgpull.models.base import Base
+from esgpull.models.utils import find_str
+
+
+class FileStatus(Enum):
+    New = "new"
+    Queued = "queued"
+    Starting = "starting"
+    Started = "started"
+    Pausing = "pausing"
+    Paused = "paused"
+    Error = "error"
+    Cancelled = "cancelled"
+    Done = "done"
+
+    @classmethod
+    def retryable(cls) -> list[FileStatus]:
+        return [cls.Error, cls.Cancelled]
+
+    @classmethod
+    def contains(cls, s: str) -> bool:
+        return s in [v.value for v in cls]
+
+
+class FileDict(TypedDict):
+    file_id: str
+    dataset_id: str
+    master_id: str
+    url: str
+    version: str
+    filename: str
+    local_path: str
+    data_node: str
+    checksum: str
+    checksum_type: str
+    size: int
+    status: NotRequired[str]
+
+
+@dataclass(init=False)
+class FastFile:
+    sha: str
+    file_id: str
+    checksum: str
+
+    def _as_bytes(self) -> bytes:
+        self_tuple = (self.file_id, self.checksum)
+        return str(self_tuple).encode()
+
+    @classmethod
+    def serialize(cls, source: dict) -> FastFile:
+        result = cls()
+        dataset_id = find_str(source["dataset_id"]).partition("|")[0]
+        filename = find_str(source["title"])
+        result.file_id = ".".join([dataset_id, filename])
+        result.checksum = find_str(source["checksum"])
+        Base.compute_sha(result)  # type: ignore
+        return result

esgpull/models/options.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+from collections.abc import Iterator, MutableMapping
+from enum import Enum
+
+import sqlalchemy as sa
+from sqlalchemy.orm import Mapped, mapped_column
+
+from esgpull.models.base import Base
+
+
+class Option(Enum):
+    false = 0, False
+    true = 1, True
+    none = 2, None
+    notset = 3, None
+
+    @classmethod
+    def _missing_(cls, value):
+        if isinstance(value, str) and value in cls._member_map_:
+            return cls[value]
+        elif isinstance(value, bool):
+            if value:
+                return cls.true
+            else:
+                return cls.false
+        elif value is None:
+            return cls.none
+        else:
+            raise ValueError(value)
+
+    def is_set(self) -> bool:
+        return self in [
+            Option.true,
+            Option.false,
+            Option.none,
+        ]
+
+    def is_bool(self) -> bool:
+        return self in [Option.true, Option.false]
+
+    def __bool__(self) -> bool:
+        if not self.is_bool():
+            raise ValueError(self)
+        return self == Option.true
+
+
+class Options(Base):
+    __tablename__ = "options"
+
+    distrib: Mapped[Option] = mapped_column(sa.Enum(Option))
+    latest: Mapped[Option] = mapped_column(sa.Enum(Option))
+    replica: Mapped[Option] = mapped_column(sa.Enum(Option))
+    retracted: Mapped[Option] = mapped_column(sa.Enum(Option))
+
+    _distrib_ = Option(False)
+    _latest_ = Option(True)
+    _replica_ = Option(None)
+    _retracted_ = Option(False)
+
+    @classmethod
+    def default(cls) -> Options:
+        return cls(
+            cls._distrib_,
+            cls._latest_,
+            cls._replica_,
+            cls._retracted_,
+        )
+
+    @classmethod
+    def _set_defaults(
+        cls,
+        distrib: str | bool | None,
+        latest: str | bool | None,
+        replica: str | bool | None,
+        retracted: str | bool | None,
+    ) -> None:
+        cls._distrib_ = Option(distrib)
+        cls._latest_ = Option(latest)
+        cls._replica_ = Option(replica)
+        cls._retracted_ = Option(retracted)
+
+    def __init__(
+        self,
+        distrib: Option | str | bool | None = Option.notset,
+        latest: Option | str | bool | None = Option.notset,
+        replica: Option | str | bool | None = Option.notset,
+        retracted: Option | str | bool | None = Option.notset,
+    ):
+        setattr(self, "distrib", distrib)
+        setattr(self, "latest", latest)
+        setattr(self, "replica", replica)
+        setattr(self, "retracted", retracted)
+
+    def __setattr__(
+        self,
+        name: str,
+        value: Option | str | bool | None,
+    ) -> None:
+        if name in self.__sql_attrs__:
+            super().__setattr__(name, value)
+        elif name in self._names:
+            super().__setattr__(name, Option(value))
+        else:
+            raise AttributeError(name)
+
+    def __getitem__(self, name: str) -> Option:
+        if name in self._names:
+            return getattr(self, name)
+        else:
+            raise KeyError(name)
+
+    def __setitem__(
+        self,
+        name: str,
+        value: Option | str | bool | None,
+    ) -> None:
+        setattr(self, name, value)
+
+    def _as_bytes(self) -> bytes:
+        self_tuple = (self.distrib, self.latest, self.replica, self.retracted)
+        return str(self_tuple).encode()
+
+    def items(
+        self,
+        use_default: bool = False,
+        keep_notset: bool = False,
+    ) -> Iterator[tuple[str, Option]]:
+        default = self.default()
+        for name in self._names:
+            option = getattr(self, name, Option.notset)
+            if option.is_set():
+                yield name, option
+            elif use_default:
+                yield name, getattr(default, name)
+            elif keep_notset:
+                yield name, option
+
+    def asdict(self) -> MutableMapping[str, bool | None]:
+        return {name: option.value[1] for name, option in self.items()}
+
+    def __bool__(self) -> bool:
+        return next(self.items(), None) is not None
+
+    def __rich_repr__(self) -> Iterator:
+        for name, option in self.items():
+            yield name, option.value
+
+    def __repr__(self) -> str:
+        cls_name = self.__class__.__name__
+        items = [f"{k}={v}" for k, v in self.__rich_repr__()]
+        return f"{cls_name}(" + ", ".join(items) + ")"
+
+    def trackable(self) -> bool:
+        return all(opt.is_set() for (_, opt) in self.items(keep_notset=True))
+
+    def apply_defaults(self, parent: Options):
+        for name, opt in self.items(keep_notset=True):
+            if opt.is_set():
+                continue
+            elif parent[name].is_set():
+                self[name] = parent[name]
+            else:
+                self[name] = self.default()[name]