esgpull 0.6.3__py3-none-any.whl

esgpull/install_config.py

@@ -0,0 +1,185 @@
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+import platformdirs
+from typing_extensions import NotRequired, TypedDict
+
+from esgpull.constants import ROOT_ENV
+from esgpull.exceptions import AlreadyInstalledName, AlreadyInstalledPath
+
+
+@dataclass(init=False)
+class Install:
+    path: Path
+    name: str | None = None
+
+    def __init__(self, path: Path | str, name: str | None = None) -> None:
+        self.path = Path(path)
+        self.name = name
+
+    def asdict(self) -> InstallDict:
+        result: InstallDict = {"path": str(self.path)}
+        if self.name is not None:
+            result["name"] = self.name
+        return result
+
+
+class InstallDict(TypedDict):
+    path: str
+    name: NotRequired[str]
+
+
+class InstallConfigDict(TypedDict):
+    current: NotRequired[int]
+    installs: list[InstallDict]
+
+
+@dataclass(init=False)
+class _InstallConfig:
+    path: Path
+    current_idx: int | None
+    installs: list[Install]
+
+    def __init__(self) -> None:
+        user_config_dir = platformdirs.user_config_path("esgpull")
+        self.path = user_config_dir / "installs.json"
+        if self.path.is_file():
+            with self.path.open() as f:
+                content = json.load(f)
+            self.current_idx = content.get("current")
+            installs = content.get("installs", [])
+            self.installs = [Install(**inst) for inst in installs]
+        else:
+            self.current_idx = None
+            self.installs = []
+
+    def fullpath(self, path: Path) -> Path:
+        return path.expanduser().resolve()
+
+    @property
+    def current(self) -> Install | None:
+        env = os.getenv(ROOT_ENV)
+        if env is not None:
+            return self.installs[int(env)]
+        elif self.current_idx is not None:
+            return self.installs[self.current_idx]
+        else:
+            return None
+
+    @property
+    def default(self) -> Path:
+        return Path.home() / ".esgpull"
+
+    def activate_msg(self, idx: int, commented: bool = False) -> str:
+        install = self.installs[idx]
+        name = install.name or install.path
+        msg = f"""
+To choose {install.path} as the current install location:
+
+$ esgpull self choose {name}
+
+
+To enable {install.path} for the current shell:
+
+$ eval $(esgpull self activate {name})
+""".strip()
+        if commented:
+            lines = msg.splitlines()
+            return "\n".join("# " + line for line in lines)
+        else:
+            return msg
+
+    def activate_needs_eval(self, idx: int) -> str:
+        export = f"export {ROOT_ENV}={idx}"
+        comment = self.activate_msg(idx, commented=True)
+        return "\n".join([export, comment])
+
+    def asdict(self) -> InstallConfigDict:
+        result: InstallConfigDict = {
+            "installs": [inst.asdict() for inst in self.installs]
+        }
+        if self.current_idx is not None:
+            result["current"] = self.current_idx
+        return result
+
+    def add(self, path: Path, name: str | None = None) -> int:
+        install = Install(self.fullpath(path), name)
+        idx_path = self.index(path=install.path)
+        if idx_path > -1:
+            raise AlreadyInstalledPath(
+                path=install.path,
+                msg=self.activate_msg(idx_path),
+            )
+        if name is not None:
+            idx_name = self.index(name=name)
+            if idx_name > -1:
+                raise AlreadyInstalledName(
+                    name=name,
+                    msg=self.activate_msg(idx_name),
+                )
+        self.installs.append(install)
+        return len(self.installs) - 1
+
+    def remove_current(self) -> bool:
+        if self.current_idx is None:
+            return False
+        else:
+            self.installs.pop(self.current_idx)
+            self.current_idx = None
+            return True
+
+    def choose(
+        self,
+        *,
+        idx: int | None = None,
+        name: str | None = None,
+        path: Path | None = None,
+    ) -> None:
+        if idx is not None:
+            self.current_idx = idx
+        elif name is not None:
+            self.current_idx = self.index(name=name)
+        elif path is not None:
+            self.current_idx = self.index(path=path)
+        else:
+            self.current_idx = None
+        if self.current_idx == -1:
+            self.current_idx = None
+
+    def write(self) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        with self.path.open("w") as f:
+            json.dump(self.asdict(), f)
+
+    def index(
+        self,
+        *,
+        name: str | None = None,
+        path: Path | None = None,
+    ) -> int:
+        if name is not None:
+            return self._index_name(name)
+        elif path is not None:
+            return self._index_path(path)
+        else:
+            raise ValueError("nothing provided")
+
+    def _index_name(self, name: str) -> int:
+        for i, inst in enumerate(self.installs):
+            if inst.name is not None and name == inst.name:
+                return i
+        return -1
+
+    def _index_path(self, path: Path) -> int:
+        path = self.fullpath(path)
+        for i, inst in enumerate(self.installs):
+            if path == inst.path:
+                return i
+        return -1
+
+
+InstallConfig = _InstallConfig()

esgpull/migrations/README

@@ -0,0 +1 @@
+Generic single-database configuration.

esgpull/migrations/env.py

@@ -0,0 +1,82 @@
+from logging.config import fileConfig
+
+from alembic import context
+
+from esgpull.config import Config
+from esgpull.database import Database
+from esgpull.models import Base
+
+# from sqlalchemy import engine_from_config, pool
+
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+target_metadata = Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+        version_table="version",
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = config.attributes.get("connection", None)
+
+    if connectable is None:
+        _config = Config()
+        db = Database.from_config(_config, run_migrations=False)
+        connectable = db._engine
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            version_table="version",
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

esgpull/migrations/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

esgpull/migrations/versions/0.3.0_update_tables.py

@@ -0,0 +1,170 @@
+"""update tables
+
+Revision ID: 0.3.0
+Revises:
+Create Date: 2023-02-08 14:35:52.386481
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "0.3.0"
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "facet",
+        sa.Column("name", sa.String(length=64), nullable=False),
+        sa.Column("value", sa.String(length=255), nullable=False),
+        sa.Column("sha", sa.String(length=40), nullable=False),
+        sa.PrimaryKeyConstraint("sha"),
+    )
+    op.create_table(
+        "file",
+        sa.Column("file_id", sa.String(length=255), nullable=False),
+        sa.Column("dataset_id", sa.String(length=255), nullable=False),
+        sa.Column("master_id", sa.String(length=255), nullable=False),
+        sa.Column("url", sa.String(length=255), nullable=False),
+        sa.Column("version", sa.String(length=16), nullable=False),
+        sa.Column("filename", sa.String(length=255), nullable=False),
+        sa.Column("local_path", sa.String(length=255), nullable=False),
+        sa.Column("data_node", sa.String(length=40), nullable=False),
+        sa.Column("checksum", sa.String(length=64), nullable=False),
+        sa.Column("checksum_type", sa.String(length=16), nullable=False),
+        sa.Column("size", sa.Integer(), nullable=False),
+        sa.Column(
+            "status",
+            sa.Enum(
+                "New",
+                "Queued",
+                "Starting",
+                "Started",
+                "Pausing",
+                "Paused",
+                "Error",
+                "Cancelled",
+                "Done",
+                name="filestatus",
+            ),
+            nullable=False,
+        ),
+        sa.Column("sha", sa.String(length=40), nullable=False),
+        sa.PrimaryKeyConstraint("sha"),
+        sa.UniqueConstraint("file_id"),
+    )
+    op.create_table(
+        "options",
+        sa.Column(
+            "distrib",
+            sa.Enum("false", "true", "none", "notset", name="option"),
+            nullable=False,
+        ),
+        sa.Column(
+            "latest",
+            sa.Enum("false", "true", "none", "notset", name="option"),
+            nullable=False,
+        ),
+        sa.Column(
+            "replica",
+            sa.Enum("false", "true", "none", "notset", name="option"),
+            nullable=False,
+        ),
+        sa.Column(
+            "retracted",
+            sa.Enum("false", "true", "none", "notset", name="option"),
+            nullable=False,
+        ),
+        sa.Column("sha", sa.String(length=40), nullable=False),
+        sa.PrimaryKeyConstraint("sha"),
+    )
+    op.create_table(
+        "selection",
+        sa.Column("sha", sa.String(length=40), nullable=False),
+        sa.PrimaryKeyConstraint("sha"),
+    )
+    op.create_table(
+        "tag",
+        sa.Column("name", sa.String(length=255), nullable=False),
+        sa.Column("description", sa.Text(), nullable=True),
+        sa.Column("sha", sa.String(length=40), nullable=False),
+        sa.PrimaryKeyConstraint("sha"),
+    )
+    op.create_table(
+        "query",
+        sa.Column("tracked", sa.Boolean(), nullable=False),
+        sa.Column("require", sa.String(length=40), nullable=True),
+        sa.Column("options_sha", sa.String(length=40), nullable=False),
+        sa.Column("selection_sha", sa.String(length=40), nullable=False),
+        sa.Column("sha", sa.String(length=40), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["options_sha"],
+            ["options.sha"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["selection_sha"],
+            ["selection.sha"],
+        ),
+        sa.PrimaryKeyConstraint("sha"),
+    )
+    op.create_table(
+        "selection_facet",
+        sa.Column("selection_sha", sa.String(length=40), nullable=False),
+        sa.Column("facet_sha", sa.String(length=40), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["facet_sha"],
+            ["facet.sha"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["selection_sha"],
+            ["selection.sha"],
+        ),
+        sa.PrimaryKeyConstraint("selection_sha", "facet_sha"),
+    )
+    op.create_table(
+        "query_file",
+        sa.Column("query_sha", sa.String(length=40), nullable=False),
+        sa.Column("file_sha", sa.String(length=40), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["file_sha"],
+            ["file.sha"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["query_sha"],
+            ["query.sha"],
+        ),
+        sa.PrimaryKeyConstraint("query_sha", "file_sha"),
+    )
+    op.create_table(
+        "query_tag",
+        sa.Column("query_sha", sa.String(length=40), nullable=False),
+        sa.Column("tag_sha", sa.String(length=40), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["query_sha"],
+            ["query.sha"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["tag_sha"],
+            ["tag.sha"],
+        ),
+        sa.PrimaryKeyConstraint("query_sha", "tag_sha"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("query_tag")
+    op.drop_table("query_file")
+    op.drop_table("selection_facet")
+    op.drop_table("query")
+    op.drop_table("tag")
+    op.drop_table("selection")
+    op.drop_table("options")
+    op.drop_table("file")
+    op.drop_table("facet")
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.3.1_update_tables.py

@@ -0,0 +1,25 @@
+"""update tables
+
+Revision ID: 0.3.1
+Revises: 0.3.0
+Create Date: 2023-03-14 11:06:43.639042
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.3.1"
+down_revision = "0.3.0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.3.2_update_tables.py

@@ -0,0 +1,26 @@
+"""update tables
+
+Revision ID: 0.3.2
+Revises: 0.3.1
+Create Date: 2023-03-14 11:27:55.617982
+
+"""
+
+
+# revision identifiers, used by Alembic.
+revision = "0.3.2"
+down_revision = "0.3.1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.3.3_update_tables.py

@@ -0,0 +1,25 @@
+"""update tables
+
+Revision ID: 0.3.3
+Revises: 0.3.2
+Create Date: 2023-03-14 12:01:05.975209
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.3.3"
+down_revision = "0.3.2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.3.4_update_tables.py

@@ -0,0 +1,25 @@
+"""update tables
+
+Revision ID: 0.3.4
+Revises: 0.3.3
+Create Date: 2023-03-14 12:09:29.232008
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.3.4"
+down_revision = "0.3.3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.3.5_update_tables.py

@@ -0,0 +1,25 @@
+"""update tables
+
+Revision ID: 0.3.5
+Revises: 0.3.4
+Create Date: 2023-03-16 09:38:48.791464
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "0.3.5"
+down_revision = "0.3.4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.3.6_update_tables.py

@@ -0,0 +1,26 @@
+"""update tables
+
+Revision ID: 0.3.6
+Revises: 0.3.5
+Create Date: 2023-03-20 16:50:37.655296
+
+"""
+
+
+# revision identifiers, used by Alembic.
+revision = "0.3.6"
+down_revision = "0.3.5"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.3.7_update_tables.py

@@ -0,0 +1,26 @@
+"""update tables
+
+Revision ID: 0.3.7
+Revises: 0.3.6
+Create Date: 2023-04-03 11:42:09.069300
+
+"""
+
+
+# revision identifiers, used by Alembic.
+revision = "0.3.7"
+down_revision = "0.3.6"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

esgpull/migrations/versions/0.3.8_update_tables.py

@@ -0,0 +1,26 @@
+"""update tables
+
+Revision ID: 0.3.8
+Revises: 0.3.7
+Create Date: 2023-04-13 11:39:58.664284
+
+"""
+
+
+# revision identifiers, used by Alembic.
+revision = "0.3.8"
+down_revision = "0.3.7"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###