lsstdesc-dataregistry 0.5.4rc1__tar.gz

lsstdesc_dataregistry-0.5.4rc1/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2022, JoanneBogart
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

lsstdesc_dataregistry-0.5.4rc1/PKG-INFO

@@ -0,0 +1,42 @@
+Metadata-Version: 2.1
+Name: lsstdesc-dataregistry
+Version: 0.5.4rc1
+Summary: Creation and user API for DESC data registry.
+Author-email: Joanne Bogart <jrb@slac.stanford.edu>, Stuart McAlpine <stuart.mcalpine@fysik.su.se>
+Keywords: desc,python,registry
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: <3.12,>=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: psycopg2
+Requires-Dist: sqlalchemy
+Requires-Dist: pyyaml
+Requires-Dist: pandas
+Provides-Extra: docs
+Requires-Dist: sphinx_rtd_theme; extra == "docs"
+
+[![python](https://img.shields.io/badge/Python-3.9-3776AB.svg?style=flat&logo=python&logoColor=white)](https://www.python.org)
+
+<img src="docs/source/_static/DREGS_logo_v2.png" width="300"/>
+
+**The ``dataregistry`` is currently undergoing beta testing, if you would like to participate please get in touch!**
+
+### What is the data registry?
+
+The data registry is a facility to store and share datasets from DESC related projects and pipelines.
+
+The data, once registered, are transferred and stored at a central location at NERSC, which can then be later accessed and queried using the ``dataregistry`` Python package, or the CLI. 
+
+### Documentation
+
+The full documentation for the data registry can be found [here](http://lsstdesc.org/dataregistry).
+
+### Contact
+
+For any further information, please get in touch!
+
+- Admin: Joanne Bogart ([@JoanneBogart](https://www.github.com/JoanneBogart))
+- Admin: Stuart McAlpine ([@stuartmcalpine](https://www.github.com/stuartmcalpine))

lsstdesc_dataregistry-0.5.4rc1/README.md

@@ -0,0 +1,22 @@
+[![python](https://img.shields.io/badge/Python-3.9-3776AB.svg?style=flat&logo=python&logoColor=white)](https://www.python.org)
+
+<img src="docs/source/_static/DREGS_logo_v2.png" width="300"/>
+
+**The ``dataregistry`` is currently undergoing beta testing, if you would like to participate please get in touch!**
+
+### What is the data registry?
+
+The data registry is a facility to store and share datasets from DESC related projects and pipelines.
+
+The data, once registered, are transferred and stored at a central location at NERSC, which can then be later accessed and queried using the ``dataregistry`` Python package, or the CLI. 
+
+### Documentation
+
+The full documentation for the data registry can be found [here](http://lsstdesc.org/dataregistry).
+
+### Contact
+
+For any further information, please get in touch!
+
+- Admin: Joanne Bogart ([@JoanneBogart](https://www.github.com/JoanneBogart))
+- Admin: Stuart McAlpine ([@stuartmcalpine](https://www.github.com/stuartmcalpine))

lsstdesc_dataregistry-0.5.4rc1/pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["setuptools >= 61.0"] # PEP 621 compliant
+build-backend = "setuptools.build_meta"
+
+[project]
+dynamic = ["version"]
+name = "lsstdesc-dataregistry"
+description = "Creation and user API for DESC data registry."
+readme = "README.md"
+authors = [
+    { name = "Joanne Bogart", email = "jrb@slac.stanford.edu" },
+    { name = "Stuart McAlpine", email = "stuart.mcalpine@fysik.su.se" }
+]
+license = { file = "LICENCE" }
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+]
+keywords = ["desc", "python", "registry"]
+# NOTE: Those needing to create a database will also need to install the
+#       GitPython package. It is not needed for accessing an existing db.
+dependencies = [
+    'psycopg2',
+    'sqlalchemy',
+    'pyyaml',
+    'pandas'
+]
+requires-python = ">=3.9,<3.12" # Supported versions (in CI)
+
+[tool.setuptools.dynamic]
+version = {attr = "dataregistry._version.__version__"}
+
+[project.optional-dependencies]
+docs = ["sphinx_rtd_theme"]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[project.scripts]
+dregs = "dataregistry_cli.cli:main"
+
+[tool.setuptools.package-data]
+"dataregistry" = ["site_config/site_rootdir.yaml", "schema/*.yaml"]

lsstdesc_dataregistry-0.5.4rc1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

lsstdesc_dataregistry-0.5.4rc1/src/dataregistry/DataRegistry.py

@@ -0,0 +1,127 @@
+from dataregistry.db_basic import DbConnection
+from dataregistry.query import Query
+from dataregistry.registrar import Registrar
+import yaml
+import os
+
+_HERE = os.path.dirname(__file__)
+_SITE_CONFIG_PATH = os.path.join(_HERE, "site_config", "site_rootdir.yaml")
+
+
+class DataRegistry:
+    def __init__(
+        self,
+        owner=None,
+        owner_type=None,
+        config_file=None,
+        schema=None,
+        root_dir=None,
+        verbose=False,
+        site=None,
+    ):
+        """
+        Primary data registry wrapper class.
+
+        The DataRegistry class links to both the Registrar class, to
+        register/modify/delete datasets, and the Query class, to query existing
+        datasets.
+
+        Links to the database is done automatically using the:
+            - the users config file (if None defaults are used)
+            - the passed schema (if None the default schema is used)
+
+        The `root_dir` is the location the data is copied to. This can be
+        manually passed, or alternately a predefined `site` can be chosen. If
+        nether are chosen, the NERSC site will be selected as the default.
+
+        Parameters
+        ----------
+        owner : str
+            To set the default owner for all registered datasets in this
+            instance.
+        owner_type : str
+            To set the default owner_type for all registered datasets in this
+            instance.
+        config_file : str
+            Path to config file, if None, default location is assumed.
+        schema : str
+            Schema to connect to, if None, default schema is assumed.
+        root_dir : str
+            Root directory for datasets, if None, default is assumed.
+        verbose : bool
+            True for more output.
+        site : str
+            Can be used instead of `root_dir`. Some predefined "sites" are
+            built in, such as "nersc", which will set the `root_dir` to the
+            data registry's default data location at NERSC.
+        """
+
+        # Establish connection to database
+        self.db_connection = DbConnection(config_file, schema=schema,
+                                          verbose=verbose)
+
+        # Work out the location of the root directory
+        self.root_dir = self._get_root_dir(root_dir, site)
+
+        # Create registrar object
+        self.Registrar = Registrar(self.db_connection, self.root_dir,
+                                   owner, owner_type)
+
+        # Create query object
+        self.Query = Query(self.db_connection, self.root_dir)
+
+    def _get_root_dir(self, root_dir, site):
+        """
+        What is the location of the root_dir we are pairing with?
+
+        In order of priority:
+            - If manually passed `root_dir` is not None, use that.
+            - If manually passed `site` is not None, use that.
+            - If env DATAREG_SITE is set, use that.
+            - Else use `site="nersc"`.
+
+        All `site`s are assumed to be postgres. Sqlite users must manually
+        specify the `root_dir.
+
+        Parameters
+        ----------
+        root_dir : str
+        site : str
+
+        Returns
+        -------
+        - : str
+            Path to root directory
+        """
+
+        # Load the site config yaml file
+        with open(_SITE_CONFIG_PATH) as f:
+            data = yaml.safe_load(f)
+
+        # Sqlite case
+        if self.db_connection._dialect == "sqlite":
+            # Sqlite cannot work with `site`s, must pass a `root_dir`
+            if root_dir is None:
+                raise ValueError("Must pass a `root_dir` using Sqlite")
+            else:
+                # root_dir cannot equal a site path when using Sqlite
+                for a, v in data.items():
+                    if root_dir == v:
+                        raise ValueError(
+                            "`root_dir` must not equal a pre-defined site with Sqlite"
+                        )
+            return root_dir
+
+        # Non Sqlite case
+        else:
+            if root_dir is None:
+                if site is not None:
+                    if site.lower() not in data.keys():
+                        raise ValueError(f"{site} is not a valid site")
+                    root_dir = data[site.lower()]
+                elif os.getenv("DATAREG_SITE"):
+                    root_dir = data[os.getenv("DATAREG_SITE").lower()]
+                else:
+                    root_dir = data["nersc"]
+
+            return root_dir

lsstdesc_dataregistry-0.5.4rc1/src/dataregistry/__init__.py

@@ -0,0 +1,6 @@
+from ._version import __version__
+from .db_basic import *
+from .registrar import *
+from .query import *
+from .git_util import *
+from .DataRegistry import DataRegistry

lsstdesc_dataregistry-0.5.4rc1/src/dataregistry/_version.py

@@ -0,0 +1 @@
+__version__ = "0.5.4-rc1"

lsstdesc_dataregistry-0.5.4rc1/src/dataregistry/db_basic.py

@@ -0,0 +1,347 @@
+from sqlalchemy import engine_from_config
+from sqlalchemy.engine import make_url
+from sqlalchemy import MetaData
+from sqlalchemy import column,  insert, select
+import yaml
+import os
+from datetime import datetime
+from dataregistry import __version__
+from dataregistry.exceptions import DataRegistryException
+
+"""
+Low-level utility routines and classes for accessing the registry
+"""
+
+SCHEMA_VERSION = "registry_beta"
+
+__all__ = [
+    "DbConnection",
+    "add_table_row",
+    "TableMetadata",
+    "SCHEMA_VERSION",
+]
+
+
+def _get_dataregistry_config(config_file=None, verbose=False):
+    """
+    Locate the data registry configuration file.
+
+    The code will check three scenarios, which are, in order of priority:
+        - The config_file has been manually passed
+        - The DATAREG_CONFIG env variable has been set
+        - The default location (the .config_reg_access file in $HOME)
+
+    If none of these are true, an exception is raised.
+
+    Parameters
+    ----------
+    config_file : str, optional
+        Manually set the location of the config file
+    verbose : bool, optional
+        True for more output
+
+    Returns
+    -------
+    config_file : str
+        Path to data registry configuration file
+    """
+
+    _default_loc = os.path.join(os.getenv("HOME"), ".config_reg_access")
+
+    # Case where the user has manually specified the location
+    if config_file is not None:
+        if verbose:
+            print(f"Using manually passed config file ({config_file})")
+        return config_file
+
+    # Case where the env variable is set
+    elif os.getenv("DATAREG_CONFIG"):
+        if verbose:
+            print(
+                "Using DATAREG_CONFIG env var for config file",
+                f"({os.getenv('DATAREG_CONFIG')})",
+            )
+        return os.getenv("DATAREG_CONFIG")
+
+    # Finally check default location in $HOME
+    elif os.path.isfile(_default_loc):
+        if verbose:
+            print("Using default location for config file", f"({_default_loc})")
+        return _default_loc
+    else:
+        raise ValueError("Unable to located data registry config file")
+
+
+def add_table_row(conn, table_meta, values, commit=True):
+    """
+    Generic insert, given connection, metadata for a table and column values to
+    be used.
+
+    Parameters
+    ----------
+    conn : SQLAlchemy Engine object
+        Connection to the database
+    table_meta : TableMetadata object
+        Table we are inserting data into
+    values : dict
+        Properties to be entered
+    commit : bool, optional
+        True to commit changes to database (default True)
+
+    Returns
+    -------
+    - : int
+        Primary key for new row if successful
+    """
+
+    result = conn.execute(insert(table_meta), [values])
+
+    if commit:
+        conn.commit()
+
+    return result.inserted_primary_key[0]
+
+
+class DbConnection:
+    def __init__(self, config_file=None, schema=None, verbose=False):
+        """
+        Simple class to act as container for connection
+
+        Parameters
+        ----------
+        config : str, optional
+            Path to config file with low-level connection information.
+            If None, default location is assumed
+        schema : str, optional
+            Schema to connect to.  If None, default schema is assumed
+        verbose : bool, optional
+            If True, produce additional output
+        """
+
+        # Extract connection info from configuration file
+        with open(_get_dataregistry_config(config_file, verbose)) as f:
+            connection_parameters = yaml.safe_load(f)
+
+        # Build the engine
+        self._engine = engine_from_config(connection_parameters)
+
+        # Pull out the working schema version
+        driver = make_url(connection_parameters["sqlalchemy.url"]).drivername
+        self._dialect = driver.split("+")[0]
+
+        if self._dialect == "sqlite":
+            self._schema = None
+        else:
+            if schema is None:
+                self._schema = SCHEMA_VERSION
+            else:
+                self._schema = schema
+
+    @property
+    def engine(self):
+        return self._engine
+
+    @property
+    def dialect(self):
+        return self._dialect
+
+    @property
+    def schema(self):
+        return self._schema
+
+
+class TableMetadata:
+    def __init__(self, db_connection, get_db_version=True):
+        """
+        Keep and dispense table metadata
+
+        Parameters
+        ----------
+        db_connection : DbConnection object
+            Stores information about the DB connection
+        get_db_version : bool, optional
+            True to extract the DB version from the provenance table
+        """
+
+        self._metadata = MetaData(schema=db_connection.schema)
+        self._engine = db_connection.engine
+        self._schema = db_connection.schema
+
+        # Load all existing tables
+        self._metadata.reflect(self._engine, db_connection.schema)
+
+        # Fetch and save db versioning, assoc. production schema
+        # if present and requested
+        self._prod_schema = None
+        if db_connection.dialect == "sqlite":
+            prov_name = "provenance"
+        else:
+            prov_name = ".".join([self._schema, "provenance"])
+
+        if prov_name not in self._metadata.tables:
+            raise DataRegistryException("Incompatible database: no Provenance table")
+
+        if prov_name in self._metadata.tables and get_db_version:
+            prov_table = self._metadata.tables[prov_name]
+            stmt = select(column("associated_production")).select_from(prov_table)
+            stmt = stmt.order_by(prov_table.c.provenance_id.desc())
+            with self._engine.connect() as conn:
+                results = conn.execute(stmt)
+                r = results.fetchone()
+            self._prod_schema = r[0]
+
+            cols = ["db_version_major", "db_version_minor", "db_version_patch"]
+
+            stmt = select(*[column(c) for c in cols])
+            stmt = stmt.select_from(prov_table)
+            stmt = stmt.order_by(prov_table.c.provenance_id.desc())
+            with self._engine.connect() as conn:
+                results = conn.execute(stmt)
+            r = results.fetchone()
+            self._db_major = r[0]
+            self._db_minor = r[1]
+            self._db_patch = r[2]
+        else:
+            self._db_major = None
+            self._db_minor = None
+            self._db_patch = None
+
+    @property
+    def db_version_major(self):
+        return self._db_major
+
+    @property
+    def db_version_minor(self):
+        return self._db_minor
+
+    @property
+    def db_version_patch(self):
+        return self._db_patch
+
+    def get(self, tbl):
+        if "." not in tbl:
+            if self._schema:
+                tbl = ".".join([self._schema, tbl])
+        if tbl not in self._metadata.tables.keys():
+            try:
+                self._metadata.reflect(self._engine, only=[tbl])
+            except Exception:
+                raise ValueError(f"No such table {tbl}")
+        return self._metadata.tables[tbl]
+
+
+def _insert_provenance(
+    db_connection,
+    db_version_major,
+    db_version_minor,
+    db_version_patch,
+    update_method,
+    comment=None,
+    associated_production="production",
+):
+    """
+    Write a row to the provenance table. Includes version of db schema,
+    version of code, etc.
+
+    Parameters
+    ----------
+    db_version_major : int
+    db_version_minor : int
+    db_version_patch : int
+    update_method : str
+        One of "create", "migrate"
+    comment : str, optional
+        Briefly describe reason for new version
+    associated_production : str, defaults to "production"
+        Name of production schema, if any, this schema may reference
+
+    Returns
+    -------
+    id : int
+        Id of new row in provenance table
+    """
+    from dataregistry.git_util import get_git_info
+    from git import InvalidGitRepositoryError
+
+    version_fields = __version__.split(".")
+    patch = version_fields[2]
+    suffix = None
+    if "-" in patch:
+        subfields = patch.split("-")
+        patch = subfields[0]
+        suffix = "-".join(subfields[1:])
+
+    values = dict()
+    values["code_version_major"] = version_fields[0]
+    values["code_version_minor"] = version_fields[1]
+    values["code_version_patch"] = patch
+    if suffix:
+        values["code_version_suffix"] = suffix
+    values["db_version_major"] = db_version_major
+    values["db_version_minor"] = db_version_minor
+    values["db_version_patch"] = db_version_patch
+    values["schema_enabled_date"] = datetime.now()
+    values["creator_uid"] = os.getenv("USER")
+    pkg_root = os.path.join(os.path.dirname(__file__), "../..")
+
+    # If this is a git repo, save hash and state
+    try:
+        git_hash, is_clean = get_git_info(pkg_root)
+        values["git_hash"] = git_hash
+        values["repo_is_clean"] = is_clean
+    except InvalidGitRepositoryError:
+        # no git repo; this is an install. Code version is sufficient
+        pass
+
+    values["update_method"] = update_method
+    if comment is not None:
+        values["comment"] = comment
+    if associated_production is not None:  # None is normal for sqlite
+        values["associated_production"] = associated_production
+    prov_table = TableMetadata(db_connection,
+                               get_db_version=False).get("provenance")
+    with db_connection.engine.connect() as conn:
+        id = add_table_row(conn, prov_table, values)
+
+        return id
+
+def _insert_keyword(
+    db_connection,
+    keyword,
+    system,
+    creator_uid=None,
+):
+    """
+    Write a row to a keyword table.
+
+    Parameters
+    ----------
+    db_connection : DbConnection class
+        Conenction to the database
+    keyword : str
+        Keyword to add
+    system : bool
+        True if this is a preset system keyword (False for user custom keyword)
+    creator_uid : int, optional
+
+    Returns
+    -------
+    id : int
+        Id of new row in keyword table
+    """
+
+    values = dict()
+    values["keyword"] = keyword
+    values["system"] = system
+    if creator_uid is None:
+        values["creator_uid"] = os.getenv("USER")
+    else:
+        values["creator_uid"] = creator_uid
+    values["creation_date"] = datetime.now()
+    values["active"] = True
+
+    keyword_table = TableMetadata(db_connection, get_db_version=False).get("keyword")
+    with db_connection.engine.connect() as conn:
+        id = add_table_row(conn, keyword_table, values)
+
+        return id

lsstdesc_dataregistry-0.5.4rc1/src/dataregistry/exceptions.py

@@ -0,0 +1,12 @@
+__all__ = ["DataRegistryException", "DataRegistryNYI"]
+
+
+class DataRegistryException(Exception):
+    pass
+
+
+class DataRegistryNYI(DataRegistryException):
+    def __init__(self, feature=""):
+        msg = f"Feature {feature} not yet implemented"
+        self.msg = msg
+        super().__init__(self.msg)

lsstdesc_dataregistry-0.5.4rc1/src/dataregistry/git_util.py

@@ -0,0 +1,21 @@
+import os
+
+__all__ = ["get_git_info"]
+
+
+def get_git_info(pkg_root):
+    """
+    Parameters
+    pkg_root   string    root directory of a git repo
+
+    Returns
+    git_hash    string   current git hash
+    is_clean    boolean
+
+    """
+    import git
+
+    repo = git.Repo(pkg_root)
+    has_uncommitted = repo.is_dirty()
+    has_untracked = len(repo.untracked_files) > 0
+    return repo.commit().hexsha, not (has_uncommitted or has_untracked)