opensampl 0.2.0__py3-none-any.whl

opensampl/__init__.py

@@ -0,0 +1,4 @@
+"""
+Adding this to ensure backwards compatibility with older versions of opensampl where orm import was:
+from opensampl import orm
+"""

opensampl/cli.py

@@ -0,0 +1,237 @@
+"""
+The openSAMPL CLI package is a click based command line interface for the openSAMPL package. It provides a way to
+interact with the database and load data into it.
+"""
+
+import json
+import sys
+from pathlib import Path
+from typing import Dict, List, Union
+
+import click
+import yaml
+from dotenv import find_dotenv, load_dotenv
+from loguru import logger
+
+from opensampl.constants import ENV_VARS
+from opensampl.db.orm import Base
+from opensampl.helpers.env import set_env
+from opensampl.load_data import create_new_tables, write_to_table
+from opensampl.vendors.constants import VENDOR_MAP, get_vendor_parser
+
+BANNER=r"""
+
+                        ____    _    __  __ ____  _
+  ___  _ __   ___ _ __ / ___|  / \  |  \/  |  _ \| |
+ / _ \| '_ \ / _ \ '_ \\___ \ / _ \ | |\/| | |_) | |
+| (_) | |_) |  __/ | | |___) / ___ \| |  | |  __/| |___
+ \___/| .__/ \___|_| |_|____/_/   \_\_|  |_|_|   |_____|
+      |_|
+    tools for processing clock data
+"""
+
+env_file = find_dotenv()
+load_dotenv()
+level = ENV_VARS.LOG_LEVEL.get_value()
+logger.configure(handlers=[{"sink": sys.stderr, "level": level.upper()}])
+
+class CaseInsensitiveGroup(click.Group):
+    """Defines Click group options as case-insensitive. By default, click groups are case-sensitive."""
+
+    def get_command(self, ctx, cmd_name):
+        """Normalize command name to lower case"""
+        cmd_name = cmd_name.lower()
+        # Match against lowercased command names
+        for name, cmd in self.commands.items():
+            if name.lower() == cmd_name:
+                return cmd
+        return None
+
+def get_table_names():
+    """Get all table names from the ORM in opensampl.db.orm"""
+    return [table.name for table in Base.metadata.sorted_tables]
+
+
+@click.group()
+def cli():
+    """CLI utility for openSAMPL"""
+
+
+@cli.command(name="init")
+def init():
+    """
+    Initialize the database.
+
+    Creates all tables as defined in the opensampl.db.orm file.
+    This is not required if you are using `opensampl-server`, as that is done as part of that initialization of the db.
+    """
+    logger.debug("Initializing database")
+    create_new_tables()
+
+
+@cli.group()
+def config():
+    """View and manage environment variables used by openSAMPL"""
+
+@config.command()
+def file():
+    """Show the path to the env file used by openSAMPL"""
+    click.echo(env_file)
+
+
+@config.command()
+@click.option("--explain", "-e", is_flag=True, help="Include descriptions of the variables")
+@click.option("--var", "-v", help="Specify a single variable to display")
+def show(explain: bool, var: str):
+    """
+    Display current environment variable configurations.
+
+    Examples
+    --------
+        opensampl config show  # Show all variables and their values
+        opensampl config show --explain  # Show all variables with descriptions
+        opensampl config show --var BACKEND_URL  # Show specific variable
+        opensampl config show -e -v BACKEND_URL  # Show specific variable with description
+
+    """
+    logger.debug(f"loaded env_file: {env_file}")
+    if var:
+        # Filter to specific variable if requested
+        vars_to_show = [v for v in ENV_VARS.all() if v.name == var]
+        if not vars_to_show:
+            click.echo(f"Error: Environment variable '{var}' not found", err=True)
+            return
+    else:
+        vars_to_show = ENV_VARS.all()
+
+    from tabulate import tabulate
+
+    # Create a list of dictionaries for the DataFrame
+    data = []
+    for env_var in vars_to_show:
+        row = {
+            "Variable": env_var.name,
+            "Value": str(env_var.get_value()),
+        }
+        if explain:
+            row.update({"Description": env_var.description})
+        data.append(row)
+    maxcolwidths = [None, None, 40] if explain else [None, None]
+    click.echo(tabulate(data, headers="keys", tablefmt="simple", maxcolwidths=maxcolwidths))
+
+
+@config.command("set")
+@click.argument("name")
+@click.argument("value")
+def config_set(name: str, value: str, temp: bool = None):
+    """
+    Set the value of an environment variable.
+
+    Note that this will only work if the variable is set in the .env file, if it is a true environment variable the
+    change will not persist.
+
+    Examples
+    --------
+        opensampl config set BACKEND_URL http://localhost:8000
+
+    """
+    set_env(name=name, value=value, temp=temp)
+
+@cli.group(cls=CaseInsensitiveGroup)
+def load():
+    """Load data into database"""
+
+
+for probe_name, vendor_type in VENDOR_MAP.items():
+    load.add_command(get_vendor_parser(probe_name).get_cli_command(), name=probe_name)
+
+
+def path_or_string(value):
+    """Get content from a file or use the string directly"""
+    # Get content - either from file or use the string directly
+    content = value
+    try:
+        path = Path(value)
+        if path.exists() and path.is_file():
+            with open(path, "r") as f:
+                content = f.read()
+    except Exception:
+        # If any error occurs during path handling, treat as raw string
+        pass
+
+    # Try parsing as YAML
+    try:
+        return yaml.safe_load(content)
+    except yaml.YAMLError as yaml_err:
+        # If YAML parsing fails, try JSON
+        try:
+            return json.loads(content)
+        except json.JSONDecodeError as json_err:
+            # If both parsing attempts fail, raise an error
+            raise click.BadParameter(
+                f"Could not parse input as YAML or JSON.\n" f"YAML error: {yaml_err}\n" f"JSON error: {json_err}"
+            )
+
+
+@load.command("table")
+@click.option(
+    "--if-exists",
+    "-i",
+    type=click.Choice(["update", "error", "replace", "ignore"]),
+    default="update",
+    help="How to handle conflicts with existing entries",
+)
+@click.argument("table_name", type=click.Choice(get_table_names()))
+@click.argument("filepath", type=path_or_string)
+def table_load(filepath: Union[Dict, List], table_name: str, if_exists: str):
+    r"""
+        Load data directly into a database table. Format can be yaml or json. Can be a list of dictionaries or a single
+        dictionary.
+
+        You do not have to specify schema, is assumed to be castdb.
+    \n\n
+        The --if-exists option controls how to handle conflicts:\n
+            - update: Only update fields that are provided and non-default (default)\n
+            - error: Raise an error if entry exists\n
+            - replace: Replace all non-primary-key fields with new values\n
+            - ignore: Skip if entry exists\n
+
+        Example:\n
+            cli.py table load locations data.json\n
+            cli.py table load probe_metadata metadata.yaml\n
+    """
+    try:
+        if isinstance(filepath, list):
+            for row in filepath:
+                write_to_table(table=table_name, data=row, if_exists=if_exists)
+        else:
+            write_to_table(table_name, filepath, if_exists=if_exists)
+        click.echo(f"Successfully wrote data to table {table_name}")
+    except Exception as e:
+        click.echo(f"Error writing to table: {str(e)}", err=True)
+        raise click.Abort()
+
+
+@cli.command(name="create")
+@click.argument("config_path", type=click.Path(exists=True, path_type=Path))
+@click.option(
+    "--update-db",
+    "-u",
+    is_flag=True,
+    help="Update the database with the new probe type",
+)
+def create_probe_command(config_path: Path, update_db: bool):
+    """
+    ** beta **
+    Create a new probe type with scaffolding, based on a config file.
+    """
+    from opensampl.helpers.create_vendor import VendorConfig
+
+    vendor_config = VendorConfig.from_config_file(config_path)
+    vendor_config.create()
+    if update_db:
+        create_new_tables()
+
+
+if __name__ == "__main__":
+    cli()

opensampl/constants.py

@@ -0,0 +1,84 @@
+"""Constants for accessing environment configurations"""
+import os
+from pathlib import Path
+from typing import Any, Optional, Type
+
+from pydantic import BaseModel
+
+
+class EnvVar(BaseModel):
+    """Defines an environment variable and its properties"""
+
+    name: str
+    description: str
+    type: Type = str
+    default: Optional[Any] = None
+
+    def get_value(self):
+        """Get value of var in environment"""
+        default = self.resolve_default()
+        if self.type is bool:
+            return os.getenv(self.name, default).lower() == 'true'
+        elif self.type is Path:
+            return Path(os.getenv(self.name, default))
+        return os.getenv(self.name, default)
+
+    def resolve_default(self):
+        """Resolve default value for env var based on type"""
+        if self.type is bool:
+            return self.default or 'false'
+        if self.type is Path and self.default is not None:
+            return Path(self.default).resolve()
+        return self.default
+
+
+class ENV_VARS:
+    """Variables referenced by openSAMPL"""
+
+    ROUTE_TO_BACKEND = EnvVar(
+        name="ROUTE_TO_BACKEND",
+        description=("Route all database operations through BACKEND_URL rather "
+                    "than applying directly using DATABASE_URL"),
+        type=bool
+    )
+    BACKEND_URL = EnvVar(
+        name="BACKEND_URL",
+        description="URL of the backend service when routing is enabled",
+    )
+    DATABASE_URL = EnvVar(
+        name="DATABASE_URL",
+        description="URL for direct database connections",
+    )
+    ARCHIVE_PATH = EnvVar(
+        name="ARCHIVE_PATH",
+        description="Default path that files are moved to after they have been processed",
+        type=Path,
+        default="archive",
+    )
+    LOG_LEVEL = EnvVar(
+        name="LOG_LEVEL",
+        description="Log level for opensampl cli",
+        default="INFO",
+    )
+    API_KEY = EnvVar(
+        name="API_KEY",
+        description="Access key for interacting with the backend",
+    )
+
+    @classmethod
+    def __iter__(cls):
+        """Get all EnvVar objects as iterable"""
+        yield from (value for key, value in cls.__dict__.items() if isinstance(value, EnvVar))
+
+    @classmethod
+    def get(cls, name: str):
+        """Get EnvVar object by name"""
+        var = getattr(cls, name, None)
+        if isinstance(var, EnvVar):
+            return var.get_value()
+        return os.getenv(name)
+
+    @classmethod
+    def all(cls) -> list[EnvVar]:
+        """Get all EnvVar objects as list"""
+        return [value for key, value in cls.__dict__.items() if isinstance(value, EnvVar)]

opensampl/db/__init__.py

@@ -0,0 +1 @@
+"""Database files used by openSAMPL"""

opensampl/db/access_orm.py

@@ -0,0 +1,128 @@
+"""ORM for managing access to openSAMPL database and endpoints"""
+import secrets
+import uuid
+from datetime import datetime
+from typing import List, Union
+
+from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text
+from sqlalchemy.orm import declarative_base, relationship
+from sqlalchemy.orm.exc import NoResultFound
+from sqlalchemy.schema import MetaData
+
+Base = declarative_base(metadata=MetaData(schema="access"))
+
+
+class APIAccessKey(Base):
+    """Table for recording and managing API access keys."""
+
+    __tablename__ = "api_access_keys"
+
+    id = Column(Integer, primary_key=True, index=True)
+    key = Column(String(64), unique=True, nullable=False, index=True)
+    created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
+    expires_at = Column(DateTime, nullable=True)
+
+    def generate_key(self):
+        """Generate the API access key."""
+        self.key = secrets.token_urlsafe(48)
+        return self.key
+
+    def is_expired(self):
+        """Check if API access key is expired."""
+        return self.expires_at is not None and datetime.utcnow() > self.expires_at
+
+
+class Views(Base):
+    """Table for recording and managing views."""
+
+    __tablename__ = "views"
+    view_id = Column(Text, primary_key=True, default=str(uuid.uuid4()))
+    name = Column(Text)
+
+    @staticmethod
+    def get_view_by_name(session, name):
+        """Get view by name"""
+        try:
+            return session.query(Views).filter_by(name=name).one()
+        except NoResultFound:
+            print(f"View with name {name} not found")
+            return None
+
+
+class Roles(Base):
+    """Table for recording and managing roles."""
+
+    __tablename__ = "roles"
+    role_id = Column(Text, primary_key=True, default=str(uuid.uuid4()))
+    name = Column(Text)
+    view_id = Column(Text, ForeignKey("views.view_id"))
+
+    @staticmethod
+    def get_role_by_name(session, name):
+        """Get role by name"""
+        try:
+            return session.query(Roles).filter_by(name=name).one()
+        except NoResultFound:
+            print(f"Role with name {name} not found")
+            return None
+
+
+class Users(Base):
+    """Table for recording and managing users."""
+
+    __tablename__ = "users"
+    user_id = Column(Text, primary_key=True, default=str(uuid.uuid4()))
+    email = Column(Text)
+
+    @staticmethod
+    def get_user_by_email(session, email):
+        """Get user by email"""
+        try:
+            return session.query(Users).filter_by(email=email).one()
+        except NoResultFound:
+            print(f"User with email {email} not found")
+            return None
+
+
+class UserRole(Base):
+    """Table for recording and managing user roles."""
+
+    __tablename__ = "user_role"
+    user_id = Column(Text, ForeignKey("users.user_id"), primary_key=True)
+    role_id = Column(Text, ForeignKey("roles.role_id"), primary_key=True)
+
+
+def add_user_role(emails: Union[str, List[str]], role_name: str, session):
+    """Add user role to the database."""
+    if isinstance(emails, str):
+        emails = [emails]
+
+    role = Roles.get_role_by_name(name=role_name)
+    if role is None:
+        print(f"Role with name {role_name} not found")
+        return
+
+    for email in emails:
+        user = Users.get_user_by_email(email=email)
+        if user is None:
+            # Create a new user if not found
+            user = Users(email=email)
+            session.add(user)
+            session.flush()  # Flush to get the generated user_id
+            print(f"New user created with email {email}")
+
+        # Check if user already has the specified role
+        if any(ur.role_id == role.role_id for ur in user.user_role):
+            print(f"User with email {email} already has role {role_name}")
+            continue
+
+        # Create a new entry in user_role table
+        user_role = UserRole(user_id=user.user_id, role_id=role.role_id)
+        session.add(user_role)
+        print(f"User with email {email} assigned role {role_name}")
+        session.commit()
+
+
+Views.roles = relationship("Roles")
+Roles.user_role = relationship("UserRole")
+Users.user_role = relationship("UserRole")

opensampl/db/orm.py

@@ -0,0 +1,198 @@
+"""Main openSAMPL database ORM"""
+import uuid
+from datetime import datetime
+
+from geoalchemy2 import Geometry, WKTElement
+from geoalchemy2.shape import to_shape
+from loguru import logger
+from sqlalchemy import NUMERIC, TIMESTAMP, Boolean, Column, Float, ForeignKey, Integer, String, Text, UniqueConstraint
+from sqlalchemy.event import listens_for
+from sqlalchemy.orm import Session, declarative_base, relationship
+from sqlalchemy.schema import MetaData
+
+SCHEMA_NAME = "castdb"
+
+
+class BaseHelpers:
+    """Mixin for Base class that adds some helper methods"""
+
+    def to_dict(self):
+        """Convert to dictionary, including changes to make it serializable"""
+        def convert_value(value):
+            if isinstance(value, datetime):
+                return value.isoformat()
+            elif hasattr(value, "__geo_interface__"):
+                return to_shape(value).__geo_interface__
+            return value
+
+        return {c.name: convert_value(getattr(self, c.name)) for c in self.__table__.columns}
+
+
+Base = declarative_base(cls=BaseHelpers, metadata=MetaData(schema=SCHEMA_NAME))
+
+
+class Locations(Base):
+    """
+    Table for storing locations
+
+    Automatically parses lat, lon, and z into point
+    """
+
+    __tablename__ = "locations"
+
+    uuid = Column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+    name = Column(Text, unique=True, nullable=False)
+    geom = Column(Geometry(geometry_type="GEOMETRY", srid=4326))
+    public = Column(Boolean, nullable=True)
+
+    probe_metadata = relationship("ProbeMetadata")
+
+    def __init__(self, **kwargs):
+        """Initialize Location object, including converting lat, lon, and z into point"""
+        if "lat" in kwargs and "lon" in kwargs:
+            lat = kwargs.pop("lat")
+            lon = kwargs.pop("lon")
+            z = kwargs.pop("z", None)
+            projection = kwargs.pop("projection", 4326)
+            if z is not None:
+                point_str = f"POINT({lon} {lat} {z})"
+            else:
+                point_str = f"POINT({lon} {lat})"
+            kwargs["geom"] = WKTElement(point_str, srid=projection)
+        super().__init__(**kwargs)
+
+
+class TestMetadata(Base):
+    """TestMetadata table for storing name, start and end of tests"""
+
+    __tablename__ = "test_metadata"
+
+    uuid = Column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+    name = Column(Text, unique=True, nullable=False)
+    start_date = Column(TIMESTAMP)
+    end_date = Column(TIMESTAMP)
+
+    probe_metadata = relationship("ProbeMetadata")
+
+
+class ProbeMetadata(Base):
+    """
+    Stores the basic information about clock probes.
+
+    A unique probe is identified by its ip address and probe_id.
+
+    Can be associated with an existing location or test by its name by providing location_name/test_name
+    """
+
+    __tablename__ = "probe_metadata"
+
+    uuid = Column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+    probe_id = Column(Text)
+    ip_address = Column(Text)
+    vendor = Column(Text)
+    model = Column(Text)
+    name = Column(Text, unique=True)
+    public = Column(Boolean, nullable=True)
+    location_uuid = Column(String(36), ForeignKey("locations.uuid"))
+    test_uuid = Column(String(36), ForeignKey("test_metadata.uuid"))
+
+    __table_args__ = (UniqueConstraint("probe_id", "ip_address", name="uq_probe_metadata_ipaddress_probeid"),)
+
+    probe_data = relationship("ProbeData")
+    adva_metadata = relationship("AdvaMetadata", back_populates="probe", uselist=False)
+
+    def __init__(self, **kwargs):
+        """Initialize Probe Metadata object, dealing with converting location name into uuid"""
+        location_name = kwargs.pop("location_name", None)
+        test_name = kwargs.pop("test_name", None)
+        super().__init__(**kwargs)
+
+        if location_name:
+            self._location_name = location_name  # Store it temporarily until we have a session
+        if test_name:
+            self._test_name = test_name
+
+    def resolve_references(self, session=None):
+        """
+        Resolve the references to location and/or test entries when given just the name, and
+        provides the correct foreign uuid key.
+
+        :param session: sqlalchemy session, used to query for the location/test
+        """
+        # If no session, attempt to figure it out from the object
+        if not session:
+            session = Session.object_session(self)
+            if not session:
+                logger.warning(
+                    "No session provided and one could not be found. Not resolving references for probe metadata."
+                )
+                return
+
+        if hasattr(self, "_location_name"):
+            location = session.query(Locations).filter_by(name=self._location_name).first()
+            if not location:
+                logger.warning(
+                    f"Could not find location with name {self._location_name}, leaving location reference null."
+                )
+            self.location_uuid = location.uuid if location else None
+            delattr(self, "_location_name")  # Clean up after resolving
+
+        if hasattr(self, "_test_name"):
+            test_meta = session.query(TestMetadata).filter_by(name=self._test_name).first()
+            if not test_meta:
+                logger.warning(
+                    f"Could not find test metadata with name {self._test_name}, leaving test reference null."
+                )
+            self.test_uuid = test_meta.uuid if test_meta else None
+            delattr(self, "_test_name")
+
+
+class ProbeData(Base):
+    """
+    Table for storing actual time data from the probes.
+
+    Each entry has a reference to the probe's uuid, timestamp for the measurement, and value for the measurement.
+    """
+
+    __tablename__ = "probe_data"
+
+    time = Column(TIMESTAMP, primary_key=True)
+    probe_uuid = Column(String(36), ForeignKey("probe_metadata.uuid"), primary_key=True)
+    value = Column(NUMERIC)
+
+
+class AdvaMetadata(Base):
+    """
+    ADVA Clock Probe specific metadata
+
+    This is metadata that is specifically provided by ADVA probes in their text file exports.
+    """
+
+    __tablename__ = "adva_metadata"
+
+    probe_uuid = Column(String, ForeignKey("probe_metadata.uuid"), primary_key=True)
+    type = Column(Text)
+    start = Column(TIMESTAMP)
+    frequency = Column(Integer)
+    timemultiplier = Column(Integer)
+    multiplier = Column(Integer)
+    title = Column(Text)
+    adva_probe = Column(Text)
+    adva_reference = Column(Text)
+    adva_reference_expected_ql = Column(Text)
+    adva_source = Column(Text)
+    adva_direction = Column(Text)
+    adva_version = Column(Float)
+    adva_status = Column(Text)
+    adva_mtie_mask = Column(Text)
+    adva_mask_margin = Column(Integer)
+
+    probe = relationship("ProbeMetadata", back_populates="adva_metadata")
+
+
+@listens_for(ProbeMetadata, "before_insert")
+def resolve_uuid(mapper, connection, target):
+    """Resolve the location_uuid and test_uuid entries for a probe before the object is inserted into the database."""
+    session = Session.object_session(target)
+    if session:
+        target.resolve_references(session)

opensampl/helpers/__init__.py

@@ -0,0 +1 @@
+"""Helper functions for the openSAMPL package"""