meerschaum 2.9.4__py3-none-any.whl → 3.0.0__py3-none-any.whl

meerschaum/connectors/instance/_InstanceConnector.py

@@ -0,0 +1,83 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define the interface for instance connectors.
+"""
+
+from __future__ import annotations
+
+import abc
+from typing import Any, Union, Dict, List, Tuple, Optional
+
+import meerschaum as mrsm
+from meerschaum.connectors._Connector import Connector
+
+
+class InstanceConnector(Connector):
+    """
+    Instance connectors define the interface for managing pipes and provide methods
+    for management of users, plugins, tokens, and other metadata built atop pipes.
+    """
+
+    IS_INSTANCE: bool = True
+    IS_THREAD_SAFE: bool = False
+
+    from ._users import (
+        get_users_pipe,
+        register_user,
+        get_user_id,
+        get_username,
+        get_users,
+        edit_user,
+        delete_user,
+        get_user_password_hash,
+        get_user_type,
+        get_user_attributes,
+    )
+
+    from ._plugins import (
+        get_plugins_pipe,
+        register_plugin,
+        get_plugin_user_id,
+        delete_plugin,
+        get_plugin_id,
+        get_plugin_version,
+        get_plugins,
+        get_plugin_user_id,
+        get_plugin_username,
+        get_plugin_attributes,
+    )
+
+    from ._tokens import (
+        get_tokens_pipe,
+        register_token,
+        edit_token,
+        invalidate_token,
+        delete_token,
+        get_token,
+        get_tokens,
+        get_token_model,
+        get_token_secret_hash,
+        token_exists,
+        get_token_scopes,
+    )
+
+    from ._pipes import (
+        register_pipe,
+        get_pipe_attributes,
+        get_pipe_id,
+        edit_pipe,
+        delete_pipe,
+        fetch_pipes_keys,
+        pipe_exists,
+        drop_pipe,
+        drop_pipe_indices,
+        sync_pipe,
+        create_pipe_indices,
+        clear_pipe,
+        get_pipe_data,
+        get_sync_time,
+        get_pipe_columns_types,
+        get_pipe_columns_indices,
+    )

meerschaum/connectors/instance/__init__.py

@@ -0,0 +1,10 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define high-level instance connector methods.
+"""
+
+from meerschaum.connectors.instance._InstanceConnector import InstanceConnector
+
+__all__ = ('InstanceConnector',)

meerschaum/connectors/instance/_pipes.py

@@ -0,0 +1,442 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define function signatures for pipes' methods.
+"""
+
+from __future__ import annotations
+
+import abc
+from typing import Any, Union, Dict, List, Tuple, Optional
+from datetime import datetime
+
+import meerschaum as mrsm
+
+@abc.abstractmethod
+def register_pipe(
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+    **kwargs: Any
+) -> mrsm.SuccessTuple:
+    """
+    Insert the pipe's attributes into the internal `pipes` table.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe to be registered.
+
+    Returns
+    -------
+    A `SuccessTuple` of the result.
+    """
+
+@abc.abstractmethod
+def get_pipe_attributes(
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+    **kwargs: Any
+) -> Dict[str, Any]:
+    """
+    Return the pipe's document from the internal `pipes` table.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe whose attributes should be retrieved.
+
+    Returns
+    -------
+    The document that matches the keys of the pipe.
+    """
+
+@abc.abstractmethod
+def get_pipe_id(
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+    **kwargs: Any
+) -> Union[str, int, None]:
+    """
+    Return the `id` for the pipe if it exists.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe whose `id` to fetch.
+
+    Returns
+    -------
+    The `id` for the pipe's document or `None`.
+    """
+
+def edit_pipe(
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+    **kwargs: Any
+) -> mrsm.SuccessTuple:
+    """
+    Edit the attributes of the pipe.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe whose in-memory parameters must be persisted.
+
+    Returns
+    -------
+    A `SuccessTuple` indicating success.
+    """
+    raise NotImplementedError
+
+def delete_pipe(
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+    **kwargs: Any
+) -> mrsm.SuccessTuple:
+    """
+    Delete a pipe's registration from the `pipes` collection.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe to be deleted.
+
+    Returns
+    -------
+    A `SuccessTuple` indicating success.
+    """
+    raise NotImplementedError
+
+@abc.abstractmethod
+def fetch_pipes_keys(
+    self,
+    connector_keys: Optional[List[str]] = None,
+    metric_keys: Optional[List[str]] = None,
+    location_keys: Optional[List[str]] = None,
+    tags: Optional[List[str]] = None,
+    debug: bool = False,
+    **kwargs: Any
+) -> List[Tuple[str, str, str]]:
+    """
+    Return a list of tuples for the registered pipes' keys according to the provided filters.
+
+    Parameters
+    ----------
+    connector_keys: list[str] | None, default None
+        The keys passed via `-c`.
+
+    metric_keys: list[str] | None, default None
+        The keys passed via `-m`.
+
+    location_keys: list[str] | None, default None
+        The keys passed via `-l`.
+
+    tags: List[str] | None, default None
+        Tags passed via `--tags` which are stored under `parameters:tags`.
+
+    Returns
+    -------
+    A list of connector, metric, and location keys in tuples.
+    You may return the string "None" for location keys in place of nulls.
+
+    Examples
+    --------
+    >>> import meerschaum as mrsm
+    >>> conn = mrsm.get_connector('example:demo')
+    >>> 
+    >>> pipe_a = mrsm.Pipe('a', 'demo', tags=['foo'], instance=conn)
+    >>> pipe_b = mrsm.Pipe('b', 'demo', tags=['bar'], instance=conn)
+    >>> pipe_a.register()
+    >>> pipe_b.register()
+    >>> 
+    >>> conn.fetch_pipes_keys(['a', 'b'])
+    [('a', 'demo', 'None'), ('b', 'demo', 'None')]
+    >>> conn.fetch_pipes_keys(metric_keys=['demo'])
+    [('a', 'demo', 'None'), ('b', 'demo', 'None')]
+    >>> conn.fetch_pipes_keys(tags=['foo'])
+    [('a', 'demo', 'None')]
+    >>> conn.fetch_pipes_keys(location_keys=[None])
+    [('a', 'demo', 'None'), ('b', 'demo', 'None')]
+    """
+
+@abc.abstractmethod
+def pipe_exists(
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+    **kwargs: Any
+) -> bool:
+    """
+    Check whether a pipe's target table exists.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe to check whether its table exists.
+
+    Returns
+    -------
+    A `bool` indicating the table exists.
+    """
+
+@abc.abstractmethod
+def drop_pipe(
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+    **kwargs: Any
+) -> mrsm.SuccessTuple:
+    """
+    Drop a pipe's collection if it exists.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe to be dropped.
+
+    Returns
+    -------
+    A `SuccessTuple` indicating success.
+    """
+    raise NotImplementedError
+
+def drop_pipe_indices(
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+    **kwargs: Any
+) -> mrsm.SuccessTuple:
+    """
+    Drop a pipe's indices.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe whose indices need to be dropped.
+
+    Returns
+    -------
+    A `SuccessTuple` indicating success.
+    """
+    return False, f"Cannot drop indices for instance connectors of type '{self.type}'."
+
+@abc.abstractmethod
+def sync_pipe(
+    self,
+    pipe: mrsm.Pipe,
+    df: 'pd.DataFrame' = None,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
+    chunksize: Optional[int] = -1,
+    check_existing: bool = True,
+    debug: bool = False,
+    **kwargs: Any
+) -> mrsm.SuccessTuple:
+    """
+    Sync a pipe using a database connection.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The Meerschaum Pipe instance into which to sync the data.
+
+    df: Optional[pd.DataFrame]
+        An optional DataFrame or equivalent to sync into the pipe.
+        Defaults to `None`.
+
+    begin: Union[datetime, int, None], default None
+        Optionally specify the earliest datetime to search for data.
+        Defaults to `None`.
+
+    end: Union[datetime, int, None], default None
+        Optionally specify the latest datetime to search for data.
+        Defaults to `None`.
+
+    chunksize: Optional[int], default -1
+        Specify the number of rows to sync per chunk.
+        If `-1`, resort to system configuration (default is `900`).
+        A `chunksize` of `None` will sync all rows in one transaction.
+        Defaults to `-1`.
+
+    check_existing: bool, default True
+        If `True`, pull and diff with existing data from the pipe. Defaults to `True`.
+
+    debug: bool, default False
+        Verbosity toggle. Defaults to False.
+
+    Returns
+    -------
+    A `SuccessTuple` of success (`bool`) and message (`str`).
+    """
+
+def create_pipe_indices(
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+    **kwargs: Any
+) -> mrsm.SuccessTuple:
+    """
+    Create a pipe's indices.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe whose indices need to be created.
+
+    Returns
+    -------
+    A `SuccessTuple` indicating success.
+    """
+    return False, f"Cannot create indices for instance connectors of type '{self.type}'."
+
+def clear_pipe(
+    self,
+    pipe: mrsm.Pipe,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    debug: bool = False,
+    **kwargs: Any
+) -> mrsm.SuccessTuple:
+    """
+    Delete rows within `begin`, `end`, and `params`.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe whose rows to clear.
+
+    begin: datetime | int | None, default None
+        If provided, remove rows >= `begin`.
+
+    end: datetime | int | None, default None
+        If provided, remove rows < `end`.
+
+    params: dict[str, Any] | None, default None
+        If provided, only remove rows which match the `params` filter.
+
+    Returns
+    -------
+    A `SuccessTuple` indicating success.
+    """
+    raise NotImplementedError
+
+@abc.abstractmethod
+def get_pipe_data(
+    self,
+    pipe: mrsm.Pipe,
+    select_columns: Optional[List[str]] = None,
+    omit_columns: Optional[List[str]] = None,
+    begin: Union[datetime, int, None] = None,
+    end: Union[datetime, int, None] = None,
+    params: Optional[Dict[str, Any]] = None,
+    debug: bool = False,
+    **kwargs: Any
+) -> Union['pd.DataFrame', None]:
+    """
+    Query a pipe's target table and return the DataFrame.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe with the target table from which to read.
+
+    select_columns: list[str] | None, default None
+        If provided, only select these given columns.
+        Otherwise select all available columns (i.e. `SELECT *`).
+
+    omit_columns: list[str] | None, default None
+        If provided, remove these columns from the selection.
+
+    begin: datetime | int | None, default None
+        The earliest `datetime` value to search from (inclusive).
+
+    end: datetime | int | None, default None
+        The lastest `datetime` value to search from (exclusive).
+
+    params: dict[str | str] | None, default None
+        Additional filters to apply to the query.
+
+    Returns
+    -------
+    The target table's data as a DataFrame.
+    """
+
+@abc.abstractmethod
+def get_sync_time(
+    self,
+    pipe: mrsm.Pipe,
+    params: Optional[Dict[str, Any]] = None,
+    newest: bool = True,
+    debug: bool = False,
+    **kwargs: Any
+) -> datetime | int | None:
+    """
+    Return the most recent value for the `datetime` axis.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe whose collection contains documents.
+
+    params: dict[str, Any] | None, default None
+        Filter certain parameters when determining the sync time.
+
+    newest: bool, default True
+        If `True`, return the maximum value for the column.
+
+    Returns
+    -------
+    The largest `datetime` or `int` value of the `datetime` axis. 
+    """
+
+@abc.abstractmethod
+def get_pipe_columns_types(
+    self,
+    pipe: mrsm.Pipe,
+    debug: bool = False,
+    **kwargs: Any
+) -> Dict[str, str]:
+    """
+    Return the data types for the columns in the target table for data type enforcement.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe whose target table contains columns and data types.
+
+    Returns
+    -------
+    A dictionary mapping columns to data types.
+    """
+
+def get_pipe_columns_indices(
+    self,
+    debug: bool = False,
+) -> Dict[str, List[Dict[str, str]]]:
+    """
+    Return a dictionary mapping columns to metadata about related indices.
+
+    Parameters
+    ----------
+    pipe: mrsm.Pipe
+        The pipe whose target table has related indices.
+
+    Returns
+    -------
+    A list of dictionaries with the keys "type" and "name".
+
+    Examples
+    --------
+    >>> pipe = mrsm.Pipe('demo', 'shirts', columns={'primary': 'id'}, indices={'size_color': ['color', 'size']})
+    >>> pipe.sync([{'color': 'red', 'size': 'M'}])
+    >>> pipe.get_columns_indices()
+    {'id': [{'name': 'demo_shirts_pkey', 'type': 'PRIMARY KEY'}], 'color': [{'name': 'IX_demo_shirts_color_size', 'type': 'INDEX'}], 'size': [{'name': 'IX_demo_shirts_color_size', 'type': 'INDEX'}]}
+    """
+    return {}

meerschaum/connectors/instance/_plugins.py

@@ -0,0 +1,159 @@
+#! /usr/bin/env python3
+# vim:fenc=utf-8
+
+"""
+Define high-level plugins methods for instance connectors.
+"""
+
+import uuid
+from typing import Union, Optional, List, Dict, Any
+
+import meerschaum as mrsm
+from meerschaum.core import Plugin
+
+
+def get_plugins_pipe(self) -> 'mrsm.Pipe':
+    """
+    Return the internal pipe for syncing plugins metadata.
+    """
+    if '_plugins_pipe' in self.__dict__:
+        return self._plugins_pipe
+
+    cache_connector = self.__dict__.get('_cache_connector', None)
+    users_pipe = self.get_users_pipe()
+    user_id_dtype = users_pipe.dtypes.get('user_id', 'uuid')
+
+    self._plugins_pipe = mrsm.Pipe(
+        'mrsm', 'plugins',
+        instance=self,
+        target='mrsm_plugins',
+        temporary=True,
+        cache=True,
+        cache_connector_keys=cache_connector,
+        static=True,
+        null_indices=False,
+        columns={
+            'primary': 'plugin_name',
+            'user_id': 'user_id',
+        },
+        dtypes={
+            'plugin_name': 'string',
+            'user_id': user_id_dtype,
+            'attributes': 'json',
+            'version': 'string',
+        },
+    )
+    return self._plugins_pipe
+
+
+def register_plugin(self, plugin: Plugin, debug: bool = False) -> mrsm.SuccessTuple:
+    """
+    Register a new plugin to the plugins table.
+    """
+    plugins_pipe = self.get_plugins_pipe()
+    users_pipe = self.get_users_pipe()
+    user_id = self.get_plugin_user_id(plugin)
+    if user_id is not None:
+        username = self.get_username(user_id, debug=debug)
+        return False, f"{plugin} is already registered to '{username}'."
+
+    doc = {
+        'plugin_name': plugin.name,
+        'version': plugin.version,
+        'attributes': plugin.attributes,
+        'user_id': plugin.user_id,
+    }
+
+    sync_success, sync_msg = plugins_pipe.sync(
+        [doc],
+        check_existing=False,
+        debug=debug,
+    )
+    if not sync_success:
+        return False, f"Failed to register {plugin}:\n{sync_msg}"
+
+    return True, "Success"
+
+
+def get_plugin_user_id(self, plugin: Plugin, debug: bool = False) -> Union[uuid.UUID, None]:
+    """
+    Return the user ID for plugin's owner.
+    """
+    plugins_pipe = self.get_plugins_pipe() 
+    return plugins_pipe.get_value('user_id', {'plugin_name': plugin.name}, debug=debug)
+
+
+def get_plugin_username(self, plugin: Plugin, debug: bool = False) -> Union[uuid.UUID, None]:
+    """
+    Return the username for plugin's owner.
+    """
+    user_id = self.get_plugin_user_id(plugin, debug=debug)
+    if user_id is None:
+        return None
+    return self.get_username(user_id, debug=debug)
+
+
+def get_plugin_id(self, plugin: Plugin, debug: bool = False) -> Union[str, None]:
+    """
+    Return a plugin's ID.
+    """
+    user_id = self.get_plugin_user_id(plugin, debug=debug)
+    return plugin.name if user_id is not None else None
+
+
+def delete_plugin(self, plugin: Plugin, debug: bool = False) -> mrsm.SuccessTuple:
+    """
+    Delete a plugin's registration.
+    """
+    plugin_id = self.get_plugin_id(plugin, debug=debug)
+    if plugin_id is None:
+        return False, f"{plugin} is not registered."
+    
+    plugins_pipe = self.get_plugins_pipe()
+    clear_success, clear_msg = plugins_pipe.clear(params={'plugin_name': plugin.name}, debug=debug)
+    if not clear_success:
+        return False, f"Failed to delete {plugin}:\n{clear_msg}"
+    return True, "Success"
+
+
+def get_plugin_version(self, plugin: Plugin, debug: bool = False) -> Union[str, None]:
+    """
+    Return the version for a plugin.
+    """
+    plugins_pipe = self.get_plugins_pipe() 
+    return plugins_pipe.get_value('version', {'plugin_name': plugin.name}, debug=debug)
+
+
+def get_plugin_attributes(self, plugin: Plugin, debug: bool = False) -> Dict[str, Any]:
+    """
+    Return the attributes for a plugin.
+    """
+    plugins_pipe = self.get_plugins_pipe() 
+    return plugins_pipe.get_value('attributes', {'plugin_name': plugin.name}, debug=debug) or {}
+
+
+def get_plugins(
+    self,
+    user_id: Optional[int] = None,
+    search_term: Optional[str] = None,
+    debug: bool = False,
+    **kw: Any
+) -> List[str]:
+    """
+    Return a list of plugin names.
+    """
+    plugins_pipe = self.get_plugins_pipe()
+    params = {}
+    if user_id:
+        params['user_id'] = user_id
+
+    df = plugins_pipe.get_data(['plugin_name'], params=params, debug=debug)
+    if df is None:
+        return []
+
+    docs = df.to_dict(orient='records')
+    return [
+        plugin_name
+        for doc in docs
+        if (plugin_name := doc['plugin_name']).startswith(search_term or '')
+    ]