meerschaum 3.0.0rc2__py3-none-any.whl → 3.0.0rc4__py3-none-any.whl

meerschaum/plugins/bootstrap.py

@@ -16,7 +16,8 @@ from meerschaum.utils.formatting._shell import clear_screen
 
 FEATURE_CHOICES: Dict[str, str] = {
     'fetch'    : 'Fetch data\n     (e.g. extracting from a remote API)\n',
-    'connector': 'Custom connector\n     (e.g. manage credentials)\n',
+    'connector': 'Connector\n     (fetch data & manage credentials)\n',
+    'instance-connector': 'Instance connector\n     (implement the pipes interface)\n',
     'action'   : 'New actions\n     (e.g. `mrsm sing song`)\n',
     'api'      : 'New API endpoints\n     (e.g. `POST /my/new/endpoint`)\n',
     'web'      : 'New web console page\n     (e.g. `/dash/my-web-app`)\n',
@@ -25,6 +26,7 @@ FEATURE_CHOICES: Dict[str, str] = {
 IMPORTS_LINES: Dict[str, str] = {
     'stdlib': (
         "from datetime import datetime, timedelta, timezone\n"
+        "from typing import Any, Union, List, Dict\n"
     ),
     'default': (
         "import meerschaum as mrsm\n"
@@ -33,6 +35,9 @@ IMPORTS_LINES: Dict[str, str] = {
     'connector': (
         "from meerschaum.connectors import Connector, make_connector\n"
     ),
+    'instance-connector': (
+        "from meerschaum.connectors import InstanceConnector, make_connector\n"
+    ),
     'action': (
         "from meerschaum.actions import make_action\n"
     ),
@@ -106,6 +111,501 @@ FEATURE_LINES: Dict[str, str] = {
         "        # populate docs with dictionaries (rows).\n"
         "        return docs\n\n\n"
     ),
+    'instance-connector': (
+        "@make_connector\n"
+        "class {plugin_name_capitalized}Connector(InstanceConnector):\n"
+        "    \"\"\"Implement '{plugin_name_lower}' connectors.\"\"\"\n\n"
+        "    REQUIRED_ATTRIBUTES: list[str] = []\n"
+        "\n"
+        "    def fetch(\n"
+        "        self,\n"
+        "        pipe: mrsm.Pipe,\n"
+        "        begin: datetime | None = None,\n"
+        "        end: datetime | None = None,\n"
+        "        **kwargs\n"
+        "    ):\n"
+        "        \"\"\"Return or yield dataframes.\"\"\"\n"
+        "        docs = []\n"
+        "        # populate docs with dictionaries (rows).\n"
+        "        return docs\n"
+        """
+    def register_pipe(
+        self,
+        pipe: mrsm.Pipe,
+        debug: bool = False,
+        **kwargs: Any
+    ) -> mrsm.SuccessTuple:
+        \"\"\"
+        Insert the pipe's attributes into the internal `pipes` table.
+
+        Parameters
+        ----------
+        pipe: mrsm.Pipe
+            The pipe to be registered.
+
+        Returns
+        -------
+        A `SuccessTuple` of the result.
+        \"\"\"
+        attributes = {{
+            'connector_keys': str(pipe.connector_keys),
+            'metric_key': str(pipe.metric_key),
+            'location_key': str(pipe.location_key),
+            'parameters': pipe._attributes.get('parameters', {{}}),
+        }}
+
+        ### TODO insert `attributes` as a row in the pipes table.
+        # self.pipes_collection.insert_one(attributes)
+
+        return True, \"Success\"
+
+    def get_pipe_attributes(
+        self,
+        pipe: mrsm.Pipe,
+        debug: bool = False,
+        **kwargs: Any
+    ) -> dict[str, Any]:
+        \"\"\"
+        Return the pipe's document from the internal `pipes` collection.
+
+        Parameters
+        ----------
+        pipe: mrsm.Pipe
+            The pipe whose attributes should be retrieved.
+
+        Returns
+        -------
+        The document that matches the keys of the pipe.
+        \"\"\"
+        query = {{
+            'connector_keys': str(pipe.connector_keys),
+            'metric_key': str(pipe.metric_key),
+            'location_key': str(pipe.location_key),
+        }}
+        ### TODO query the `pipes` table either using these keys or `get_pipe_id()`.
+        result = {{}}
+        # result = self.pipes_collection.find_one(query) or {{}}
+        return result
+
+    def get_pipe_id(
+        self,
+        pipe: mrsm.Pipe,
+        debug: bool = False,
+        **kwargs: Any
+    ) -> str | int | None:
+        \"\"\"
+        Return the ID for the pipe if it exists.
+
+        Parameters
+        ----------
+        pipe: mrsm.Pipe
+            The pipe whose ID to fetch.
+
+        Returns
+        -------
+        The ID for the pipe or `None`.
+        \"\"\"
+        query = {{
+            'connector_keys': str(pipe.connector_keys),
+            'metric_key': str(pipe.metric_key),
+            'location_key': str(pipe.location_key),
+        }}
+        ### TODO fetch the ID mapped to this pipe.
+        return None
+
+    def edit_pipe(
+        self,
+        pipe: mrsm.Pipe,
+        debug: bool = False,
+        **kwargs: Any
+    ) -> mrsm.SuccessTuple:
+        \"\"\"
+        Edit the attributes of the pipe.
+
+        Parameters
+        ----------
+        pipe: mrsm.Pipe
+            The pipe whose in-memory parameters must be persisted.
+
+        Returns
+        -------
+        A `SuccessTuple` indicating success.
+        \"\"\"
+        query = {{
+            'connector_keys': str(pipe.connector_keys),
+            'metric_key': str(pipe.metric_key),
+            'location_key': str(pipe.location_key),
+        }}
+        pipe_parameters = pipe._attributes.get('parameters', {{}})
+        ### TODO Update the row with new parameters.
+        # self.pipes_collection.update_one(query, {{'$set': {{'parameters': pipe_parameters}}}})
+        return True, "Success"
+
+    def delete_pipe(
+        self,
+        pipe: mrsm.Pipe,
+        debug: bool = False,
+        **kwargs: Any
+    ) -> mrsm.SuccessTuple:
+        \"\"\"
+        Delete a pipe's registration from the `pipes` collection.
+
+        Parameters
+        ----------
+        pipe: mrsm.Pipe
+            The pipe to be deleted.
+
+        Returns
+        -------
+        A `SuccessTuple` indicating success.
+        \"\"\"
+        ### TODO Delete the pipe's row from the pipes table.
+        # self.pipes_collection.delete_one({{'_id': pipe_id}})
+        return True, "Success"
+
+    def fetch_pipes_keys(
+        self,
+        connector_keys: list[str] | None = None,
+        metric_keys: list[str] | None = None,
+        location_keys: list[str] | None = None,
+        tags: list[str] | None = None,
+        debug: bool = False,
+        **kwargs: Any
+    ) -> list[tuple[str, str, str]]:
+        \"\"\"
+        Return a list of tuples for the registered pipes' keys according to the provided filters.
+
+        Parameters
+        ----------
+        connector_keys: list[str] | None, default None
+            The keys passed via `-c`.
+
+        metric_keys: list[str] | None, default None
+            The keys passed via `-m`.
+
+        location_keys: list[str] | None, default None
+            The keys passed via `-l`.
+
+        tags: List[str] | None, default None
+            Tags passed via `--tags` which are stored under `parameters:tags`.
+
+        Returns
+        -------
+        A list of connector, metric, and location keys in tuples.
+        You may return the string "None" for location keys in place of nulls.
+
+        Examples
+        --------
+        >>> import meerschaum as mrsm
+        >>> conn = mrsm.get_connector('example:demo')
+        >>> 
+        >>> pipe_a = mrsm.Pipe('a', 'demo', tags=['foo'], instance=conn)
+        >>> pipe_b = mrsm.Pipe('b', 'demo', tags=['bar'], instance=conn)
+        >>> pipe_a.register()
+        >>> pipe_b.register()
+        >>> 
+        >>> conn.fetch_pipes_keys(['a', 'b'])
+        [('a', 'demo', 'None'), ('b', 'demo', 'None')]
+        >>> conn.fetch_pipes_keys(metric_keys=['demo'])
+        [('a', 'demo', 'None'), ('b', 'demo', 'None')]
+        >>> conn.fetch_pipes_keys(tags=['foo'])
+        [('a', 'demo', 'None')]
+        >>> conn.fetch_pipes_keys(location_keys=[None])
+        [('a', 'demo', 'None'), ('b', 'demo', 'None')]
+        
+        \"\"\"
+        from meerschaum.utils.misc import separate_negation_values
+
+        in_ck, nin_ck = separate_negation_values([str(val) for val in (connector_keys or [])])
+        in_mk, nin_mk = separate_negation_values([str(val) for val in (metric_keys or [])])
+        in_lk, nin_lk = separate_negation_values([str(val) for val in (location_keys or [])])
+        in_tags, nin_tags = separate_negation_values([str(val) for val in (tags or [])])
+
+        ### TODO build a query like so, only including clauses if the given list is not empty.
+        ### The `tags` clause is an OR ("?|"), meaning any of the tags may match.
+        ### 
+        ### 
+        ### SELECT connector_keys, metric_key, location_key
+        ### FROM pipes
+        ### WHERE connector_keys IN ({{in_ck}})
+        ###   AND connector_keys NOT IN ({{nin_ck}})
+        ###   AND metric_key IN ({{in_mk}})
+        ###   AND metric_key NOT IN ({{nin_mk}})
+        ###   AND location_key IN ({{in_lk}})
+        ###   AND location_key NOT IN ({{nin_lk}})
+        ###   AND (parameters->'tags')::JSONB ?| ARRAY[{{tags}}]
+        ###   AND NOT (parameters->'tags')::JSONB ?| ARRAY[{{nin_tags}}]
+        return []
+
+    def pipe_exists(
+        self,
+        pipe: mrsm.Pipe,
+        debug: bool = False,
+        **kwargs: Any
+    ) -> bool:
+        \"\"\"
+        Check whether a pipe's target table exists.
+
+        Parameters
+        ----------
+        pipe: mrsm.Pipe
+            The pipe to check whether its table exists.
+
+        Returns
+        -------
+        A `bool` indicating the table exists.
+        \"\"\"
+        table_name = pipe.target
+        ### TODO write a query to determine the existence of `table_name`.
+        table_exists = False
+        return table_exists
+
+    def drop_pipe(
+        self,
+        pipe: mrsm.Pipe,
+        debug: bool = False,
+        **kwargs: Any
+    ) -> mrsm.SuccessTuple:
+        \"\"\"
+        Drop a pipe's collection if it exists.
+
+        Parameters
+        ----------
+        pipe: mrsm.Pipe
+            The pipe to be dropped.
+
+        Returns
+        -------
+        A `SuccessTuple` indicating success.
+        \"\"\"
+        ### TODO write a query to drop `table_name`.
+        table_name = pipe.target
+        return True, \"Success\"
+
+    def sync_pipe(
+        self,
+        pipe: mrsm.Pipe,
+        df: 'pd.DataFrame',
+        debug: bool = False,
+        **kwargs: Any
+    ) -> mrsm.SuccessTuple:
+        \"\"\"
+        Upsert new documents into the pipe's target table.
+
+        Parameters
+        ----------
+        pipe: mrsm.Pipe
+            The pipe to which the data should be upserted.
+
+        df: pd.DataFrame
+            The data to be synced.
+
+        Returns
+        -------
+        A `SuccessTuple` indicating success.
+        \"\"\"
+        ### TODO Write the upsert logic for the target table.
+        ### `pipe.filter_existing()` is provided for your convenience to
+        ### remove duplicates and separate inserts from updates.
+
+        unseen_df, update_df, delta_df = pipe.filter_existing(df, debug=debug)
+        return True, \"Success\"
+
+    def clear_pipe(
+        self,
+        pipe: mrsm.Pipe,
+        begin: datetime | int | None = None,
+        end: datetime | int | None = None,
+        params: dict[str, Any] | None = None,
+        debug: bool = False,
+    ) -> mrsm.SuccessTuple:
+        \"\"\"
+        Delete rows within `begin`, `end`, and `params`.
+
+        Parameters
+        ----------
+        pipe: mrsm.Pipe
+            The pipe whose rows to clear.
+
+        begin: datetime | int | None, default None
+            If provided, remove rows >= `begin`.
+
+        end: datetime | int | None, default None
+            If provided, remove rows < `end`.
+
+        params: dict[str, Any] | None, default None
+            If provided, only remove rows which match the `params` filter.
+
+        Returns
+        -------
+        A `SuccessTuple` indicating success.
+        \"\"\"
+        ### TODO Write a query to remove rows which match `begin`, `end`, and `params`.
+        return True, \"Success\"
+
+    def get_pipe_data(
+        self,
+        pipe: mrsm.Pipe,
+        select_columns: list[str] | None = None,
+        omit_columns: list[str] | None = None,
+        begin: datetime | int | None = None,
+        end: datetime | int | None = None,
+        params: dict[str, Any] | None = None,
+        debug: bool = False,
+        **kwargs: Any
+    ) -> Union['pd.DataFrame', None]:
+        \"\"\"
+        Query a pipe's target table and return the DataFrame.
+
+        Parameters
+        ----------
+        pipe: mrsm.Pipe
+            The pipe with the target table from which to read.
+
+        select_columns: list[str] | None, default None
+            If provided, only select these given columns.
+            Otherwise select all available columns (i.e. `SELECT *`).
+
+        omit_columns: list[str] | None, default None
+            If provided, remove these columns from the selection.
+
+        begin: datetime | int | None, default None
+            The earliest `datetime` value to search from (inclusive).
+
+        end: datetime | int | None, default None
+            The lastest `datetime` value to search from (exclusive).
+
+        params: dict[str | str] | None, default None
+            Additional filters to apply to the query.
+
+        Returns
+        -------
+        The target table's data as a DataFrame.
+        \"\"\"
+        if not pipe.exists(debug=debug):
+            return None
+
+        table_name = pipe.target
+        dt_col = pipe.columns.get(\"datetime\", None)
+
+        ### TODO Write a query to fetch from `table_name`
+        ###      and apply the filters `begin`, `end`, and `params`.
+        ### 
+        ###      To improve performance, add logic to only read from
+        ###      `select_columns` and not `omit_columns` (if provided).
+        ### 
+        ### SELECT {{', '.join(cols_to_select)}}
+        ### FROM \"{{table_name}}\"
+        ### WHERE \"{{dt_col}}\" >= '{{begin}}'
+        ###   AND \"{{dt_col}}\" <  '{{end}}'
+
+        ### The function `parse_df_datetimes()` is a convenience function
+        ### to cast a list of dictionaries into a DataFrame and convert datetime columns.
+        from meerschaum.utils.dataframe import parse_df_datetimes
+        rows = []
+        return parse_df_datetimes(rows)
+
+    def get_sync_time(
+        self,
+        pipe: mrsm.Pipe,
+        params: dict[str, Any] | None = None,
+        newest: bool = True,
+        debug: bool = False,
+        **kwargs: Any
+    ) -> datetime | int | None:
+        \"\"\"
+        Return the most recent value for the `datetime` axis.
+
+        Parameters
+        ----------
+        pipe: mrsm.Pipe
+            The pipe whose collection contains documents.
+
+        params: dict[str, Any] | None, default None
+            Filter certain parameters when determining the sync time.
+
+        newest: bool, default True
+            If `True`, return the maximum value for the column.
+
+        Returns
+        -------
+        The largest `datetime` or `int` value of the `datetime` axis. 
+        \"\"\"
+        ### TODO write a query to get the largest value for `dt_col`.
+        ### If `newest` is `False`, return the smallest value.
+        ### Apply the `params` filter in case of multiplexing.
+        return None
+
+    def get_pipe_columns_types(
+        self,
+        pipe: mrsm.Pipe,
+        debug: bool = False,
+        **kwargs: Any
+    ) -> dict[str, str]:
+        \"\"\"
+        Return the data types for the columns in the target table for data type enforcement.
+
+        Parameters
+        ----------
+        pipe: mrsm.Pipe
+            The pipe whose target table contains columns and data types.
+
+        Returns
+        -------
+        A dictionary mapping columns to data types.
+        \"\"\"
+        table_name = pipe.target
+        ### TODO write a query to fetch the columns contained in `table_name`.
+        columns_types = {{}}
+
+        ### Return a dictionary mapping the columns
+        ### to their Pandas dtypes, e.g.:
+        ### `{{'foo': 'int64'`}}`
+        ### or to SQL-style dtypes, e.g.:
+        ### `{{'bar': 'INT'}}`
+        return columns_types
+
+    def get_pipe_rowcount(
+        self,
+        pipe: mrsm.Pipe,
+        begin: datetime | int | None = None,
+        end: datetime | int | None = None,
+        params: dict[str, Any] | None = None,
+        remote: bool = False,
+        debug: bool = False,
+        **kwargs: Any
+    ) -> int:
+        \"\"\"
+        Return the rowcount for the pipe's table.
+
+        Parameters
+        ----------
+        pipe: mrsm.Pipe
+            The pipe whose table should be counted.
+
+        begin: datetime | int | None, default None
+            If provided, only count rows >= `begin`.
+
+        end: datetime | int | None, default None
+            If provided, only count rows < `end`.
+
+        params: dict[str, Any] | None
+            If provided, only count rows othat match the `params` filter.
+
+        remote: bool, default False
+            If `True`, return the rowcount for the pipe's fetch definition.
+            In this case, `self` refers to `Pipe.connector`, not `Pipe.instance_connector`.
+
+        Returns
+        -------
+        The rowcount for this pipe's table according the given parameters.
+        \"\"\"
+        ### TODO write a query to count how many rows exist in `table_name` according to the filters.
+        table_name = pipe.target
+        count = 0
+        return count
+"""
+    ),
     'action': (
         "@make_action\n"
         "def {action_name}(**kwargs) -> mrsm.SuccessTuple:\n"
@@ -212,8 +712,10 @@ def bootstrap_plugin(
     body_text += FEATURE_LINES['header'].format(**plugin_labels)
     body_text += IMPORTS_LINES['stdlib'].format(**plugin_labels)
     body_text += IMPORTS_LINES['default'].format(**plugin_labels)
-    if 'connector' in features:
+    if 'connector' in features and 'instance-connector' not in features:
         body_text += IMPORTS_LINES['connector'].format(**plugin_labels)
+    if 'instance-connector' in features:
+        body_text += IMPORTS_LINES['instance-connector'].format(**plugin_labels)
     if 'action' in features:
         body_text += IMPORTS_LINES['action'].format(**plugin_labels)
     if 'api' in features and 'web' in features:
@@ -231,9 +733,12 @@ def bootstrap_plugin(
         body_text += FEATURE_LINES['register'].format(**plugin_labels)
         body_text += FEATURE_LINES['fetch'].format(**plugin_labels)
 
-    if 'connector' in features:
+    if 'connector' in features and 'instance-connector' not in features:
         body_text += FEATURE_LINES['connector'].format(**plugin_labels)
 
+    if 'instance-connector' in features:
+        body_text += FEATURE_LINES['instance-connector'].format(**plugin_labels)
+
     if 'action' in features:
         body_text += FEATURE_LINES['action'].format(**plugin_labels)
 

meerschaum/utils/_get_pipes.py

@@ -128,11 +128,12 @@ def get_pipes(
     ```
     """
 
+    import json
+    from collections import defaultdict
     from meerschaum.config import get_config
     from meerschaum.utils.warnings import error
     from meerschaum.utils.misc import filter_keywords
     from meerschaum.utils.pool import get_pool
-    from collections import defaultdict
 
     if connector_keys is None:
         connector_keys = []
@@ -188,25 +189,48 @@ def get_pipes(
         debug = debug
     )
     if result is None:
-        error(f"Unable to build pipes!")
+        error("Unable to build pipes!")
 
     ### Populate the `pipes` dictionary with Pipes based on the keys
     ### obtained from the chosen `method`.
     from meerschaum import Pipe
     pipes = {}
-    for ck, mk, lk in result:
+    for keys_tuple in result:
+        ck, mk, lk = keys_tuple[0], keys_tuple[1], keys_tuple[2]
+        pipe_tags_or_parameters = keys_tuple[3] if len(keys_tuple) == 4 else None
+        pipe_parameters = (
+            pipe_tags_or_parameters
+            if isinstance(pipe_tags_or_parameters, (dict, str))
+            else None
+        )
+        if isinstance(pipe_parameters, str):
+            pipe_parameters = json.loads(pipe_parameters)
+        pipe_tags = (
+            pipe_tags_or_parameters
+            if isinstance(pipe_tags_or_parameters, list)
+            else (
+                pipe_tags_or_parameters.get('tags', None)
+                if isinstance(pipe_tags_or_parameters, dict)
+                else None
+            )
+        )
+
         if ck not in pipes:
             pipes[ck] = {}
 
         if mk not in pipes[ck]:
             pipes[ck][mk] = {}
 
-        pipes[ck][mk][lk] = Pipe(
+        pipe = Pipe(
             ck, mk, lk,
             mrsm_instance = connector,
+            parameters = pipe_parameters,
+            tags = pipe_tags,
             debug = debug,
             **filter_keywords(Pipe, **kw)
         )
+        pipe.__dict__['_tags'] = pipe_tags
+        pipes[ck][mk][lk] = pipe
 
     if not as_list and not as_tags_dict:
         return pipes
@@ -218,7 +242,9 @@ def get_pipes(
 
     pool = get_pool(workers=(workers if connector.IS_THREAD_SAFE else 1))
     def gather_pipe_tags(pipe: mrsm.Pipe) -> Tuple[mrsm.Pipe, List[str]]:
-        return pipe, (pipe.tags or [])
+        _tags = pipe.__dict__.get('_tags', None)
+        gathered_tags = _tags if _tags is not None else pipe.tags
+        return pipe, (gathered_tags or [])
 
     tags_pipes = defaultdict(lambda: [])
     pipes_tags = dict(pool.map(gather_pipe_tags, pipes_list))

meerschaum/utils/dataframe.py

@@ -1508,7 +1508,7 @@ def enforce_dtypes(
             )
         )
 
-        if debug:
+        if debug and (explicitly_numeric or df_numeric_cols or mixed_numeric_types):
             from meerschaum.utils.formatting import make_header
             msg = (
                 make_header(f"Coercing column '{col}' to numeric:", left_pad=0)
@@ -1517,7 +1517,13 @@ def enforce_dtypes(
                 + f"  Current type: {typ if col not in df_numeric_cols else 'Decimal'}"
                 + ("\n  Column is explicitly numeric." if explicitly_numeric else "")
             ) if cast_to_numeric else (
-                f"Will not coerce column '{col}' to numeric."
+                f"Will not coerce column '{col}' to numeric.\n"
+                f"  Numeric columns in dataframe: {df_numeric_cols}\n"
+                f"  Mixed numeric types: {mixed_numeric_types}\n"
+                f"  Explicitly float: {explicitly_float}\n"
+                f"  Explicitly int: {explicitly_int}\n"
+                f"  All NaN: {all_nan}\n"
+                f"  Coerce numeric: {coerce_numeric}"
             )
             dprint(msg)
 

meerschaum/utils/dtypes/__init__.py

@@ -233,8 +233,8 @@ def are_dtypes_equal(
         return True
 
     date_dtypes = (
-        'date', 'date32[pyarrow]', 'date32[day][pyarrow]',
-        'date64[pyarrow]', 'date64[ms][pyarrow]',
+        'date', 'date32', 'date32[pyarrow]', 'date32[day][pyarrow]',
+        'date64', 'date64[pyarrow]', 'date64[ms][pyarrow]',
     )
     if ldtype in date_dtypes and rdtype in date_dtypes:
         return True
@@ -1141,7 +1141,6 @@ def round_time(
 ) -> datetime:
     """
     Round a datetime object to a multiple of a timedelta.
-    http://stackoverflow.com/questions/3463930/how-to-round-the-minute-of-a-datetime-object-python
 
     Parameters
     ----------