meerschaum 3.0.5__py3-none-any.whl → 3.0.6__py3-none-any.whl

meerschaum/_internal/static.py

@@ -192,6 +192,16 @@ STATIC_CONFIG: Dict[str, Any] = {
                 'requirements': {'database'},
                 'defaults': {},
             },
+            'geopackage': {
+                'engine': 'sqlite',
+                'create_engine': _default_create_engine_args,
+                'omit_create_engine': {'method',},
+                'to_sql': {
+                    'method': 'multi',
+                },
+                'requirements': {'database'},
+                'defaults': {},
+            },
             'duckdb': {
                 'engine': 'duckdb',
                 'create_engine': {},

meerschaum/actions/copy.py

@@ -99,7 +99,7 @@ def _copy_pipes(
         new_pipe = Pipe(
             ck, mk, lk,
             instance=instance_keys,
-            parameters=pipe.parameters.copy(),
+            parameters=pipe.get_parameters(apply_symlinks=False),
         )
 
         if new_pipe.get_id(debug=debug) is not None:

meerschaum/actions/delete.py

@@ -349,7 +349,7 @@ def _delete_connectors(
     for c in to_delete:
         try:
             ### Remove database files.
-            if c.flavor in ('sqlite', 'duckdb'):
+            if c.flavor in ('sqlite', 'duckdb', 'geopackage'):
                 if ':memory:' not in c.database and pathlib.Path(c.database).exists():
                     if force or yes_no(
                         f"Detected '{c.flavor}' database '{c.database}'. "

meerschaum/api/dash/callbacks/dashboard.py

@@ -51,7 +51,7 @@ from meerschaum.utils.misc import filter_keywords, flatten_list, string_to_dict
 from meerschaum.utils.yaml import yaml
 from meerschaum.actions import get_subactions, actions
 from meerschaum._internal.arguments._parser import parser
-from meerschaum.connectors.sql._fetch import set_pipe_query
+from meerschaum.connectors.sql._fetch import set_pipe_query, get_pipe_query
 dash = attempt_import('dash', lazy=False, check_update=CHECK_UPDATE)
 dbc = attempt_import('dash_bootstrap_components', lazy=False, check_update=CHECK_UPDATE)
 dcc, html = import_dcc(check_update=CHECK_UPDATE), import_html(check_update=CHECK_UPDATE)
@@ -919,14 +919,31 @@ def update_pipe_sql_click(n_clicks, sql_editor_text):
         success, msg = False, f"Unable to update SQL definition for {pipe}."
     else:
         try:
-            set_pipe_query(pipe, sql_editor_text)
-            success, msg = pipe.edit(debug=debug)
+            success, msg = set_pipe_query(pipe, sql_editor_text)
         except Exception as e:
             success, msg = False, f"Invalid SQL query:\n{e}"
 
     return alert_from_success_tuple((success, msg))
 
 
+@dash_app.callback(
+    Output({'type': 'sql-editor', 'index': MATCH}, 'value'),
+    Input({'type': 'resolve-symlinks-switch', 'index': MATCH}, 'value'),
+    prevent_initial_call=True,
+)
+def toggle_sql_symlinks(value):
+    triggered = dash.callback_context.triggered
+    if triggered[0]['value'] is None:
+        raise PreventUpdate
+
+    pipe = pipe_from_ctx(triggered, 'value')
+    if pipe is None:
+        raise PreventUpdate
+
+    query = get_pipe_query(pipe, apply_symlinks=value)
+    return query
+
+
 @dash_app.callback(
     Output({'type': 'sync-success-div', 'index': MATCH}, 'children'),
     Input({'type': 'update-sync-button', 'index': MATCH}, 'n_clicks'),

meerschaum/api/dash/pipes.py

@@ -690,16 +690,32 @@ def accordion_items_from_pipe(
         items_bodies['sql'] = html.Div([
             sql_editor,
             html.Br(),
-            dbc.Row([
-                dbc.Col([update_sql_button], width=2),
-                dbc.Col([
-                    html.Div(
-                        id={'type': 'update-sql-success-div', 'index': pipe_meta_str}
-                    )
+            dbc.Row(
+                [
+                    dbc.Col(update_sql_button, width='auto'),
+                    dbc.Col(
+                        dbc.Switch(
+                            label="Resolve symlinks",
+                            value=True,
+                            id={'type': 'resolve-symlinks-switch', 'index': pipe_meta_str},
+                        ),
+                        width='auto',
+                    ),
                 ],
-                width=True,
-                )
-            ]),
+                justify='between',
+                align='center',
+            ),
+            dbc.Row(
+                dbc.Col(
+                    html.Div(
+                        id={
+                            'type': 'update-sql-success-div',
+                            'index': pipe_meta_str,
+                        },
+                    ),
+                    width=True,
+                ),
+            ),
         ])
 
     if 'recent-data' in active_items:

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "3.0.5"
+__version__ = "3.0.6"

meerschaum/connectors/sql/_SQLConnector.py

@@ -36,6 +36,7 @@ class SQLConnector(InstanceConnector):
         exec_queries,
         get_connection,
         _cleanup_connections,
+        _init_geopackage_table,
     )
     from meerschaum.utils.sql import test_connection
     from ._fetch import fetch, get_pipe_metadef
@@ -176,7 +177,7 @@ class SQLConnector(InstanceConnector):
             **kw
         )
 
-        if self.__dict__.get('flavor', None) == 'sqlite':
+        if self.__dict__.get('flavor', None) in ('sqlite', 'geopackage'):
             self._reset_attributes()
             self._set_attributes(
                 'sql',
@@ -187,7 +188,7 @@ class SQLConnector(InstanceConnector):
             ### For backwards compatability reasons, set the path for sql:local if its missing.
             if self.label == 'local' and not self.__dict__.get('database', None):
                 from meerschaum.config._paths import SQLITE_DB_PATH
-                self.database = str(SQLITE_DB_PATH)
+                self.database = SQLITE_DB_PATH.as_posix()
 
         ### ensure flavor and label are set accordingly
         if 'flavor' not in self.__dict__:
@@ -291,7 +292,7 @@ class SQLConnector(InstanceConnector):
         """
         if self.flavor in ('duckdb', 'oracle'):
             return False
-        if self.flavor == 'sqlite':
+        if self.flavor in ('sqlite', 'geopackage'):
             return ':memory:' not in self.URI
         return True
 

meerschaum/connectors/sql/_cli.py

@@ -24,6 +24,7 @@ flavor_clis = {
     'mariadb'        : 'mycli',
     'percona'        : 'mycli',
     'sqlite'         : 'litecli',
+    'geopackage'     : 'litecli',
     'mssql'          : 'mssqlcli',
     'duckdb'         : 'gadwall',
 }
@@ -105,7 +106,7 @@ def _cli_exit(
     ### NOTE: The `DATABASE_URL` property must be initialized first in case the database is not
     ### yet defined (e.g. 'sql:local').
     cli_arg_str = self.DATABASE_URL
-    if self.flavor in ('sqlite', 'duckdb'):
+    if self.flavor in ('sqlite', 'duckdb', 'geopackage'):
         cli_arg_str = (
             str(self.database)
             if 'database' in self.__dict__

meerschaum/connectors/sql/_create_engine.py

@@ -19,6 +19,7 @@ from meerschaum._internal.static import STATIC_CONFIG
 flavor_configs = STATIC_CONFIG['sql']['create_engine_flavors']
 install_flavor_drivers = {
     'sqlite': ['aiosqlite'],
+    'geopackage': ['aiosqlite'],
     'duckdb': ['duckdb', 'duckdb_engine'],
     'mysql': ['pymysql'],
     'mariadb': ['pymysql'],
@@ -91,7 +92,7 @@ def create_engine(
         sqlalchemy.dialects.registry.register(*flavor_dialects[self.flavor])
 
     ### self._sys_config was deepcopied and can be updated safely
-    if self.flavor in ("sqlite", "duckdb"):
+    if self.flavor in ("sqlite", "duckdb", "geopackage"):
         engine_str = f"{_engine}:///{_database}" if not _uri else _uri
         if 'create_engine' not in self._sys_config:
             self._sys_config['create_engine'] = {}

meerschaum/connectors/sql/_fetch.py

@@ -221,7 +221,11 @@ def get_pipe_metadef(
     return meta_def.rstrip()
 
 
-def get_pipe_query(pipe: mrsm.Pipe, warn: bool = True) -> Union[str, None]:
+def get_pipe_query(
+    pipe: mrsm.Pipe,
+    apply_symlinks: bool = True,
+    warn: bool = True,
+) -> Union[str, None]:
     """
     Run through the possible keys for a pipe's query and return the first match.
 
@@ -233,14 +237,17 @@ def get_pipe_query(pipe: mrsm.Pipe, warn: bool = True) -> Union[str, None]:
     import textwrap
     from meerschaum.utils.warnings import warn as _warn
     from meerschaum.utils.pipes import replace_pipes_syntax
-    if pipe.parameters.get('fetch', {}).get('definition', None):
-        definition = pipe.parameters['fetch']['definition']
+
+    parameters = pipe.get_parameters(apply_symlinks=apply_symlinks)
+
+    if parameters.get('fetch', {}).get('definition', None):
+        definition = parameters['fetch']['definition']
     elif pipe.parameters.get('definition', None):
-        definition = pipe.parameters['definition']
+        definition = parameters['definition']
     elif pipe.parameters.get('query', None):
-        definition = pipe.parameters['query']
+        definition = parameters['query']
     elif pipe.parameters.get('sql', None):
-        definition = pipe.parameters['sql']
+        definition = parameters['sql']
     else:
         if warn:
             _warn(
@@ -249,11 +256,12 @@ def get_pipe_query(pipe: mrsm.Pipe, warn: bool = True) -> Union[str, None]:
             )
         return None
 
-    definition = replace_pipes_syntax(definition)
+    if apply_symlinks:
+        definition = replace_pipes_syntax(definition)
     return textwrap.dedent(definition.lstrip().rstrip())
 
 
-def set_pipe_query(pipe: mrsm.Pipe, query: str) -> None:
+def set_pipe_query(pipe: mrsm.Pipe, query: str) -> mrsm.SuccessTuple:
     """
     Run through the possible keys for a pipe's query and set the first match.
 
@@ -262,22 +270,17 @@ def set_pipe_query(pipe: mrsm.Pipe, query: str) -> None:
     - query
     - sql
     """
-    if 'fetch' in pipe.parameters and 'definition' in pipe.parameters['fetch']:
-        if pipe.parameters.get('fetch', None) is None:
-            pipe.parameters['fetch'] = {}
-        dict_to_set = pipe.parameters['fetch']
-        key_to_set = 'definition'
-    elif 'definition' in pipe.parameters:
-        dict_to_set = pipe.parameters
-        key_to_set = 'definition'
-    elif 'query' in pipe.parameters:
-        dict_to_set = pipe.parameters
-        key_to_set = 'query'
+    parameters = pipe.get_parameters()
+    if 'fetch' in parameters and 'definition' in parameters['fetch']:
+        patch_dict = {'fetch': {'defintion': query}}
+    elif 'definition' in parameters:
+        patch_dict = {'definition': query}
+    elif 'query' in parameters:
+        patch_dict = {'query': query}
     else:
-        dict_to_set = pipe.parameters
-        key_to_set = 'sql'
+        patch_dict = {'sql': query}
 
-    dict_to_set[key_to_set] = query
+    return pipe.update_parameters(patch_dict)
 
 
 def _simple_fetch_query(

meerschaum/connectors/sql/_pipes.py

@@ -706,7 +706,7 @@ def get_create_index_queries(
                 ),
             ])
         elif not autoincrement and primary_key in existing_cols_pd_types:
-            if self.flavor == 'sqlite':
+            if self.flavor in ('sqlite', 'geopackage'):
                 new_table_name = sql_item_name(
                     f'_new_{pipe.target}',
                     self.flavor,
@@ -888,7 +888,7 @@ def get_create_index_queries(
         f"CREATE UNIQUE INDEX {unique_index_name} ON {_pipe_name} ({unique_index_cols_str})"
     )
     constraint_queries = [create_unique_index_query]
-    if self.flavor != 'sqlite':
+    if self.flavor not in ('sqlite', 'geopackage'):
         constraint_queries.append(add_constraint_query)
     if upsert and indices_cols_str:
         index_queries[unique_index_name] = constraint_queries
@@ -977,7 +977,12 @@ def get_drop_index_queries(
         if ix_unquoted.lower() not in existing_indices:
             continue
 
-        if ix_key == 'unique' and upsert and self.flavor not in ('sqlite',) and not is_hypertable:
+        if (
+            ix_key == 'unique'
+            and upsert
+            and self.flavor not in ('sqlite', 'geopackage')
+            and not is_hypertable
+        ):
             constraint_name_unquoted = ix_unquoted.replace('IX_', 'UQ_')
             constraint_name = sql_item_name(constraint_name_unquoted, self.flavor)
             constraint_or_index = (
@@ -1558,6 +1563,11 @@ def create_pipe_table_from_df(
         get_create_schema_if_not_exists_queries,
     )
     from meerschaum.utils.dtypes.sql import get_db_type_from_pd_type
+    if self.flavor == 'geopackage':
+        init_success, init_msg = self._init_geopackage_table(df, pipe.target, debug=debug)
+        if not init_success:
+            return init_success, init_msg
+
     primary_key = pipe.columns.get('primary', None)
     primary_key_typ = (
         pipe.dtypes.get(primary_key, str(df.dtypes.get(primary_key, 'int')))
@@ -1614,6 +1624,9 @@ def create_pipe_table_from_df(
         if success
         else f"Failed to create {target_name}."
     )
+    if success and self.flavor == 'geopackage':
+        return self._init_geopackage_table(df, target, debug=debug)
+
     return success, msg
 
 
@@ -3078,7 +3091,7 @@ def get_pipe_columns_types(
     if not pipe.exists(debug=debug):
         return {}
 
-    if self.flavor not in ('oracle', 'mysql', 'mariadb', 'sqlite'):
+    if self.flavor not in ('oracle', 'mysql', 'mariadb', 'sqlite', 'geopackage'):
         return get_table_cols_types(
             pipe.target,
             self,
@@ -3436,7 +3449,7 @@ def get_alter_columns_queries(
         for col, (db_typ, typ) in altered_cols.items()
     }
 
-    if self.flavor == 'sqlite':
+    if self.flavor in ('sqlite', 'geopackage'):
         temp_table_name = '-' + session_id + '_' + target
         rename_query = (
             "ALTER TABLE "
@@ -3884,7 +3897,7 @@ def get_pipe_schema(self, pipe: mrsm.Pipe) -> Union[str, None]:
     -------
     A schema string or `None` if nothing is configured.
     """
-    if self.flavor == 'sqlite':
+    if self.flavor in ('sqlite', 'geopackage'):
         return self.schema
     return pipe.parameters.get('schema', self.schema)
 

meerschaum/connectors/sql/_sql.py

@@ -27,7 +27,7 @@ _bulk_flavors = {
 }
 ### flavors that do not support chunks
 _disallow_chunks_flavors = ['duckdb']
-_max_chunks_flavors = {'sqlite': 1000}
+_max_chunks_flavors = {'sqlite': 1000, 'geopackage': 1000}
 SKIP_READ_TRANSACTION_FLAVORS: list[str] = ['mssql']
 
 
@@ -1347,3 +1347,33 @@ def _cleanup_connections(self) -> None:
             connection.close()
         except Exception:
             pass
+
+
+def _init_geopackage_table(
+    self,
+    df: 'pd.DataFrame',
+    table: str,
+    debug: bool = False,
+) -> SuccessTuple:
+    """
+    Initialize the geopackage schema tables from a DataFrame.
+    """
+    import pathlib
+    database = self.__dict__.get('database', self.parse_uri(self.URI).get('database', None))
+    if not database:
+        return False, f"Could not determine database for '{self}'."
+
+    database_path = pathlib.Path(database)
+    mode = 'w' if not database_path.exists() else 'a'
+
+    try:
+        df.head(0).to_file(
+            database_path.as_posix(),
+            layer=table,
+            driver='GPKG',
+            index=False,
+            mode=mode,
+        )
+    except Exception as e:
+        return False, f"Failed to init table '{table}':\n{e}"
+    return True, "Success"

meerschaum/connectors/sql/_uri.py

@@ -51,7 +51,7 @@ def from_uri(
     if 'database' not in params:
         error("Unable to determine the database from the provided URI.")
 
-    if flavor in ('sqlite', 'duckdb'):
+    if flavor in ('sqlite', 'duckdb', 'geopackage'):
         if params['database'] == ':memory:':
             params['label'] = label or f'memory_{flavor}'
         else:

meerschaum/connectors/sql/tables/__init__.py

@@ -74,7 +74,7 @@ def get_tables(
     cache_expired = refresh or (
         (
             _check_create_cache(conn, debug=debug)
-            if conn.flavor != 'sqlite'
+            if conn.flavor not in ('sqlite', 'duckdb', 'geopackage')
             else True
         )
         if conn.type == 'sql'
@@ -264,7 +264,7 @@ def get_tables(
 
                 _write_create_cache(mrsm.get_connector(str(mrsm_instance)), debug=debug)
 
-        if conn.flavor != 'sqlite':
+        if conn.flavor not in ('sqlite', 'duckdb', 'geopackage'):
             with open(pickle_path, 'wb') as f:
                 pickle.dump(conn.metadata, f)
 

meerschaum/core/Pipe/_attributes.py

@@ -155,7 +155,7 @@ def columns(self) -> Union[Dict[str, str], None]:
     cols = self.parameters.get('columns', {})
     if not isinstance(cols, dict):
         return {}
-    return {col_ix: col for col_ix, col in cols.items() if col}
+    return {col_ix: col for col_ix, col in cols.items() if col and col_ix}
 
 
 @columns.setter

meerschaum/plugins/__init__.py

@@ -13,7 +13,7 @@ import functools
 
 import meerschaum as mrsm
 from meerschaum.utils.typing import Callable, Any, Union, Optional, Dict, List, Tuple
-from meerschaum.utils.threading import Lock, RLock
+from meerschaum.utils.threading import RLock
 from meerschaum.core.Plugin import Plugin
 
 _api_plugins: Dict[str, List[Callable[['fastapi.App'], Any]]] = {}
@@ -760,7 +760,7 @@ def unload_custom_actions(plugins: Optional[List[str]] = None, debug: bool = Fal
     from meerschaum._internal.entry import _shell
     import meerschaum._internal.shell as shell_pkg
 
-    plugins = plugins or list(_plugins_actions.keys())
+    plugins = plugins if plugins is not None else list(_plugins_actions)
 
     for plugin in plugins:
         action_names = _plugins_actions.get(plugin, [])
@@ -799,7 +799,7 @@ def unload_plugins(
     _synced_symlinks = False
 
     all_plugins = get_plugins_names()
-    plugins = plugins or all_plugins
+    plugins = plugins if plugins is not None else all_plugins
     if debug:
         dprint(f"Unloading plugins: {plugins}")
 

meerschaum/utils/_get_pipes.py

@@ -38,6 +38,7 @@ def get_pipes(
     method: str = 'registered',
     workers: Optional[int] = None,
     debug: bool = False,
+    _cache_parameters: bool = True,
     **kw: Any
 ) -> Union[PipesDict, List[mrsm.Pipe], Dict[str, mrsm.Pipe]]:
     """
@@ -94,7 +95,6 @@ def get_pipes(
 
     **kw: Any:
         Keyword arguments to pass to the `meerschaum.Pipe` constructor.
-        
 
     Returns
     -------
@@ -209,7 +209,7 @@ def get_pipes(
             pipe_tags_or_parameters
             if isinstance(pipe_tags_or_parameters, list)
             else (
-                pipe_tags_or_parameters.get('tags', None)
+                pipe_tags_or_parameters.get('tags', [])
                 if isinstance(pipe_tags_or_parameters, dict)
                 else None
             )