meerschaum 2.1.7__py3-none-any.whl → 2.2.0__py3-none-any.whl

meerschaum/api/resources/templates/termpage.html

@@ -70,6 +70,9 @@ window.addEventListener(
         if (!fl){ continue; }
         fl_val = event.data['input_flags_texts'][index];
         if (!fl_val){ continue; }
+        if (fl_val.includes(' ')){
+          fl_val = "'" + fl_val + "'";
+        }
         flags_str += " " + fl + " " + fl_val;
     }
 

meerschaum/api/routes/_login.py

@@ -39,10 +39,11 @@ def login(
         else (data.username, data.password)
     ) if not no_auth else ('no-auth', 'no-auth')
 
-    from meerschaum.core.User._User import get_pwd_context
+    from meerschaum.core.User._User import verify_password
     user = User(username, password)
-    correct_password = no_auth or get_pwd_context().verify(
-        password, get_api_connector().get_user_password_hash(user, debug=debug)
+    correct_password = no_auth or verify_password(
+        password,
+        get_api_connector().get_user_password_hash(user, debug=debug)
     )
     if not correct_password:
         raise InvalidCredentialsException
@@ -51,7 +52,7 @@ def login(
     expires_delta = timedelta(minutes=expires_minutes)
     expires_dt = datetime.now(timezone.utc).replace(tzinfo=None) + expires_delta
     access_token = manager.create_access_token(
-        data = dict(sub=username),
+        data = {'sub': username},
         expires = expires_delta
     )
     return {

meerschaum/api/routes/_plugins.py

@@ -90,18 +90,21 @@ def register_plugin(
             pass
 
     plugin = Plugin(name, version=version, attributes=attributes)
+    if curr_user is None:
+        return (
+            False,
+            "Cannot register a plugin without logging in (are you running with `--insecure`)?"
+        )
+
     if curr_user is not None:
         plugin_user_id = get_api_connector().get_plugin_user_id(plugin)
         curr_user_id = get_api_connector().get_user_id(curr_user) if curr_user is not None else -1
         if plugin_user_id is not None and plugin_user_id != curr_user_id:
             return False, f"User '{curr_user.username}' cannot edit plugin '{plugin}'."
         plugin.user_id = curr_user_id
-    else:
-        plugin.user_id = -1
 
     success, msg = get_api_connector().register_plugin(plugin, make_archive=False, debug=debug)
 
-    ### TODO delete and install new version of plugin on success
     if success:
         archive_path = plugin.archive_path
         temp_archive_path = pathlib.Path(str(archive_path) + '.tmp')

meerschaum/config/_dash.py

@@ -0,0 +1,11 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+
+"""
+Define default configuration for the Dash application.
+"""
+
+default_dash_config = {
+    'max_num_pipes_cards': 20,
+}

meerschaum/config/_default.py

@@ -44,6 +44,7 @@ default_meerschaum_config = {
             },
             'local': {
                 'host': 'localhost',
+                'port': 8000,
             },
             'mrsm': {
                 'host': 'api.mrsm.io',
@@ -151,7 +152,6 @@ default_config['pipes'] = default_pipes_config
 default_config['plugins'] = default_plugins_config
 from meerschaum.config._jobs import default_jobs_config
 default_config['jobs'] = default_jobs_config
-#  default_config['experimental'] = default_experimental_config
 ### add configs from other packages
 try:
     import meerschaum.config.stack
@@ -160,6 +160,8 @@ except ImportError as e:
 finally:
     from meerschaum.config.stack import default_stack_config
 default_config['stack'] = default_stack_config
+from meerschaum.config._dash import default_dash_config
+default_config['dash'] = default_dash_config
 
 default_header_comment = """
 #####################################################################

meerschaum/config/_jobs.py

@@ -9,13 +9,22 @@ Default configuration for jobs.
 default_jobs_config = {
     'timeout_seconds': 8,
     'check_timeout_interval_seconds': 0.1,
-    'logs' : {
+    'terminal': {
+        'lines': 40,
+        'columns': 70,
+    },
+    'logs': {
+        'timestamps': {
+            'enabled': True,
+            'format': '%Y-%m-%d %H:%M',
+            'follow_format': '%H:%M',
+        },
         'num_files_to_keep': 5,
         'max_file_size': 100_000,
         'lines_to_show': 30,
-        'refresh_files_seconds': 5.0,
-        'min_buffer_len': 15,
-        'colors' : [
+        'refresh_files_seconds': 5,
+        'min_buffer_len': 5,
+        'colors': [
             'cyan',
             'magenta',
             'orange3',

meerschaum/config/_paths.py

@@ -129,6 +129,7 @@ paths = {
 
     'PLUGINS_RESOURCES_PATH'         : ('{INTERNAL_RESOURCES_PATH}', 'plugins'),
     'PLUGINS_INTERNAL_LOCK_PATH'     : ('{INTERNAL_RESOURCES_PATH}', 'plugins.lock'),
+    'PLUGINS_PACKAGES_INTERNAL_PATH' : ('{INTERNAL_RESOURCES_PATH}', 'packaged_plugins'),
     'PLUGINS_ARCHIVES_RESOURCES_PATH': ('{PLUGINS_RESOURCES_PATH}', '.archives'),
     'PLUGINS_TEMP_RESOURCES_PATH'    : ('{PLUGINS_RESOURCES_PATH}', '.tmp'),
     'PLUGINS_INIT_PATH'              : ('{PLUGINS_RESOURCES_PATH}', '__init__.py'),
@@ -153,6 +154,7 @@ paths = {
 
     'DAEMON_RESOURCES_PATH'          : ('{ROOT_DIR_PATH}', 'jobs'),
     'LOGS_RESOURCES_PATH'            : ('{ROOT_DIR_PATH}', 'logs'),
+    'DAEMON_ERROR_LOG_PATH'          : ('{ROOT_DIR_PATH}', 'daemon_errors.log'),
 }
 
 def set_root(root: Union[Path, str]):

meerschaum/config/_sync.py

@@ -52,7 +52,7 @@ def sync_yaml_configs(
         if not path.exists():
             return "", {}
         header_comment = ""
-        with open(path, 'r') as f:
+        with open(path, 'r', encoding='utf-8') as f:
             if _yaml is not None:
                 config = yaml.load(f)
             else:
@@ -84,7 +84,7 @@ def sync_yaml_configs(
     new_path = sub_path
 
     ### write changes
-    with open(new_path, 'w+') as f:
+    with open(new_path, 'w+', encoding='utf-8') as f:
         f.write(new_header)
         f.write(new_config_text)
     if permissions is not None:
@@ -133,4 +133,3 @@ def sync_files(keys: Optional[List[str]] = None):
     for k in keys:
         if k in key_functions:
             key_functions[k]()
-

meerschaum/config/_version.py

@@ -2,4 +2,4 @@
 Specify the Meerschaum release version.
 """
 
-__version__ = "2.1.7"
+__version__ = "2.2.0"

meerschaum/config/stack/__init__.py

@@ -33,7 +33,7 @@ api_host = "api"
 
 env_dict = {
     'COMPOSE_PROJECT_NAME' : 'mrsm',
-    'TIMESCALEDB_VERSION' : 'latest-pg15-oss',
+    'TIMESCALEDB_VERSION' : 'latest-pg16-oss',
     'POSTGRES_USER' : f'{db_user}',
     'POSTGRES_PASSWORD' : f'{db_pass}',
     'POSTGRES_DB' : f'{db_base}',
@@ -97,7 +97,6 @@ compose_header = """
 
 
 default_docker_compose_config = {
-    'version': '3.9',
     'services': {
         'db': {
             'environment': {
@@ -233,11 +232,11 @@ NECESSARY_FILES = [STACK_COMPOSE_PATH, GRAFANA_DATASOURCE_PATH, GRAFANA_DASHBOAR
 def get_necessary_files():
     from meerschaum.config import get_config
     return {
-        STACK_COMPOSE_PATH : (
+        STACK_COMPOSE_PATH: (
             get_config('stack', STACK_COMPOSE_FILENAME, substitute=True), compose_header
         ),
-        GRAFANA_DATASOURCE_PATH : get_config('stack', 'grafana', 'datasource', substitute=True),
-        GRAFANA_DASHBOARD_PATH : get_config('stack', 'grafana', 'dashboard', substitute=True),
+        GRAFANA_DATASOURCE_PATH: get_config('stack', 'grafana', 'datasource', substitute=True),
+        GRAFANA_DASHBOARD_PATH: get_config('stack', 'grafana', 'dashboard', substitute=True),
     }
 
 
@@ -251,8 +250,8 @@ def write_stack(
     return sync_files(['stack'])
    
 def edit_stack(
-        action : Optional[List[str]] = None,
-        debug : bool = False,
+        action: Optional[List[str]] = None,
+        debug: bool = False,
         **kw
     ):
     """Open docker-compose.yaml or .env for editing."""

meerschaum/config/stack/grafana/__init__.py

@@ -17,7 +17,7 @@ default_datasource = {
             'type': 'postgres',
             'jsonData': {
                 'sslmode': 'disable',
-                'postgresVersion': 1400,
+                'postgresVersion': 1500,
                 'timescaledb': True,
             },
             'user': db_user,

meerschaum/config/static/__init__.py

@@ -60,6 +60,7 @@ STATIC_CONFIG: Dict[str, Any] = {
         'gid': 'MRSM_GID',
         'noask': 'MRSM_NOASK',
         'id': 'MRSM_SERVER_ID',
+        'daemon_id': 'MRSM_DAEMON_ID',
         'uri_regex': r'MRSM_([a-zA-Z0-9]*)_(\d*[a-zA-Z][a-zA-Z0-9-_+]*$)',
         'prefix': 'MRSM_',
     },
@@ -103,11 +104,13 @@ STATIC_CONFIG: Dict[str, Any] = {
     },
     'users': {
         'password_hash': {
+            'algorithm_name': 'sha256',
+            'salt_bytes': 16,
             'schemes': [
                 'pbkdf2_sha256',
             ],
             'default': 'pbkdf2_sha256',
-            'pbkdf2_sha256__default_rounds': 30000,
+            'pbkdf2_sha256__default_rounds': 3_000_000,
         },
         'min_username_length': 1,
         'max_username_length': 26,

meerschaum/connectors/__init__.py

@@ -317,6 +317,8 @@ def load_plugin_connectors():
     from meerschaum.plugins import get_plugins, import_plugins
     to_import = []
     for plugin in get_plugins():
+        if plugin is None:
+            continue
         with open(plugin.__file__, encoding='utf-8') as f:
             text = f.read()
         if 'make_connector' in text:

meerschaum/connectors/api/_plugins.py

@@ -49,6 +49,7 @@ def register_plugin(
 def install_plugin(
         self,
         name: str,
+        skip_deps: bool = False,
         force: bool = False,
         debug: bool = False
     ) -> SuccessTuple:
@@ -78,7 +79,7 @@ def install_plugin(
             success, msg = False, fail_msg
         return success, msg
     plugin = Plugin(name, archive_path=archive_path, repo_connector=self)
-    return plugin.install(force=force, debug=debug)
+    return plugin.install(skip_deps=skip_deps, force=force, debug=debug)
 
 def get_plugins(
         self,

meerschaum/connectors/sql/SQLConnector.py

@@ -128,8 +128,10 @@ class SQLConnector(Connector):
         """
         if 'uri' in kw:
             uri = kw['uri']
-            if uri.startswith('postgres://'):
-                uri = uri.replace('postgres://', 'postgresql://', 1)
+            if uri.startswith('postgres') and not uri.startswith('postgresql'):
+                uri = uri.replace('postgres', 'postgresql', 1)
+            if uri.startswith('postgresql') and not uri.startswith('postgresql+'):
+                uri = uri.replace('postgresql://', 'postgresql+psycopg', 1)
             if uri.startswith('timescaledb://'):
                 uri = uri.replace('timescaledb://', 'postgresql://', 1)
                 flavor = 'timescaledb'

meerschaum/connectors/sql/_create_engine.py

@@ -28,7 +28,7 @@ default_create_engine_args = {
 }
 flavor_configs = {
     'timescaledb'      : {
-        'engine'       : 'postgresql',
+        'engine'       : 'postgresql+psycopg',
         'create_engine' : default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql' : {},
@@ -38,7 +38,7 @@ flavor_configs = {
         },
     },
     'postgresql'         : {
-        'engine'       : 'postgresql',
+        'engine'       : 'postgresql+psycopg',
         'create_engine' : default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql' : {},
@@ -48,7 +48,7 @@ flavor_configs = {
         },
     },
     'citus'            : {
-        'engine'       : 'postgresql',
+        'engine'       : 'postgresql+psycopg',
         'create_engine' : default_create_engine_args,
         'omit_create_engine': {'method',},
         'to_sql' : {},
@@ -154,10 +154,10 @@ install_flavor_drivers = {
     'duckdb': ['duckdb', 'duckdb_engine'],
     'mysql': ['pymysql'],
     'mariadb': ['pymysql'],
-    'timescaledb': ['psycopg2'],
-    'postgresql': ['psycopg2'],
-    'citus': ['psycopg2'],
-    'cockroachdb': ['psycopg2', 'sqlalchemy_cockroachdb', 'sqlalchemy_cockroachdb.psycopg2'],
+    'timescaledb': ['psycopg'],
+    'postgresql': ['psycopg'],
+    'citus': ['psycopg'],
+    'cockroachdb': ['psycopg', 'sqlalchemy_cockroachdb', 'sqlalchemy_cockroachdb.psycopg'],
     'mssql': ['pyodbc'],
     'oracle': ['cx_Oracle'],
 }
@@ -165,7 +165,7 @@ require_patching_flavors = {'cockroachdb': [('sqlalchemy-cockroachdb', 'sqlalche
 
 flavor_dialects = {
     'cockroachdb': (
-        'cockroachdb', 'sqlalchemy_cockroachdb.psycopg2', 'CockroachDBDialect_psycopg2'
+        'cockroachdb', 'sqlalchemy_cockroachdb.psycopg', 'CockroachDBDialect_psycopg'
     ),
     'duckdb': ('duckdb', 'duckdb_engine', 'Dialect'),
 }
@@ -242,7 +242,7 @@ def create_engine(
 
         ### Sometimes the timescaledb:// flavor can slip in.
         if _uri and self.flavor in ('timescaledb',) and self.flavor in _uri:
-            engine_str = engine_str.replace(f'{self.flavor}://', 'postgresql://')
+            engine_str = engine_str.replace(f'{self.flavor}', 'postgresql', 1)
 
     if debug:
         dprint(

meerschaum/connectors/sql/_instance.py

@@ -155,7 +155,9 @@ def _drop_old_temporary_tables(
     temp_tables_table = get_tables(mrsm_instance=self, create=False, debug=debug)['temp_tables']
     last_check = getattr(self, '_stale_temporary_tables_check_timestamp', 0)
     now_ts = time.perf_counter()
-    if refresh or not last_check or (now_ts - last_check) > 60:
+    if not last_check:
+        self._stale_temporary_tables_check_timestamp = 0
+    if refresh or (now_ts - last_check) < 60:
         self._stale_temporary_tables_check_timestamp = now_ts
         return self._drop_temporary_tables(debug=debug)
 

meerschaum/connectors/sql/_pipes.py

@@ -752,7 +752,7 @@ def get_pipe_data(
         debug = debug,
         **kw
     )
-
+    
     if is_dask:
         index_col = pipe.columns.get('datetime', None)
         kw['index_col'] = index_col
@@ -763,6 +763,7 @@ def get_pipe_data(
         if typ == 'numeric' and col in dtypes
     ]
     kw['coerce_float'] = kw.get('coerce_float', (len(numeric_columns) == 0))
+    
     df = self.read(
         query,
         dtype = dtypes,
@@ -1478,43 +1479,11 @@ def sync_pipe_inplace(
     from meerschaum.utils.misc import generate_password
     from meerschaum.utils.debug import dprint
 
-    sqlalchemy, sqlalchemy_orm = mrsm.attempt_import('sqlalchemy', 'sqlalchemy.orm')
-    metadef = self.get_pipe_metadef(
-        pipe,
-        params = params,
-        begin = begin,
-        end = end,
-        check_existing = check_existing,
-        debug = debug,
-    )
-    pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
-    upsert = pipe.parameters.get('upsert', False) and f'{self.flavor}-upsert' in update_queries
-    internal_schema = self.internal_schema
-    database = getattr(self, 'database', self.parse_uri(self.URI).get('database', None))
-
-    if not pipe.exists(debug=debug):
-        create_pipe_query = get_create_table_query(
-            metadef,
-            pipe.target,
-            self.flavor,
-            schema = self.get_pipe_schema(pipe),
-        )
-        result = self.exec(create_pipe_query, debug=debug)
-        if result is None:
-            return False, f"Could not insert new data into {pipe} from its SQL query definition."
-        if not self.create_indices(pipe, debug=debug):
-            warn(f"Failed to create indices for {pipe}. Continuing...")
-
-        rowcount = pipe.get_rowcount(debug=debug)
-        return True, f"Inserted {rowcount}, updated 0 rows."
-
-    session = sqlalchemy_orm.Session(self.engine)
-    connectable = session if self.flavor != 'duckdb' else self
-
     transact_id = generate_password(3)
     def get_temp_table_name(label: str) -> str:
         return '-' + transact_id + '_' + label + '_' + pipe.target
 
+    internal_schema = self.internal_schema
     temp_table_roots = ['backtrack', 'new', 'delta', 'joined', 'unseen', 'update']
     temp_tables = {
         table_root: get_temp_table_name(table_root)
@@ -1528,6 +1497,17 @@ def sync_pipe_inplace(
         )
         for table_root, table_name_raw in temp_tables.items()
     }
+    metadef = self.get_pipe_metadef(
+        pipe,
+        params = params,
+        begin = begin,
+        end = end,
+        check_existing = check_existing,
+        debug = debug,
+    )
+    pipe_name = sql_item_name(pipe.target, self.flavor, self.get_pipe_schema(pipe))
+    upsert = pipe.parameters.get('upsert', False) and f'{self.flavor}-upsert' in update_queries
+    database = getattr(self, 'database', self.parse_uri(self.URI).get('database', None))
 
     def clean_up_temp_tables(ready_to_drop: bool = False):
         log_success, log_msg = self._log_temporary_tables_creation(
@@ -1541,6 +1521,36 @@ def sync_pipe_inplace(
         )
         if not log_success:
             warn(log_msg)
+        drop_stale_success, drop_stale_msg = self._drop_old_temporary_tables(
+            refresh = False,
+            debug = debug,
+        )
+        if not drop_stale_success:
+            warn(drop_stale_msg)
+        return drop_stale_success, drop_stale_msg
+
+    sqlalchemy, sqlalchemy_orm = mrsm.attempt_import('sqlalchemy', 'sqlalchemy.orm')
+    if not pipe.exists(debug=debug):
+        create_pipe_query = get_create_table_query(
+            metadef,
+            pipe.target,
+            self.flavor,
+            schema = self.get_pipe_schema(pipe),
+        )
+        result = self.exec(create_pipe_query, debug=debug)
+        if result is None:
+            _ = clean_up_temp_tables()
+            return False, f"Could not insert new data into {pipe} from its SQL query definition."
+
+        if not self.create_indices(pipe, debug=debug):
+            warn(f"Failed to create indices for {pipe}. Continuing...")
+
+        rowcount = pipe.get_rowcount(debug=debug)
+        _ = clean_up_temp_tables()
+        return True, f"Inserted {rowcount}, updated 0 rows."
+
+    session = sqlalchemy_orm.Session(self.engine)
+    connectable = session if self.flavor != 'duckdb' else self
 
     create_new_query = get_create_table_query(
         metadef,
@@ -1908,10 +1918,6 @@ def sync_pipe_inplace(
     )
     _ = clean_up_temp_tables(ready_to_drop=True)
 
-    drop_stale_success, drop_stale_msg = self._drop_old_temporary_tables(refresh=False, debug=debug)
-    if not drop_stale_success:
-        warn(drop_stale_msg)
-
     return True, msg
 
 
@@ -2372,6 +2378,16 @@ def get_pipe_columns_types(
     """
     if not pipe.exists(debug=debug):
         return {}
+
+    if self.flavor == 'duckdb':
+        from meerschaum.utils.sql import get_table_cols_types
+        return get_table_cols_types(
+            pipe.target,
+            self,
+            flavor = self.flavor,
+            schema = self.schema,
+        )
+
     table_columns = {}
     try:
         pipe_table = self.get_pipe_table(pipe, debug=debug)

meerschaum/connectors/sql/_plugins.py

@@ -108,9 +108,7 @@ def get_plugin_version(
     plugins_tbl = get_tables(mrsm_instance=self, debug=debug)['plugins']
     from meerschaum.utils.packages import attempt_import
     sqlalchemy = attempt_import('sqlalchemy')
-
     query = sqlalchemy.select(plugins_tbl.c.version).where(plugins_tbl.c.plugin_name == plugin.name)
-
     return self.value(query, debug=debug)
 
 def get_plugin_user_id(

meerschaum/connectors/sql/_sql.py

@@ -943,17 +943,15 @@ def psql_insert_copy(
         ) for row in data_iter
     )
 
+    table_name = sql_item_name(table.name, 'postgresql', table.schema)
+    columns = ', '.join(f'"{k}"' for k in keys)
+    sql = f"COPY {table_name} ({columns}) FROM STDIN WITH CSV NULL '\\N'"
+
     dbapi_conn = conn.connection
     with dbapi_conn.cursor() as cur:
-        s_buf = StringIO()
-        writer = csv.writer(s_buf)
-        writer.writerows(data_iter)
-        s_buf.seek(0)
-
-        columns = ', '.join(f'"{k}"' for k in keys)
-        table_name = sql_item_name(table.name, 'postgresql', table.schema)
-        sql = f"COPY {table_name} ({columns}) FROM STDIN WITH CSV NULL '\\N'"
-        cur.copy_expert(sql=sql, file=s_buf)
+        with cur.copy(sql) as copy:
+            writer = csv.writer(copy)
+            writer.writerows(data_iter)
 
 
 def format_sql_query_for_dask(query: str) -> 'sqlalchemy.sql.selectable.Select':