meerschaum 2.8.3__py3-none-any.whl → 2.9.0__py3-none-any.whl

meerschaum/utils/schedule.py

@@ -95,7 +95,7 @@ def schedule_function(
     A `SuccessTuple` upon exit.
     """
     import asyncio
-    from meerschaum.utils.misc import filter_keywords, round_time
+    from meerschaum.utils.misc import filter_keywords
 
     global _scheduler
     kw['debug'] = debug
@@ -103,7 +103,7 @@ def schedule_function(
 
     _ = mrsm.attempt_import('attrs', lazy=False)
     apscheduler = mrsm.attempt_import('apscheduler', lazy=False)
-    now = round_time(datetime.now(timezone.utc), timedelta(minutes=1))
+    now = datetime.now(timezone.utc)
     trigger = parse_schedule(schedule, now=now)
     _scheduler = apscheduler.AsyncScheduler(identity='mrsm-scheduler')
     try:
@@ -296,7 +296,7 @@ def parse_start_time(schedule: str, now: Optional[datetime] = None) -> datetime:
     dateutil_parser = mrsm.attempt_import('dateutil.parser')
     starting_parts = schedule.split(STARTING_KEYWORD)
     starting_str = ('now' if len(starting_parts) == 1 else starting_parts[-1]).strip()
-    now = now or round_time(datetime.now(timezone.utc), timedelta(minutes=1))
+    now = now or datetime.now(timezone.utc)
     try:
         if starting_str == 'now':
             starting_ts = now

meerschaum/utils/sql.py

@@ -41,13 +41,13 @@ version_queries = {
 }
 SKIP_IF_EXISTS_FLAVORS = {'mssql', 'oracle'}
 DROP_IF_EXISTS_FLAVORS = {
-    'timescaledb', 'postgresql', 'citus', 'mssql', 'mysql', 'mariadb', 'sqlite',
+    'timescaledb', 'postgresql', 'postgis', 'citus', 'mssql', 'mysql', 'mariadb', 'sqlite',
 }
 DROP_INDEX_IF_EXISTS_FLAVORS = {
-    'mssql', 'timescaledb', 'postgresql', 'sqlite', 'citus',
+    'mssql', 'timescaledb', 'postgresql', 'postgis', 'sqlite', 'citus',
 }
 SKIP_AUTO_INCREMENT_FLAVORS = {'citus', 'duckdb'}
-COALESCE_UNIQUE_INDEX_FLAVORS = {'timescaledb', 'postgresql', 'citus'}
+COALESCE_UNIQUE_INDEX_FLAVORS = {'timescaledb', 'postgresql', 'postgis', 'citus'}
 UPDATE_QUERIES = {
     'default': """
         UPDATE {target_table_name} AS f
@@ -73,6 +73,12 @@ UPDATE_QUERIES = {
         FROM {patch_table_name}
         ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
     """,
+    'postgis-upsert': """
+        INSERT INTO {target_table_name} ({patch_cols_str})
+        SELECT {patch_cols_str}
+        FROM {patch_table_name}
+        ON CONFLICT ({join_cols_str}) DO {update_or_nothing} {sets_subquery_none_excluded}
+    """,
     'citus-upsert': """
         INSERT INTO {target_table_name} ({patch_cols_str})
         SELECT {patch_cols_str}
@@ -482,6 +488,7 @@ table_wrappers = {
     'citus'      : ('"', '"'),
     'duckdb'     : ('"', '"'),
     'postgresql' : ('"', '"'),
+    'postgis'    : ('"', '"'),
     'sqlite'     : ('"', '"'),
     'mysql'      : ('`', '`'),
     'mariadb'    : ('`', '`'),
@@ -494,6 +501,7 @@ max_name_lens = {
     'mssql'      : 128,
     'oracle'     : 30,
     'postgresql' : 64,
+    'postgis'    : 64,
     'timescaledb': 64,
     'citus'      : 64,
     'cockroachdb': 64,
@@ -501,10 +509,11 @@ max_name_lens = {
     'mysql'      : 64,
     'mariadb'    : 64,
 }
-json_flavors = {'postgresql', 'timescaledb', 'citus', 'cockroachdb'}
+json_flavors = {'postgresql', 'postgis', 'timescaledb', 'citus', 'cockroachdb'}
 NO_SCHEMA_FLAVORS = {'oracle', 'sqlite', 'mysql', 'mariadb', 'duckdb'}
 DEFAULT_SCHEMA_FLAVORS = {
     'postgresql': 'public',
+    'postgis': 'public',
     'timescaledb': 'public',
     'citus': 'public',
     'cockroachdb': 'public',
@@ -519,7 +528,7 @@ NO_CTE_FLAVORS = {'mysql', 'mariadb'}
 NO_SELECT_INTO_FLAVORS = {'sqlite', 'oracle', 'mysql', 'mariadb', 'duckdb'}
 
 
-def clean(substring: str) -> str:
+def clean(substring: str) -> None:
     """
     Ensure a substring is clean enough to be inserted into a SQL query.
     Raises an exception when banned words are used.
@@ -549,6 +558,7 @@ def dateadd_str(
         Currently supported flavors:
 
         - `'postgresql'`
+        - `'postgis'`
         - `'timescaledb'`
         - `'citus'`
         - `'cockroachdb'`
@@ -653,7 +663,7 @@ def dateadd_str(
     )
 
     da = ""
-    if flavor in ('postgresql', 'timescaledb', 'cockroachdb', 'citus'):
+    if flavor in ('postgresql', 'postgis', 'timescaledb', 'cockroachdb', 'citus'):
         begin = (
             f"CAST({begin} AS {db_type})" if begin != 'now'
             else f"CAST(NOW() AT TIME ZONE 'utc' AS {db_type})"
@@ -922,6 +932,7 @@ def build_where(
     params: Dict[str, Any],
     connector: Optional[mrsm.connectors.sql.SQLConnector] = None,
     with_where: bool = True,
+    flavor: str = 'postgresql',
 ) -> str:
     """
     Build the `WHERE` clause based on the input criteria.
@@ -941,6 +952,9 @@ def build_where(
     with_where: bool, default True:
         If `True`, include the leading `'WHERE'` string.
 
+    flavor: str, default 'postgresql'
+        If `connector` is `None`, fall back to this flavor.
+
     Returns
     -------
     A `str` of the `WHERE` clause from the input `params` dictionary for the connector's flavor.
@@ -969,13 +983,11 @@ def build_where(
             warn(f"Aborting build_where() due to possible SQL injection.")
             return ''
 
-    if connector is None:
-        from meerschaum import get_connector
-        connector = get_connector('sql')
+    query_flavor = getattr(connector, 'flavor', flavor) if connector is not None else flavor
     where = ""
     leading_and = "\n    AND "
     for key, value in params.items():
-        _key = sql_item_name(key, connector.flavor, None)
+        _key = sql_item_name(key, query_flavor, None)
         ### search across a list (i.e. IN syntax)
         if isinstance(value, Iterable) and not isinstance(value, (dict, str)):
             includes = [
@@ -1286,7 +1298,24 @@ def get_table_cols_types(
         if cols_types_docs and not cols_types_docs_filtered:
             cols_types_docs_filtered = cols_types_docs
 
-        return {
+        ### NOTE: Check for PostGIS GEOMETRY columns.
+        geometry_cols_types = {}
+        user_defined_cols = [
+            doc
+            for doc in cols_types_docs_filtered
+            if str(doc.get('type', None)).upper() == 'USER-DEFINED'
+        ]
+        if user_defined_cols:
+            geometry_cols_types.update(
+                get_postgis_geo_columns_types(
+                    connectable,
+                    table,
+                    schema=schema,
+                    debug=debug,
+                )
+            )
+
+        cols_types = {
             (
                 doc['column']
                 if flavor != 'oracle' else (
@@ -1307,6 +1336,8 @@ def get_table_cols_types(
             )
             for doc in cols_types_docs_filtered
         }
+        cols_types.update(geometry_cols_types)
+        return cols_types
     except Exception as e:
         warn(f"Failed to fetch columns for table '{table}':\n{e}")
         return {}
@@ -1548,7 +1579,7 @@ def get_update_queries(
     from meerschaum.utils.debug import dprint
     from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.dtypes.sql import DB_FLAVORS_CAST_DTYPES, get_pd_type_from_db_type
-    flavor = flavor or (connectable.flavor if isinstance(connectable, SQLConnector) else None)
+    flavor = flavor or getattr(connectable, 'flavor', None)
     if not flavor:
         raise ValueError("Provide a flavor if using a SQLAlchemy session.")
     if (
@@ -1809,6 +1840,8 @@ def get_null_replacement(typ: str, flavor: str) -> str:
     """
     from meerschaum.utils.dtypes import are_dtypes_equal
     from meerschaum.utils.dtypes.sql import DB_FLAVORS_CAST_DTYPES
+    if 'geometry' in typ.lower():
+        return '010100000000008058346FCDC100008058346FCDC1'
     if 'int' in typ.lower() or typ.lower() in ('numeric', 'number'):
         return '-987654321'
     if 'bool' in typ.lower() or typ.lower() == 'bit':
@@ -2493,3 +2526,98 @@ def get_reset_autoincrement_queries(
         )
         for query in reset_queries
     ]
+
+
+def get_postgis_geo_columns_types(
+    connectable: Union[
+        'mrsm.connectors.sql.SQLConnector',
+        'sqlalchemy.orm.session.Session',
+        'sqlalchemy.engine.base.Engine'
+    ],
+    table: str,
+    schema: Optional[str] = 'public',
+    debug: bool = False,
+) -> Dict[str, str]:
+    """
+    Return the 
+    """
+    from meerschaum.utils.dtypes import get_geometry_type_srid
+    default_type, default_srid = get_geometry_type_srid()
+    default_type = default_type.upper()
+
+    clean(table)
+    clean(str(schema))
+    schema = schema or 'public'
+    truncated_schema_name = truncate_item_name(schema, flavor='postgis')
+    truncated_table_name = truncate_item_name(table, flavor='postgis')
+    query = (
+        "SELECT \"f_geometry_column\" AS \"column\", 'GEOMETRY' AS \"func\", \"type\", \"srid\"\n"
+        "FROM \"geometry_columns\"\n"
+        f"WHERE \"f_table_schema\" = '{truncated_schema_name}'\n"
+        f"    AND \"f_table_name\" = '{truncated_table_name}'\n"
+        "UNION ALL\n"
+        "SELECT \"f_geography_column\" AS \"column\", 'GEOGRAPHY' AS \"func\", \"type\", \"srid\"\n"
+        "FROM \"geography_columns\"\n"
+        f"WHERE \"f_table_schema\" = '{truncated_schema_name}'\n"
+        f"    AND \"f_table_name\" = '{truncated_table_name}'\n"
+    )
+    debug_kwargs = {'debug': debug} if isinstance(connectable, mrsm.connectors.SQLConnector) else {}
+    result_rows = [
+        row
+        for row in connectable.execute(query, **debug_kwargs).fetchall()
+    ]
+    cols_type_tuples = {
+        row[0]: (row[1], row[2], row[3])
+        for row in result_rows
+    }
+
+    geometry_cols_types = {
+        col: (
+            f"{func}({typ.upper()}, {srid})"
+            if srid
+            else (
+                func
+                + (
+                    f'({typ.upper()})'
+                    if typ.upper() not in ('GEOMETRY', 'GEOGRAPHY')
+                    else ''
+                )
+            )
+        )
+        for col, (func, typ, srid) in cols_type_tuples.items()
+    }
+    return geometry_cols_types
+
+
+def get_create_schema_if_not_exists_queries(
+    schema: str,
+    flavor: str,
+) -> List[str]:
+    """
+    Return the queries to create a schema if it does not yet exist.
+    For databases which do not support schemas, an empty list will be returned.
+    """
+    if not schema:
+        return []
+    
+    if flavor in NO_SCHEMA_FLAVORS:
+        return []
+
+    clean(schema)
+
+    if flavor == 'mssql':
+        return [
+            (
+                f"IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{schema}')\n"
+                "BEGIN\n"
+                f"    EXEC('CREATE SCHEMA {sql_item_name(schema, flavor)}');\n"
+                "END;"
+            )
+        ]
+
+    if flavor == 'oracle':
+        return []
+
+    return [
+        f"CREATE SCHEMA IF NOT EXISTS {sql_item_name(schema, flavor)};"
+    ]

meerschaum/utils/venv/__init__.py

@@ -86,7 +86,10 @@ def activate_venv(
         target = target_path.as_posix()
 
         if venv in active_venvs_order:
-            sys.path.remove(target)
+            try:
+                sys.path.remove(target)
+            except Exception:
+                pass
             try:
                 active_venvs_order.remove(venv)
             except Exception:
@@ -410,6 +413,8 @@ def init_venv(
             pass
 
     def wait_for_lock():
+        if platform.system() == 'Windows':
+            return
         max_lock_seconds = 30.0
         sleep_message_seconds = 5.0
         step_sleep_seconds = 0.1
@@ -595,7 +600,7 @@ def venv_exec(
     as_proc: bool = False,
     capture_output: bool = True,
     debug: bool = False,
-) -> Union[bool, Tuple[int, bytes, bytes]]:
+) -> Union[bool, Tuple[int, bytes, bytes], 'subprocess.Popen']:
     """
     Execute Python code in a subprocess via a virtual environment's interpeter.
     Return `True` if the code successfully executes, `False` on failure.
@@ -630,6 +635,8 @@ def venv_exec(
     import subprocess
     import platform
     from meerschaum.utils.debug import dprint
+    from meerschaum.utils.process import _child_processes
+
     executable = venv_executable(venv=venv)
     cmd_list = [executable, '-c', code]
     if env is None:
@@ -656,6 +663,7 @@ def venv_exec(
         **group_kwargs
     )
     if as_proc:
+        _child_processes.append(process)
         return process
     stdout, stderr = process.communicate()
     exit_code = process.returncode

{meerschaum-2.8.3.dist-info → meerschaum-2.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: meerschaum
-Version: 2.8.3
+Version: 2.9.0
 Summary: Sync Time-Series Pipes with Meerschaum
 Home-page: https://meerschaum.io
 Author: Bennett Meares
@@ -87,6 +87,10 @@ Requires-Dist: mycli>=1.23.2; extra == "cli"
 Requires-Dist: litecli>=1.5.0; extra == "cli"
 Requires-Dist: mssql-cli>=1.0.0; extra == "cli"
 Requires-Dist: gadwall>=0.2.0; extra == "cli"
+Provides-Extra: gis
+Requires-Dist: pyproj>=3.7.1; extra == "gis"
+Requires-Dist: geopandas>=1.0.1; extra == "gis"
+Requires-Dist: shapely>=2.0.7; extra == "gis"
 Provides-Extra: stack
 Requires-Dist: docker-compose>=1.29.2; extra == "stack"
 Provides-Extra: build
@@ -114,7 +118,6 @@ Requires-Dist: mkdocs-linkcheck>=1.0.6; extra == "docs"
 Requires-Dist: mkdocs-redirects>=1.0.4; extra == "docs"
 Requires-Dist: jinja2==3.0.3; extra == "docs"
 Provides-Extra: gui
-Requires-Dist: toga>=0.3.0-dev29; extra == "gui"
 Requires-Dist: pywebview>=3.6.3; extra == "gui"
 Requires-Dist: pycparser>=2.21.0; extra == "gui"
 Provides-Extra: extras
@@ -132,6 +135,7 @@ Requires-Dist: partd>=1.4.2; extra == "sql"
 Requires-Dist: pytz; extra == "sql"
 Requires-Dist: joblib>=0.17.0; extra == "sql"
 Requires-Dist: SQLAlchemy>=2.0.5; extra == "sql"
+Requires-Dist: GeoAlchemy2>=0.17.1; extra == "sql"
 Requires-Dist: databases>=0.4.0; extra == "sql"
 Requires-Dist: aiosqlite>=0.16.0; extra == "sql"
 Requires-Dist: asyncpg>=0.21.0; extra == "sql"
@@ -164,6 +168,9 @@ Requires-Dist: fasteners>=0.19.0; extra == "sql"
 Requires-Dist: virtualenv>=20.1.0; extra == "sql"
 Requires-Dist: attrs>=24.2.0; extra == "sql"
 Requires-Dist: uv>=0.2.11; extra == "sql"
+Requires-Dist: pyproj>=3.7.1; extra == "sql"
+Requires-Dist: geopandas>=1.0.1; extra == "sql"
+Requires-Dist: shapely>=2.0.7; extra == "sql"
 Provides-Extra: dash
 Requires-Dist: Flask-Compress>=1.10.1; extra == "dash"
 Requires-Dist: dash>=2.6.2; extra == "dash"
@@ -192,6 +199,7 @@ Requires-Dist: partd>=1.4.2; extra == "api"
 Requires-Dist: pytz; extra == "api"
 Requires-Dist: joblib>=0.17.0; extra == "api"
 Requires-Dist: SQLAlchemy>=2.0.5; extra == "api"
+Requires-Dist: GeoAlchemy2>=0.17.1; extra == "api"
 Requires-Dist: databases>=0.4.0; extra == "api"
 Requires-Dist: aiosqlite>=0.16.0; extra == "api"
 Requires-Dist: asyncpg>=0.21.0; extra == "api"
@@ -224,6 +232,9 @@ Requires-Dist: fasteners>=0.19.0; extra == "api"
 Requires-Dist: virtualenv>=20.1.0; extra == "api"
 Requires-Dist: attrs>=24.2.0; extra == "api"
 Requires-Dist: uv>=0.2.11; extra == "api"
+Requires-Dist: pyproj>=3.7.1; extra == "api"
+Requires-Dist: geopandas>=1.0.1; extra == "api"
+Requires-Dist: shapely>=2.0.7; extra == "api"
 Requires-Dist: pprintpp>=0.4.0; extra == "api"
 Requires-Dist: asciitree>=0.3.3; extra == "api"
 Requires-Dist: typing-extensions>=4.7.1; extra == "api"
@@ -286,7 +297,9 @@ Requires-Dist: aiomysql>=0.0.21; extra == "full"
 Requires-Dist: sqlalchemy-cockroachdb>=2.0.0; extra == "full"
 Requires-Dist: duckdb>=1.0.0; extra == "full"
 Requires-Dist: duckdb-engine>=0.13.0; extra == "full"
-Requires-Dist: toga>=0.3.0-dev29; extra == "full"
+Requires-Dist: pyproj>=3.7.1; extra == "full"
+Requires-Dist: geopandas>=1.0.1; extra == "full"
+Requires-Dist: shapely>=2.0.7; extra == "full"
 Requires-Dist: pywebview>=3.6.3; extra == "full"
 Requires-Dist: pycparser>=2.21.0; extra == "full"
 Requires-Dist: numpy>=1.18.5; extra == "full"
@@ -297,6 +310,7 @@ Requires-Dist: partd>=1.4.2; extra == "full"
 Requires-Dist: pytz; extra == "full"
 Requires-Dist: joblib>=0.17.0; extra == "full"
 Requires-Dist: SQLAlchemy>=2.0.5; extra == "full"
+Requires-Dist: GeoAlchemy2>=0.17.1; extra == "full"
 Requires-Dist: databases>=0.4.0; extra == "full"
 Requires-Dist: aiosqlite>=0.16.0; extra == "full"
 Requires-Dist: asyncpg>=0.21.0; extra == "full"