quollio-core 0.4.4__py3-none-any.whl → 0.4.10__py3-none-any.whl

quollio_core/dbt_projects/databricks/models/sources.yml

@@ -0,0 +1,84 @@
+version: 2
+
+sources: 
+  - name: access
+    database: system
+    schema: access
+    tables: 
+      - name: table_lineage
+        description: Describes table level lineage
+        columns:
+          - name: source_table_full_name
+            description: ''
+            type: string
+          - name: target_table_full_name
+            description: ''
+            type: string
+          - name: target_type
+            description: ''
+            type: string
+          - name: event_time
+            description: ''
+            type: timestamp
+
+      - name: column_lineage
+        description: Describes column level lineage
+        columns:
+          - name: source_table_full_name
+            description: ''
+            type: string
+          - name: target_table_full_name
+            description: ''
+            type: string
+          - name: event_time
+            description: ''
+            type: timestamp
+          - name: source_column_name
+            description: ''
+            type: string
+          - name: target_column_name
+            description: ''
+            type: string
+  
+  - name: inf_sch
+    database: system
+    schema: information_schema
+    tables: 
+      - name: tables
+        description: Lists existing tables (i.e., not deleted).
+        columns:
+          - name: table_catalog
+            description: ''
+            type: string
+          - name: table_schema
+            description: ''
+            type: string
+          - name: table_name
+            description: ''
+            type: string
+
+      - name: views
+        description: Lists existing views (i.e., not deleted). Views are treated as tables.
+        columns:
+          - name: table_catalog
+            description: ''
+            type: string
+          - name: table_schema
+            description: ''
+            type: string
+          - name: table_name
+            description: ''
+            type: string
+
+      - name: columns
+        description: ''
+        columns:
+          - name: table_catalog
+            description: ''
+            type: string
+          - name: table_schema
+            description: ''
+            type: string
+          - name: table_name
+            description: ''
+            type: string

quollio_core/dbt_projects/databricks/package-lock.yml

@@ -0,0 +1,14 @@
+packages:
+- package: dbt-labs/dbt_utils
+  version: 1.1.1
+- package: dbt-labs/spark_utils
+  version: 0.3.0
+- package: dbt-labs/codegen
+  version: 0.12.1
+- package: dbt-labs/dbt_external_tables
+  version: 0.8.7
+- package: dbt-labs/dbt_project_evaluator
+  version: 0.8.1
+- package: brooklyn-data/dbt_artifacts
+  version: 2.6.2
+sha1_hash: cbb324267dbf6c6fb7de11b162e4fbafd1e32a9c

quollio_core/dbt_projects/databricks/packages.yml

@@ -0,0 +1,13 @@
+packages:
+  - package: dbt-labs/dbt_utils
+    version: [">=0.0.0", "<2.0.0"]
+  - package: dbt-labs/spark_utils
+    version: [">=0.0.0", "<1.0.0"]
+  - package: dbt-labs/codegen
+    version: [">=0.0.0", "<1.0.0"]
+  - package: dbt-labs/dbt_external_tables
+    version: [">=0.0.0", "<1.0.0"]
+  - package: dbt-labs/dbt_project_evaluator
+    version: [">=0.0.0", "<1.0.0"]
+  - package: brooklyn-data/dbt_artifacts
+    version: [">=2.0.0", "<3.0.0"]

quollio_core/dbt_projects/databricks/profiles/profiles_template.yml

@@ -0,0 +1,14 @@
+quollio_intelligence_databricks:
+  target: project
+  outputs:
+    project:
+      type: databricks
+      host: {{ host }}
+      http_path: {{ http_path }}
+      catalog: {{ catalog }}
+      schema: {{ schema }}
+      auth_type: oauth
+      client_id: {{ client_id }}
+      client_secret: {{ client_secret }}
+      databricks_port: {{ databricks_port }}
+

quollio_core/dbt_projects/redshift/dbt_project.yml

@@ -18,4 +18,4 @@ clean-targets:
 models:
   +dbt-osmosis: "{model}.yml"
   +grants:
-    select: ["{{ var('query_user') }}"]
+    select: ["\"{{ var('query_user') }}\""]

quollio_core/dbt_projects/redshift/macros/materialization/divided_view.sql

@@ -1,59 +1,126 @@
 {%- materialization divided_view, default %}
 {%- set identifier = model['alias'] %}
 {%- set target_relations = [] %}
-{%- set chunk = config.get('chunk') %}
 {%- set grant_config = config.get('grants') %}
 
 {{ run_hooks(pre_hooks, inside_transaction=False) }}
 -- `BEGIN` happens here:
 {{ run_hooks(pre_hooks, inside_transaction=True) }}
 
--- fetch records
-{%- set query_quollio_stats_profiling_columns -%}
-SELECT * FROM {{  ref('quollio_stats_profiling_columns')  }} WHERE table_name not like 'quollio_%'
+-- fetch target_tables
+{%- set query_stats_target_tables -%}
+    SELECT
+      distinct
+      database_name
+      , schema_name
+      , table_name
+    FROM
+      {{ ref('quollio_stats_profiling_columns') }}
+    WHERE
+      table_name not like 'quollio_%%'
 {%- endset -%}
-{%- set results = run_query(query_quollio_stats_profiling_columns) -%}
+{%- set results = run_query(query_stats_target_tables) -%}
 {%- if execute -%}
-{%- set records = results.rows -%}
+{%- set stats_target_tables = results.rows -%}
 {%- else -%}
-{%- set records = [] -%}
+{%- set stats_target_tables = [] -%}
+{%- endif -%}
+
+-- skip creating views if the target profiling columns don't exist.
+{%- if stats_target_tables | length == 0 -%}
+  {% call statement("main") %}
+    {{ log("No records found. Just execute select stmt for skipping call statement.", info=True) }}
+    select null
+  {% endcall %}
+  {%- set full_refresh_mode = (should_full_refresh()) -%}
+  {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}
 {%- endif -%}
 
 -- build sql
-{%- for i in range(0, records|length, chunk) -%}
-  {%- set build_sql %}
-  {%- for record in records[i: i+chunk] -%}
-    {%- if not loop.first %}UNION{% endif %}
+{%- for stats_target_table in stats_target_tables -%}
+  -- get columns for statistics. 
+  -- LISTAGG function can't be used for sys table, then it's necessary to get column for each table. 
+  -- See https://docs.aws.amazon.com/redshift/latest/dg/c_join_PG.html.
+  {%- set stats_target_columns %}
+      SELECT
+        database_name
+        , schema_name
+        , table_name
+        , column_name
+        , is_bool
+        , is_calculable
+      FROM
+        {{ ref('quollio_stats_profiling_columns') }}
+      WHERE
+        database_name = '{{stats_target_table[0]}}'
+        AND schema_name = '{{stats_target_table[1]}}'
+        AND table_name = '{{stats_target_table[2]}}'
+  {%- endset -%}
+
+  {%- set results = run_query(stats_target_columns) -%}
+  {%- set stats_target_columns = results.rows -%}
+
+  {%- set sql_for_column_stats %}
+  {%- for stats_target_column in stats_target_columns -%}
+    {%- if not loop.first -%}UNION{% endif %}
     SELECT
-      DISTINCT
-      '{{record[0]}}'::varchar as db_name
-      , '{{record[1]}}'::varchar as schema_name
-      , '{{record[2]}}'::varchar as table_name
-      , '{{record[3]}}'::varchar as column_name
-      , {% if var("skip_heavy") == false and record[5] == true %}cast(max("{{record[3]}}") as varchar){% else %}null::varchar{% endif %} AS max_value
-      , {% if var("skip_heavy") == false and record[5] == true %}cast(min("{{record[3]}}") as varchar){% else %}null::varchar{% endif %} AS min_value
-      -- requires full table scan
-      , {% if var("skip_heavy") == false %}cast(SUM(NVL2("{{record[3]}}", 0, 1)) as integer){% else %}null::integer{% endif %} AS null_count
-      , APPROXIMATE COUNT(DISTINCT "{{record[3]}}") AS cardinality
-      -- requires full table scan
-      , {% if var("skip_heavy") == false and record[5] == true %}cast(avg("{{record[3]}}")as varchar){% else %}null::varchar{% endif %} AS avg_value
-      , {% if var("skip_heavy") == false and record[5] == true %}(SELECT cast(median("{{record[3]}}") as varchar) FROM {{record[2]}}){% else %}null::varchar{% endif %} AS median_value
-      -- requires full table scan
-      , {% if var("skip_heavy") == false and record[4] == false %}
-          (SELECT cast("{{record[3]}}" as varchar) FROM (
-            SELECT "{{record[3]}}", ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS row_num FROM {{record[2]}} GROUP BY "{{record[3]}}"
-          ) WHERE row_num = 1)
-        {% else %}null::varchar{% endif %} AS mode_value
-      , {% if record[5] == true %}cast(STDDEV_SAMP("{{record[3]}}") as integer){% else %}null::integer{% endif %} AS stddev_value
-    FROM {{ record[0] }}.{{ record[1] }}.{{ record[2] }}
+      main.db_name
+      , main.schema_name
+      , main.table_name
+      , main.column_name
+      , main.max_value
+      , main.min_value
+      , main.null_count
+      , main.cardinality
+      , main.avg_value
+      , main.median_value
+      , mode.mode_value
+      , main.stddev_value
+    FROM
+      (
+      SELECT
+        DISTINCT
+        '{{stats_target_column[0]}}'::varchar as db_name
+        , '{{stats_target_column[1]}}'::varchar as schema_name
+        , '{{stats_target_column[2]}}'::varchar as table_name
+        , '{{stats_target_column[3]}}'::varchar as column_name
+        , {% if var("aggregate_all") == True and stats_target_column[5] == True %}cast(max("{{stats_target_column[3]}}") as varchar){% else %}null::varchar{% endif %} AS max_value
+        , {% if var("aggregate_all") == True and stats_target_column[5] == True %}cast(min("{{stats_target_column[3]}}") as varchar){% else %}null::varchar{% endif %} AS min_value
+        -- requires full table scan
+        , {% if var("aggregate_all") == True %}cast(SUM(NVL2("{{stats_target_column[3]}}", 0, 1)) as integer){% else %}null::integer{% endif %} AS null_count
+        , APPROXIMATE COUNT(DISTINCT "{{stats_target_column[3]}}") AS cardinality
+        -- requires full table scan
+        , {% if var("aggregate_all") == True and stats_target_column[5] == True %}cast(avg("{{stats_target_column[3]}}")as varchar){% else %}null::varchar{% endif %} AS avg_value
+        , {% if var("aggregate_all") == True and stats_target_column[5] == True %}cast(median("{{stats_target_column[3]}}") as varchar){% else %}null::varchar{% endif %} AS median_value
+        -- requires full table scan
+        , {% if stats_target_column[5] == True %}cast(STDDEV_SAMP("{{stats_target_column[3]}}") as integer){% else %}null::integer{% endif %} AS stddev_value
+      FROM {{ stats_target_column[0] }}.{{ stats_target_column[1] }}.{{ stats_target_column[2] }}
+    ) main, (
+      {%- if var("aggregate_all") == True and stats_target_column[4] == false %}
+        SELECT
+          cast("{{stats_target_column[3]}}" as varchar) mode_value
+        FROM (
+           SELECT
+            DISTINCT
+            "{{stats_target_column[3]}}"
+            , ROW_NUMBER() OVER (ORDER BY COUNT(*) DESC) AS row_num
+          FROM {{ stats_target_column[0] }}.{{ stats_target_column[1] }}.{{ stats_target_column[2] }}
+          GROUP BY
+            "{{stats_target_column[3]}}"
+        )
+        WHERE
+          row_num = 1
+      {% else %}
+        SELECT null as mode_value {%- endif -%}
+    ) mode
   {% endfor -%}
   {%- endset %}
   -- create a view with a index as suffix
-  {%- set target_identifier = "%s_%d"|format(model['name'], loop.index) %}
+  {%- set target_identifier = "%s_%s_%s_%s"|format(model['name'], stats_target_table[0], stats_target_table[1], stats_target_table[2]) %}
   {%- set target_relation = api.Relation.create(identifier=target_identifier, schema=schema, database=database, type='view') %}
   -- {{ drop_relation_if_exists(target_relation) }}
   {% call statement("main") %}
-    {{ get_replace_view_sql(target_relation, build_sql) }}
+    {{ get_replace_view_sql(target_relation, sql_for_column_stats) }}
   {% endcall %}
   {%- set full_refresh_mode = (should_full_refresh()) -%}
   {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}

quollio_core/dbt_projects/redshift/models/quollio_stats_columns.sql

@@ -1,7 +1,6 @@
 {{
     config(
-        materialized='divided_view',
-        chunk=20
+        materialized='divided_view'
     )
 }}
 -- depends_on: {{ ref('quollio_stats_profiling_columns') }}

quollio_core/dbt_projects/redshift/package-lock.yml

@@ -1,4 +1,4 @@
 packages:
 - package: dbt-labs/dbt_utils
   version: 1.1.1
-sha1_hash: 23451c207c1d4dd71b5925f12a5cd66f2ebb2b3b
+sha1_hash: a158c48c59c2bb7d729d2a4e215aabe5bb4f3353

quollio_core/dbt_projects/snowflake/macros/materialization/divided_view.sql

@@ -1,51 +1,74 @@
 {%- materialization divided_view, default %}
 {%- set identifier = model['alias'] %}
 {%- set target_relations = [] %}
-{%- set chunk = config.get('chunk') %}
 {%- set grant_config = config.get('grants') %}
 
 {{ run_hooks(pre_hooks, inside_transaction=False) }}
 -- `BEGIN` happens here:
 {{ run_hooks(pre_hooks, inside_transaction=True) }}
 
--- fetch records
-{%- set query_quollio_stats_profiling_columns -%}
-SELECT * FROM {{  ref('quollio_stats_profiling_columns')  }} WHERE NOT startswith(table_name, 'QUOLLIO_')
+-- fetch target_tables
+{%- set query_stats_target_tables -%}
+    SELECT
+      TABLE_CATALOG
+      , TABLE_SCHEMA
+      , TABLE_NAME
+      , OBJECT_AGG(COLUMN_NAME, IS_CALCULABLE) AS COLUMNS_OBJ
+    FROM
+      {{ ref('quollio_stats_profiling_columns') }}
+    WHERE NOT startswith(table_name, 'QUOLLIO_')
+    GROUP BY
+      TABLE_CATALOG
+      , TABLE_SCHEMA
+      , TABLE_NAME
 {%- endset -%}
-{%- set results = run_query(query_quollio_stats_profiling_columns) -%}
+{%- set results = run_query(query_stats_target_tables) -%}
 {%- if execute -%}
-{%- set records = results.rows -%}
+{%- set stats_target_tables = results.rows -%}
 {%- else -%}
-{%- set records = [] -%}
+{%- set stats_target_tables = [] -%}
+{%- endif -%}
+
+-- skip creating views if the target profiling columns don't exist.
+{%- if stats_target_tables | length == 0 -%}
+  {% call statement("main") %}
+    {{ log("No records found. Just execute select stmt for skipping call statement.", info=True) }}
+    select null
+  {% endcall %}
+  {%- set full_refresh_mode = (should_full_refresh()) -%}
+  {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}
 {%- endif -%}
 
--- build sql
-{%- for i in range(0, records|length, chunk) -%}
-  {%- set build_sql %}
-  {%- for record in records[i: i+chunk] -%}
+-- create view for each table
+{%- for stats_target_table in stats_target_tables -%}
+  -- build sql for column value aggregation.
+  {%- set sql_for_column_stats %}
+  {% set columns_json = fromjson(stats_target_table[3]) %}
+  {%- for col_name, is_calclable in columns_json.items() -%}
     {%- if not loop.first %}UNION{% endif %}
     SELECT
       DISTINCT
-      '{{record[0]}}' as db_name
-      , '{{record[1]}}' as schema_name
-      , '{{record[2]}}' as table_name
-      , '{{record[3]}}' as column_name
-      , {% if record[5] == true %}CAST(max("{{record[3]}}") AS STRING){% else %}null{% endif %} AS max_value
-      , {% if record[5] == true %}CAST(min("{{record[3]}}") AS STRING){% else %}null{% endif %} AS min_value
-      , COUNT_IF("{{record[3]}}" IS NULL) AS null_count
-      , APPROX_COUNT_DISTINCT("{{record[3]}}") AS cardinality
-      , {% if record[5] == true %}avg("{{record[3]}}"){% else %}null{% endif %} AS avg_value
-      , {% if record[5] == true %}median("{{record[3]}}"){% else %}null{% endif %} AS median_value
-      , {% if record[5] == true %}approx_top_k("{{record[3]}}")[0][0]{% else %}null{% endif %} AS mode_value
-      , {% if record[5] == true %}stddev("{{record[3]}}"){% else %}null{% endif %} AS stddev_value
-    FROM {{ record[0] }}.{{ record[1] }}.{{ record[2] }} {{ var("sample_method") }}
+      '{{stats_target_table[0]}}' as db_name
+      , '{{stats_target_table[1]}}' as schema_name
+      , '{{stats_target_table[2]}}' as table_name
+      , '{{col_name}}' as column_name
+      , {% if is_calclable == True %}CAST(MAX("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS max_value
+      , {% if is_calclable == True %}CAST(MIN("{{col_name}}") AS STRING){% else %}NULL{% endif %} AS min_value
+      , COUNT_IF("{{col_name}}" IS NULL) AS null_count
+      , APPROX_COUNT_DISTINCT("{{col_name}}") AS cardinality
+      , {% if is_calclable == True %}AVG("{{col_name}}"){% else %}NULL{% endif %} AS avg_value
+      , {% if is_calclable == True %}MEDIAN("{{col_name}}"){% else %}NULL{% endif %} AS median_value
+      , {% if is_calclable == True %}APPROX_TOP_K("{{col_name}}")[0][0]{% else %}NULL{% endif %} AS mode_value
+      , {% if is_calclable == True %}STDDEV("{{col_name}}"){% else %}NULL{% endif %} AS stddev_value
+    FROM "{{stats_target_table[0]}}"."{{stats_target_table[1]}}"."{{stats_target_table[2]}}" {{ var("sample_method") }}
   {% endfor -%}
   {%- endset %}
+
   -- create a view with a index as suffix
-  {%- set target_identifier = "%s_%d"|format(model['name'], loop.index) %}
-  {%- set target_relation = api.Relation.create(identifier=target_identifier, schema=schema, database=database, type='view') %}
+  {%- set stats_view_identifier = "%s_%s_%s_%s"|format(model['name'], stats_target_table[0], stats_target_table[1], stats_target_table[2]) %}
+  {%- set target_relation = api.Relation.create(identifier=stats_view_identifier, schema=schema, database=database, type='view') %}
   {% call statement("main") %}
-    {{ get_create_view_as_sql(target_relation, build_sql) }}
+    {{ get_create_view_as_sql(target_relation, sql_for_column_stats) }}
   {% endcall %}
   {%- set full_refresh_mode = (should_full_refresh()) -%}
   {%- set should_revoke = should_revoke(target_relation, full_refresh_mode) %}

quollio_core/dbt_projects/snowflake/models/quollio_stats_columns.sql

@@ -1,7 +1,6 @@
 {{
     config(
-        materialized='divided_view',
-        chunk=20
+        materialized='divided_view'
     )
 }}
 -- depends_on: {{ ref('quollio_stats_profiling_columns') }}

quollio_core/dbt_projects/snowflake/models/quollio_stats_profiling_columns.sql

@@ -36,24 +36,61 @@ WITH columns AS (
       table_catalog
       , table_schema
       , name  
+), m_view_sys_columns AS (
+  SELECT
+    cols.table_catalog
+    , cols.table_schema
+    , cols.table_name
+    , cols.column_name
+    , cols.data_type
+  FROM
+    {{ source('account_usage', 'COLUMNS') }} cols
+  LEFT OUTER JOIN
+    {{ source('account_usage', 'TABLES') }} tbls
+  ON
+    cols.table_catalog = tbls.table_catalog
+    AND cols.table_schema = tbls.table_schema
+    AND cols.table_name = tbls.table_name
+  WHERE
+    tbls.table_type = 'MATERIALIZED VIEW'
+    AND cols.column_name = 'SYS_MV_SOURCE_PARTITION'
+), implicit_columns_removed AS (
+  SELECT
+    c.table_catalog
+    , c.table_schema
+    , c.table_name
+    , c.column_name
+    , c.data_type
+  FROM
+    columns c
+  INNER JOIN
+    accessible_tables a
+  ON
+    c.table_catalog = a.table_catalog
+    AND c.table_schema = a.table_schema
+    AND c.table_name = a.name
+  MINUS
+  SELECT
+    table_catalog
+    , table_schema
+    , table_name
+    , column_name
+    , data_type
+  FROM
+    m_view_sys_columns
+), final AS (
+  SELECT
+    table_catalog
+    , table_schema
+    , table_name
+    , column_name
+    , data_type
+    , case when data_type in('NUMBER','DECIMAL', 'DEC', 'NUMERIC',
+                             'INT', 'INTEGER', 'BIGINT', 'SMALLINT',
+                             'TINYINT', 'BYTEINT')
+                             THEN true
+           else false END AS is_calculable
+  FROM
+    implicit_columns_removed
 )
-
-SELECT
-  c.table_catalog
-  , c.table_schema
-  , c.table_name
-  , c.column_name
-  , c.data_type
-  , case when c.data_type in('NUMBER','DECIMAL', 'DEC', 'NUMERIC',
-                           'INT', 'INTEGER', 'BIGINT', 'SMALLINT',
-                           'TINYINT', 'BYTEINT')
-                           THEN true
-         else false END AS is_calculable
-FROM
-  columns c
-INNER JOIN
-  accessible_tables a
-ON
-  c.table_catalog = a.table_catalog
-  AND c.table_schema = a.table_schema
-  AND c.table_name = a.name
+select * from final

quollio_core/helper/core.py

@@ -31,3 +31,7 @@ def setup_dbt_profile(connections_json: Dict[str, str], template_path: str, temp
     with open(profile_path, "w") as profiles:
         yaml.dump(yaml.safe_load(profiles_body), profiles, default_flow_style=False, allow_unicode=True)
     return
+
+
+def trim_prefix(s: str, prefix: str) -> str:
+    return s.lstrip(prefix)

quollio_core/helper/env_default.py

@@ -6,6 +6,8 @@ Currently requires explicit naming of env vars to check for
 
 import argparse
 import os
+from distutils.util import strtobool
+from typing import Union
 
 
 # Courtesy of http://stackoverflow.com/a/10551190 with env-var retrieval fixed
@@ -16,7 +18,10 @@ class EnvDefault(argparse.Action):
     def __init__(self, envvar, required=True, default=None, **kwargs):
         # override values if envvar exists
         if envvar in os.environ:
-            default = os.environ[envvar]
+            if kwargs.get("nargs", None) is None:
+                default = os.environ[envvar]
+            else:
+                default = os.environ[envvar].split(" ")
         if required and default:
             required = False
         super(EnvDefault, self).__init__(default=default, required=required, **kwargs)
@@ -25,9 +30,30 @@ class EnvDefault(argparse.Action):
         setattr(namespace, self.dest, values)
 
 
+class EnvStoreTrue(argparse._StoreTrueAction):
+    """An argparse action class that auto-sets missing default values from env vars for store_true."""
+
+    def __init__(self, envvar, required=True, default=None, **kwargs):
+        # Only pass the arguments that argparse._StoreTrueAction expects
+        action_kwargs = {key: value for key, value in kwargs.items() if key in ("option_strings", "dest")}
+        if envvar in os.environ:
+            default = _convert_value_to_bool(os.environ[envvar])
+        if required and default:
+            required = False
+        super(EnvStoreTrue, self).__init__(default=default, required=required, **action_kwargs)
+
+
 # functional sugar for the above
-def env_default(envvar):
+def env_default(envvar, store_true=False):
     def wrapper(**kwargs):
+        if store_true:
+            return EnvStoreTrue(envvar, **kwargs)
         return EnvDefault(envvar, **kwargs)
 
     return wrapper
+
+
+def _convert_value_to_bool(v: Union[str, bool]) -> bool:
+    if isinstance(v, str):
+        return bool(strtobool(v))
+    return v

quollio_core/helper/log.py

@@ -0,0 +1,17 @@
+import logging
+
+
+def set_log_level(level: str = "info") -> None:
+    fmt = "%(asctime)s - %(levelname)s - %(name)s - %(message)s"
+    if level == "info":
+        logging.basicConfig(level=logging.INFO, format=fmt)
+    elif level == "debug":
+        logging.basicConfig(level=logging.DEBUG, format=fmt)
+    elif level == "warn":
+        logging.basicConfig(level=logging.WARNING, format=fmt)
+    elif level == "error":
+        logging.basicConfig(level=logging.ERROR, format=fmt)
+    elif level == "critical":
+        logging.basicConfig(level=logging.CRITICAL, format=fmt)
+    else:
+        logging.basicConfig(level=logging.NOTSET, format=fmt)