quollio-core 0.4.11__py3-none-any.whl → 0.4.13__py3-none-any.whl

quollio_core/__init__.py

@@ -1,4 +1,4 @@
 """Quollio Core"""
 
-__version__ = "0.4.11"
+__version__ = "0.4.13"
 __author__ = "Quollio Technologies, Inc"

quollio_core/bigquery.py

@@ -1,19 +1,37 @@
 import argparse
 import json
-import logging
+
+from google.auth.credentials import Credentials
 
 from quollio_core.helper.env_default import env_default
-from quollio_core.helper.log import set_log_level
-from quollio_core.profilers.bigquery import bigquery_table_lineage
+from quollio_core.helper.log_utils import configure_logging, error_handling_decorator, logger
+from quollio_core.profilers.bigquery import bigquery_table_lineage, bigquery_table_stats
 from quollio_core.repository import qdc
-from quollio_core.repository.bigquery import get_credentials, get_org_id
+from quollio_core.repository.bigquery import BigQueryClient, get_credentials, get_org_id
+
+
+def initialize_credentials(credentials_json: str) -> Credentials:
+    return get_credentials(json.loads(credentials_json))
+
 
-logger = logging.getLogger(__name__)
+def initialize_org_id(credentials_json: str) -> str:
+    return get_org_id(json.loads(credentials_json))
 
 
+def initialize_bq_client(credentials: Credentials, project_id: str) -> BigQueryClient:
+    return BigQueryClient(credentials=credentials, project_id=project_id)
+
+
+@error_handling_decorator
 def load_lineage(
-    qdc_client: qdc.QDCExternalAPIClient, project_id: str, regions: list, tenant_id: str, credentials: dict, org_id: str
-):
+    tenant_id: str,
+    project_id: str,
+    regions: list,
+    org_id: str,
+    credentials: Credentials,
+    qdc_client: qdc.QDCExternalAPIClient,
+) -> None:
+    logger.info("Loading lineage data.")
     bigquery_table_lineage(
         qdc_client=qdc_client,
         tenant_id=tenant_id,
@@ -22,29 +40,53 @@ def load_lineage(
         credentials=credentials,
         org_id=org_id,
     )
+    logger.info("Lineage data loaded successfully.")
+
+
+@error_handling_decorator
+def load_stats(
+    conn: BigQueryClient,
+    tenant_id: str,
+    org_id: str,
+    qdc_client: qdc.QDCExternalAPIClient,
+    dataplex_stats_tables: list,
+) -> None:
+    logger.info("Loading statistics data.")
+    bigquery_table_stats(
+        bq_client=conn,
+        qdc_client=qdc_client,
+        tenant_id=tenant_id,
+        org_id=org_id,
+        dataplex_stats_tables=dataplex_stats_tables,
+    )
+    logger.info("Statistics data loaded successfully.")
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
-        prog="Quollio Intelligence Agent for Google BigQuery",
-        description="Collect lineage and stats from Google BigQuery and load to Quollio Data Catalog",
+        prog="Quollio Intelligence Agent for BigQuery",
+        description="Load lineage and stats to Quollio from BigQuery using Dataplex and BigQuery APIs",
         epilog="Copyright (c) 2024 Quollio Technologies, Inc.",
     )
     parser.add_argument(
         "commands",
-        choices=["load_lineage"],
+        choices=["load_lineage", "load_stats"],
         type=str,
         nargs="+",
         help="""
         The command to execute.
-        'load_lineage': Load lineage data from Google Data Catalog to Quollio,
+        'load_lineage': Load lineage data from created views to Quollio,
+        'load_stats': Load stats from created views to Quollio,
         """,
     )
     parser.add_argument(
-        "--credentials",
+        "--log_level",
         type=str,
-        action=env_default("GOOGLE_APPLICATION_CREDENTIALS"),
-        help="Crendentials for Google Cloud Platform",
+        choices=["debug", "info", "warn", "error", "none"],
+        action=env_default("LOG_LEVEL"),
+        default="info",
+        required=False,
+        help="The log level for dbt commands. Default value is info",
     )
     parser.add_argument(
         "--tenant_id",
@@ -53,6 +95,27 @@ if __name__ == "__main__":
         required=False,
         help="The tenant id (company id) where the lineage and stats are loaded",
     )
+    parser.add_argument(
+        "--project_id",
+        type=str,
+        default=None,
+        required=False,
+        help="Project ID of the BigQuery project to load lineage and stats from (default is loaded from credentials)",
+    )
+    parser.add_argument(
+        "--regions",
+        type=str,
+        action=env_default("GCP_REGIONS"),
+        required=True,
+        help="Comma-separated list of regions BigQuery data is in",
+    )
+    parser.add_argument(
+        "--credentials_json",
+        type=str,
+        action=env_default("GOOGLE_APPLICATION_CREDENTIALS"),
+        required=True,
+        help="Credentials JSON",
+    )
     parser.add_argument(
         "--api_url",
         type=str,
@@ -74,50 +137,47 @@ if __name__ == "__main__":
         required=False,
         help="The client secret that is created on Quollio console to let clients access Quollio External API",
     )
+
     parser.add_argument(
-        "--project_id",
-        type=str,
-        action=env_default("GCP_PROJECT_ID"),
-        required=False,
-        help="GCP Project ID",
-    )
-    parser.add_argument(
-        "--regions",
-        type=str,
-        action=env_default("GCP_REGIONS"),
-        required=False,
-        help="GCP regions where the data is located. Multiple regions can be provided separated by space.",
-        nargs="+",
-    )
-    parser.add_argument(
-        "--log_level",
+        "--dataplex_stats_tables",
         type=str,
-        choices=["debug", "info", "warn", "error", "none"],
-        action=env_default("LOG_LEVEL"),
+        action=env_default("DATAPLEX_STATS_TABLES"),
         required=False,
-        help="The log level for dbt commands. Default value is info",
+        help="Comma-separated list of dataplex stats tables - <project_id>.<dataset_id>.<table_id>",
     )
 
     args = parser.parse_args()
-    set_log_level(level=args.log_level)
 
-    if len(args.commands) == 0:
-        raise ValueError("No command is provided")
+    # Validate that dataplex_stats_tables is provided if load_stats is in commands
+    if "load_stats" in args.commands and not args.dataplex_stats_tables:
+        parser.error("--dataplex_stats_tables is required when 'load_stats' command is used")
 
-    if "load_lineage" in args.commands:
-        qdc_client = qdc.QDCExternalAPIClient(
-            base_url=args.api_url, client_id=args.client_id, client_secret=args.client_secret
-        )
+    configure_logging(args.log_level)
 
-        credentials_json = json.loads(args.credentials)
-        credentials = get_credentials(credentials_json=credentials_json)
-        org_id = get_org_id(credentials_json=credentials_json)
+    credentials = initialize_credentials(args.credentials_json)
+    org_id = initialize_org_id(args.credentials_json)
+    qdc_client = qdc.initialize_qdc_client(args.api_url, args.client_id, args.client_secret)
+    bq_client = initialize_bq_client(credentials, args.project_id)
+    if args.project_id is None:
+        args.project_id = json.loads(args.credentials_json)["project_id"]
+    regions = args.regions.split(",")
 
+    if "load_lineage" in args.commands:
         load_lineage(
-            qdc_client=qdc_client,
-            project_id=args.project_id,
-            regions=args.regions,
             tenant_id=args.tenant_id,
+            project_id=args.project_id,
+            regions=regions,
+            org_id=org_id,
             credentials=credentials,
+            qdc_client=qdc_client,
+        )
+
+    if "load_stats" in args.commands:
+        tables = args.dataplex_stats_tables.split(",")
+        load_stats(
+            conn=bq_client,
+            tenant_id=args.tenant_id,
             org_id=org_id,
+            qdc_client=qdc_client,
+            dataplex_stats_tables=tables,
         )

quollio_core/bricks.py

@@ -10,6 +10,7 @@ from quollio_core.profilers.databricks import (
     databricks_column_stats,
     databricks_table_level_lineage,
 )
+from quollio_core.profilers.stats import get_column_stats_items
 from quollio_core.repository import databricks as db
 from quollio_core.repository import dbt, qdc
 
@@ -21,7 +22,6 @@ def build_view(
     target_tables: str = "",
     log_level: str = "info",
 ) -> None:
-
     logger.info("Build profiler views using dbt")
     # set parameters
     dbt_client = dbt.DBTClient()
@@ -64,7 +64,6 @@ def load_lineage(
     tenant_id: str,
     enable_column_lineage: bool = False,
 ) -> None:
-
     logger.info("Generate Databricks table to table lineage.")
     databricks_table_level_lineage(
         conn=conn,
@@ -98,7 +97,6 @@ def load_column_stats(
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
 ) -> None:
-
     logger.info("Generate Databricks column stats.")
     databricks_column_stats(
         conn=conn,
@@ -240,6 +238,19 @@ if __name__ == "__main__":
         help="Whether to ingest column lineage into QDIC or not. Default value is False",
     )
 
+    stats_items = get_column_stats_items()
+    parser.add_argument(
+        "--target_stats_items",
+        type=str,
+        nargs="*",
+        choices=stats_items,
+        default=stats_items,
+        action=env_default("DATABRICKS_STATS_ITEMS"),
+        required=False,
+        help="The items for statistic values.\
+              You can choose the items to be aggregated for stats. All items are selected by default.",
+    )
+
     args = parser.parse_args()
     set_log_level(level=args.log_level)
 
@@ -284,5 +295,6 @@ if __name__ == "__main__":
             endpoint=args.host,
             qdc_client=qdc_client,
             tenant_id=args.tenant_id,
+            stats_items=args.target_stats_items,
             monitoring_table_suffix=args.monitoring_table_suffix,
         )

quollio_core/helper/log_utils.py

@@ -0,0 +1,48 @@
+import inspect
+import logging
+
+LOG_LEVELS = {
+    "critical": logging.CRITICAL,
+    "error": logging.ERROR,
+    "warning": logging.WARNING,
+    "info": logging.INFO,
+    "debug": logging.DEBUG,
+    "notset": logging.NOTSET,
+}
+
+logger = logging.getLogger(__name__)
+
+
+def configure_logging(level: str = "INFO"):
+    """Configure logging settings."""
+    log_level = LOG_LEVELS.get(level.lower())
+    if log_level is None:
+        raise ValueError(f"Unknown log level: {level}")
+
+    logging.basicConfig(
+        level=log_level,
+        format="%(asctime)s - %(levelname)s - %(funcName)s:%(lineno)d - %(message)s",
+    )
+    logger.setLevel(log_level)
+    logger.info(f"Logging is configured to {level} level.")
+
+
+def error_handling_decorator(func):
+    """Decorator for consistent error handling in CLI commands."""
+
+    def wrapper(*args, **kwargs):
+        func_name = func.__name__
+        try:
+            logger.debug(f"Starting {func_name}")
+            result = func(*args, **kwargs)
+            logger.debug(f"Completed {func_name} successfully")
+            return result
+        except Exception as e:
+
+            current_frame = inspect.currentframe()
+            error_frame = current_frame.f_back
+            line_number = error_frame.f_lineno
+            logger.error(f"Error in {func_name} at line {line_number}: {str(e)}", exc_info=True)
+            raise
+
+    return wrapper

quollio_core/profilers/bigquery.py

@@ -1,26 +1,28 @@
-import logging
-from typing import Any, Dict, List
+from typing import Dict, List
 
+from google.auth.credentials import Credentials
+
+from quollio_core.helper.log_utils import error_handling_decorator, logger
 from quollio_core.profilers.lineage import gen_table_lineage_payload, parse_bigquery_table_lineage
+from quollio_core.profilers.stats import gen_table_stats_payload
 from quollio_core.repository import qdc
 from quollio_core.repository.bigquery import BigQueryClient, GCPLineageClient, get_entitiy_reference, get_search_request
 
-logger = logging.getLogger(__name__)
-
 
+@error_handling_decorator
 def bigquery_table_lineage(
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
     project_id: str,
     regions: list,
     org_id: str,
-    credentials: Any,
-):
+    credentials: Credentials,
+) -> None:
     lineage_client = GCPLineageClient(credentials)
-    bq_client = BigQueryClient(credentials)
+    bq_client = BigQueryClient(credentials, project_id)
 
-    datasets = bq_client.list_datasets(project_id)
-    all_tables = generate_table_list(datasets, bq_client)
+    datasets = bq_client.list_dataset_ids()
+    all_tables = generate_table_list(bq_client, datasets)
     lineage_links = generate_lineage_links(all_tables, lineage_client, project_id, regions)
     lineage_links = parse_bigquery_table_lineage(lineage_links)
 
@@ -29,7 +31,7 @@ def bigquery_table_lineage(
     req_count = 0
     for update_table_lineage_input in update_table_lineage_inputs:
         logger.info(
-            "Generating table lineage. downstream: %s -> %s-> %s",
+            "Generating table lineage. downstream: %s -> %s -> %s",
             update_table_lineage_input.downstream_database_name,
             update_table_lineage_input.downstream_schema_name,
             update_table_lineage_input.downstream_table_name,
@@ -43,29 +45,65 @@ def bigquery_table_lineage(
     logger.info("Generating table lineage is finished. %s lineages are ingested.", req_count)
 
 
-def generate_table_list(datasets: List[str], bq_client: BigQueryClient) -> List[str]:
+@error_handling_decorator
+def bigquery_table_stats(
+    qdc_client: qdc.QDCExternalAPIClient,
+    bq_client: BigQueryClient,
+    tenant_id: str,
+    org_id: str,
+    dataplex_stats_tables: list,
+) -> None:
+    profiling_results = []
+    for table in dataplex_stats_tables:
+        logger.info("Profiling columns using Dataplex stats table: %s", table)
+        profiling_results.extend(column_stats_from_dataplex(bq_client, table))
+
+    stats = gen_table_stats_payload(tenant_id, org_id, profiling_results)
+
+    for stat in stats:
+        status_code = qdc_client.update_stats_by_id(
+            global_id=stat.global_id,
+            payload=stat.body.as_dict(),
+        )
+        if status_code == 200:
+            logger.info(
+                "Stats for column %s -> %s -> %s -> %s is successfully ingested.",
+                stat.db,
+                stat.schema,
+                stat.table,
+                stat.column,
+            )
+            logger.debug("Stats for column id %s is successfully ingested.", stat.global_id)
+
+
+def generate_table_list(bq_client: BigQueryClient, datasets: List[str]) -> List[str]:
     all_tables = []
     for dataset in datasets:
         all_tables.extend(
             [
                 table
-                for table in bq_client.list_tables(dataset.dataset_id)
-                if table.table_type in ["TABLE", "VIEW", "MATERIALIZED_VIEW"]
-            ]
+                for table in bq_client.list_tables(dataset)
+                if table["table_type"] in ["TABLE", "VIEW", "MATERIALIZED_VIEW"]
+            ],
         )
 
     all_table_names = []
     for table in all_tables:
-        all_table_names.append(f"{table.project}.{table.dataset_id}.{table.table_id}")
+        all_table_names.append(f"{bq_client.client.project}.{table['dataset_id']}.{table['table_id']}")
 
     return all_table_names
 
 
 def generate_lineage_links(
-    all_tables: List[str], lineage_client: GCPLineageClient, project_id: str, regions: List[str]
+    all_tables: List[str],
+    lineage_client: GCPLineageClient,
+    project_id: str,
+    regions: List[str],
 ) -> Dict[str, List[str]]:
     lineage_links = {}
     for table in all_tables:
+        if "quollio" in table.lower():
+            continue
         downstream = get_entitiy_reference()
         downstream.fully_qualified_name = f"bigquery:{table}"
 
@@ -74,8 +112,34 @@ def generate_lineage_links(
             response = lineage_client.get_links(request=request)
             for lineage in response:
                 target_table = str(lineage.target.fully_qualified_name).replace("bigquery:", "")
+                source_table = str(lineage.source.fully_qualified_name).replace("bigquery:", "")
                 if target_table not in lineage_links:
                     lineage_links[target_table] = []
-                lineage_links[target_table].append(str(lineage.source.fully_qualified_name).replace("bigquery:", ""))
+                if source_table not in lineage_links[target_table]:
+                    lineage_links[target_table].append(source_table)
 
     return lineage_links
+
+
+def column_stats_from_dataplex(bq_client: BigQueryClient, profiling_table: str) -> List[Dict]:
+    query = f"""
+    SELECT
+        data_source.table_project_id AS DB_NAME,
+        data_source.dataset_id AS SCHEMA_NAME,
+        data_source.table_id AS TABLE_NAME,
+        column_name AS COLUMN_NAME,
+        min_value AS MIN_VALUE,
+        max_value AS MAX_VALUE,
+        average_value AS AVG_VALUE,
+        quartile_median AS MEDIAN_VALUE,
+        standard_deviation AS STDDEV_VALUE,
+        top_n[0][0] AS MODE_VALUE,
+        CAST((percent_null / 100) * job_rows_scanned AS INT) as NULL_COUNT,
+        CAST((percent_unique / 100) * job_rows_scanned AS INT) as CARDINALITY
+    FROM `{profiling_table}`
+    """
+    logger.debug(f"Executing Query: {query}")
+    results = bq_client.client.query(query).result()
+
+    # Convert RowIterator to a list of dictionaries
+    return [dict(row) for row in results]

quollio_core/profilers/databricks.py

@@ -6,7 +6,7 @@ from quollio_core.profilers.lineage import (
     gen_table_lineage_payload,
     parse_databricks_table_lineage,
 )
-from quollio_core.profilers.stats import gen_table_stats_payload
+from quollio_core.profilers.stats import gen_table_stats_payload, get_is_target_stats_items, render_sql_for_stats
 from quollio_core.repository import databricks, qdc
 
 logger = logging.getLogger(__name__)
@@ -125,59 +125,63 @@ def _get_monitoring_tables(
 
 
 def _get_column_stats(
-    conn: databricks.DatabricksConnectionConfig, monitoring_table_suffix: str = "_profile_metrics"
+    conn: databricks.DatabricksConnectionConfig,
+    stats_items: List[str],
+    monitoring_table_suffix: str = "_profile_metrics",
 ) -> List[Dict[str, str]]:
     tables = _get_monitoring_tables(conn, monitoring_table_suffix)
     if not tables:
         return []
     stats = []
+    is_aggregate_items = get_is_target_stats_items(stats_items=stats_items)
     for table in tables:
         monitored_table = table["table_fqdn"].removesuffix("_profile_metrics")
         monitored_table = monitored_table.split(".")
         if len(monitored_table) != 3:
             raise ValueError(f"Invalid table name: {table['table_fqdn']}")
         with databricks.DatabricksQueryExecutor(config=conn) as databricks_executor:
-            query = """
-                    WITH profile_record_history AS (
-                        SELECT
-                            COLUMN_NAME
-                            , distinct_count as CARDINALITY
-                            , MAX as MAX_VALUE
-                            , MIN as MIN_VALUE
-                            , AVG as AVG_VALUE
-                            , MEDIAN as MEDIAN_VALUE
-                            , STDDEV as STDDEV_VALUE
-                            , NUM_NULLS as NULL_COUNT
-                            , frequent_items[0].item AS MODE_VALUE
-                            , row_number() over(partition by column_name order by window desc) rownum
-                        FROM
-                            {monitoring_table}
-                        WHERE
-                            column_name not in (':table')
-                    )
-                    SELECT
-                        "{monitored_table_catalog}" as DB_NAME
-                        , "{monitored_table_schema}" as SCHEMA_NAME
-                        , "{monitored_table_name}" as TABLE_NAME
-                        , COLUMN_NAME
-                        , CARDINALITY
-                        , MAX_VALUE
-                        , MIN_VALUE
-                        , AVG_VALUE
-                        , MEDIAN_VALUE
-                        , STDDEV_VALUE
-                        , NULL_COUNT
-                        , MODE_VALUE
-                    FROM
-                        profile_record_history
-                    WHERE
-                        rownum = 1
-                """.format(
+            cte = """
+            WITH profile_record_history AS (
+                SELECT
+                    COLUMN_NAME
+                    , distinct_count as cardinality
+                    , MAX as max_value
+                    , MIN as min_value
+                    , AVG as avg_value
+                    , MEDIAN as median_value
+                    , STDDEV as stddev_value
+                    , NUM_NULLS as null_count
+                    , get(frequent_items, 0).item AS mode_value
+                    , row_number() over(partition by column_name order by window desc) rownum
+                FROM
+                    {monitoring_table}
+                WHERE
+                    column_name not in (':table')
+            ), profile_record AS (
+            SELECT
+                "{monitored_table_catalog}" as db_name
+                , "{monitored_table_schema}" as schema_name
+                , "{monitored_table_name}" as table_name
+                , column_name
+                , max_value
+                , min_value
+                , null_count
+                , cardinality
+                , avg_value
+                , median_value
+                , mode_value
+                , stddev_value
+            FROM
+                profile_record_history
+            WHERE
+                rownum = 1
+            )""".format(
                 monitoring_table=table["table_fqdn"],
                 monitored_table_catalog=monitored_table[0],
                 monitored_table_schema=monitored_table[1],
                 monitored_table_name=monitored_table[2],
             )
+            query = render_sql_for_stats(is_aggregate_items=is_aggregate_items, table_fqn="profile_record", cte=cte)
             logger.debug(f"The following sql will be fetched to retrieve stats values. {query}")
             stats.append(databricks_executor.get_query_results(query))
     return stats
@@ -188,10 +192,12 @@ def databricks_column_stats(
     endpoint: str,
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
+    stats_items: List[str],
     monitoring_table_suffix: str = "_profile_metrics",
 ) -> None:
-    table_stats = _get_column_stats(conn, monitoring_table_suffix)
+    table_stats = _get_column_stats(conn, stats_items, monitoring_table_suffix)
     for table in table_stats:
+        logger.debug("Table %s will be aggregated.", table)
         stats = gen_table_stats_payload(tenant_id=tenant_id, endpoint=endpoint, stats=table)
         for stat in stats:
             status_code = qdc_client.update_stats_by_id(

quollio_core/profilers/redshift.py

@@ -1,8 +1,13 @@
 import logging
+from typing import List
 
 from quollio_core.profilers.lineage import gen_table_lineage_payload, gen_table_lineage_payload_inputs
 from quollio_core.profilers.sqllineage import SQLLineage
-from quollio_core.profilers.stats import gen_table_stats_payload_from_tuple
+from quollio_core.profilers.stats import (
+    gen_table_stats_payload_from_tuple,
+    get_is_target_stats_items,
+    render_sql_for_stats,
+)
 from quollio_core.repository import qdc, redshift
 
 logger = logging.getLogger(__name__)
@@ -76,38 +81,24 @@ def redshift_table_stats(
     conn: redshift.RedshiftConnectionConfig,
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
+    stats_items: List[str],
 ) -> None:
-
+    is_aggregate_items = get_is_target_stats_items(stats_items=stats_items)
     with redshift.RedshiftQueryExecutor(config=conn) as redshift_executor:
         stats_query = _gen_get_stats_views_query(
             db=conn.database,
             schema=conn.schema,
         )
         stats_views = redshift_executor.get_query_results(query=stats_query)
+        logger.info("Found %s for table statistics.", len(stats_views))
 
         req_count = 0
         for stats_view in stats_views:
-            stats_query = """
-            SELECT
-                db_name
-                , schema_name
-                , table_name
-                , column_name
-                , max_value
-                , min_value
-                , null_count
-                , cardinality
-                , avg_value
-                , median_value
-                , mode_value
-                , stddev_value
-            FROM
-                {db}.{schema}.{table}
-            """.format(
-                db=stats_view[0],
-                schema=stats_view[1],
-                table=stats_view[2],
+            table_fqn = "{catalog}.{schema}.{table}".format(
+                catalog=stats_view[0], schema=stats_view[1], table=stats_view[2]
             )
+            stats_query = render_sql_for_stats(is_aggregate_items=is_aggregate_items, table_fqn=table_fqn)
+            logger.debug(f"The following sql will be fetched to retrieve stats values. {stats_query}")
             stats_result = redshift_executor.get_query_results(query=stats_query)
             payloads = gen_table_stats_payload_from_tuple(tenant_id=tenant_id, endpoint=conn.host, stats=stats_result)
             for payload in payloads:

quollio_core/profilers/snowflake.py

@@ -1,4 +1,5 @@
 import logging
+from typing import List
 
 from quollio_core.profilers.lineage import (
     gen_column_lineage_payload,
@@ -6,7 +7,7 @@ from quollio_core.profilers.lineage import (
     parse_snowflake_results,
 )
 from quollio_core.profilers.sqllineage import SQLLineage
-from quollio_core.profilers.stats import gen_table_stats_payload
+from quollio_core.profilers.stats import gen_table_stats_payload, get_is_target_stats_items, render_sql_for_stats
 from quollio_core.repository import qdc, snowflake
 
 logger = logging.getLogger(__name__)
@@ -154,6 +155,7 @@ def snowflake_table_stats(
     conn: snowflake.SnowflakeConnectionConfig,
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
+    stats_items: List[str],
 ) -> None:
     with snowflake.SnowflakeQueryExecutor(conn) as sf_executor:
         stats_query = _gen_get_stats_views_query(
@@ -163,28 +165,12 @@ def snowflake_table_stats(
         stats_views = sf_executor.get_query_results(query=stats_query)
 
         req_count = 0
+        is_aggregate_items = get_is_target_stats_items(stats_items=stats_items)
         for stats_view in stats_views:
-            stats_query = """
-            SELECT
-                db_name
-                , schema_name
-                , table_name
-                , column_name
-                , max_value
-                , min_value
-                , null_count
-                , cardinality
-                , avg_value
-                , median_value
-                , mode_value
-                , stddev_value
-            FROM
-                {db}.{schema}.{table}
-            """.format(
-                db=stats_view["TABLE_CATALOG"],
-                schema=stats_view["TABLE_SCHEMA"],
-                table=stats_view["TABLE_NAME"],
+            table_fqn = "{catalog}.{schema}.{table}".format(
+                catalog=stats_view["TABLE_CATALOG"], schema=stats_view["TABLE_SCHEMA"], table=stats_view["TABLE_NAME"]
             )
+            stats_query = render_sql_for_stats(is_aggregate_items=is_aggregate_items, table_fqn=table_fqn)
             logger.debug(f"The following sql will be fetched to retrieve stats values. {stats_query}")
             stats_result = sf_executor.get_query_results(query=stats_query)
             payloads = gen_table_stats_payload(tenant_id=tenant_id, endpoint=conn.account_id, stats=stats_result)

quollio_core/profilers/stats.py

@@ -1,8 +1,10 @@
 import logging
-from dataclasses import asdict, dataclass
+from dataclasses import asdict, dataclass, fields
 from decimal import ROUND_HALF_UP, Decimal
 from typing import Dict, List, Tuple, Union
 
+from jinja2 import Template
+
 from quollio_core.helper.core import new_global_id
 
 logger = logging.getLogger(__name__)
@@ -71,30 +73,35 @@ def convert_value_type(obj, cast_str: bool = False):
 def gen_table_stats_payload(tenant_id: str, endpoint: str, stats: List[Dict[str, str]]) -> List[StatsRequest]:
     payloads = list()
     for stat in stats:
+        db_name = stat.get("DB_NAME", stat.get("db_name"))
+        schema_name = stat.get("SCHEMA_NAME", stat.get("schema_name"))
+        table_name = stat.get("TABLE_NAME", stat.get("table_name"))
+        column_name = stat.get("COLUMN_NAME", stat.get("column_name"))
         global_id_arg = "{db}{schema}{table}{column}".format(
-            db=stat["DB_NAME"], schema=stat["SCHEMA_NAME"], table=stat["TABLE_NAME"], column=stat["COLUMN_NAME"]
+            db=db_name, schema=schema_name, table=table_name, column=column_name
         )
         table_global_id = new_global_id(
             tenant_id=tenant_id, cluster_id=endpoint, data_id=global_id_arg, data_type="column"
         )
+        column_stats_input = ColumnStatsInput(
+            cardinality=convert_value_type(stat.get("CARDINALITY", stat.get("cardinality"))),
+            max=convert_value_type(stat.get("MAX_VALUE", stat.get("max_value")), True),
+            mean=convert_value_type(stat.get("AVG_VALUE", stat.get("avg_value")), True),
+            median=convert_value_type(stat.get("MEDIAN_VALUE", stat.get("median_value")), True),
+            min=convert_value_type(stat.get("MIN_VALUE", stat.get("min_value")), True),
+            mode=convert_value_type(stat.get("MODE_VALUE", stat.get("mode_value")), True),
+            number_of_null=convert_value_type(stat.get("NULL_COUNT", stat.get("null_count"))),
+            number_of_unique=convert_value_type(stat.get("CARDINALITY", stat.get("cardinality"))),
+            stddev=convert_value_type(stat.get("STDDEV_VALUE", stat.get("stddev_value")), True),
+        )
         stats_request = StatsRequest(
             global_id=table_global_id,
-            db=stat["DB_NAME"],
-            schema=stat["SCHEMA_NAME"],
-            table=stat["TABLE_NAME"],
-            column=stat["COLUMN_NAME"],
+            db=db_name,
+            schema=schema_name,
+            table=table_name,
+            column=column_name,
             body=StatsInput(
-                column_stats=ColumnStatsInput(
-                    cardinality=convert_value_type(stat["CARDINALITY"]),
-                    max=convert_value_type(stat["MAX_VALUE"], True),
-                    mean=convert_value_type(stat["AVG_VALUE"], True),
-                    median=convert_value_type(stat["MEDIAN_VALUE"], True),
-                    min=convert_value_type(stat["MIN_VALUE"], True),
-                    mode=convert_value_type(stat["MODE_VALUE"], True),
-                    number_of_null=convert_value_type(stat["NULL_COUNT"]),
-                    number_of_unique=convert_value_type(stat["CARDINALITY"]),
-                    stddev=convert_value_type(stat["STDDEV_VALUE"], True),
-                ),
+                column_stats=column_stats_input,
                 # MEMO: Table stats can be collected with metadata agent.
                 # Then, It's not necessary to update with this system for now.
                 table_stats=TableStatsInput(count=0, size=0.0),
@@ -138,3 +145,57 @@ def gen_table_stats_payload_from_tuple(
         )
         payloads.append(stats_request)
     return payloads
+
+
+def render_sql_for_stats(is_aggregate_items: Dict[str, bool], table_fqn: str, cte: str = "") -> str:
+    sql_template_for_stats = Template(
+        """
+        {% if cte -%}
+          {{ cte }}
+        {% endif -%}
+        SELECT
+            db_name
+            , schema_name
+            , table_name
+            , column_name
+            , {% if agg_max == True -%} max_value {% else -%} null as max_value {% endif %}
+            , {% if agg_min == True -%} min_value {% else -%} null as min_value {% endif %}
+            , {% if agg_null_count == True -%} null_count {% else -%} null as null_count {% endif %}
+            , {% if agg_cardinality == True -%} cardinality {% else -%} null as cardinality {% endif %}
+            , {% if agg_avg == True -%} avg_value {% else -%} null as avg_value {% endif %}
+            , {% if agg_median == True -%} median_value {% else -%} null as median_value {% endif %}
+            , {% if agg_mode == True -%} mode_value {% else -%} null as mode_value {% endif %}
+            , {% if agg_stddev == True -%} stddev_value {% else -%} null as stddev_value {% endif %}
+        FROM
+            {{ table_fqn }}
+    """
+    )
+    query = sql_template_for_stats.render(
+        agg_max=is_aggregate_items["max"],
+        agg_min=is_aggregate_items["min"],
+        agg_null_count=is_aggregate_items["number_of_null"],
+        agg_cardinality=is_aggregate_items["cardinality"],
+        agg_avg=is_aggregate_items["mean"],
+        agg_median=is_aggregate_items["median"],
+        agg_mode=is_aggregate_items["mode"],
+        agg_stddev=is_aggregate_items["stddev"],
+        table_fqn=table_fqn,
+        cte=cte,
+    )
+    return query
+
+
+def get_is_target_stats_items(stats_items: List[str]) -> List[Dict[str, bool]]:
+    target_stats_fields = get_column_stats_items()
+    is_aggregate_items = dict()
+    for target_stats_field in target_stats_fields:
+        is_aggregate_items[target_stats_field] = False
+
+    for stats_item in stats_items:
+        is_aggregate_items[stats_item] = True
+
+    return is_aggregate_items
+
+
+def get_column_stats_items() -> List[str]:
+    return [field.name for field in fields(ColumnStatsInput)]

quollio_core/redshift.py

@@ -10,6 +10,7 @@ from quollio_core.profilers.redshift import (
     redshift_table_level_sqllineage,
     redshift_table_stats,
 )
+from quollio_core.profilers.stats import get_column_stats_items
 from quollio_core.repository import dbt, qdc, redshift
 
 logger = logging.getLogger(__name__)
@@ -98,13 +99,19 @@ def load_stats(
     conn: redshift.RedshiftConnectionConfig,
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
+    stats_items: str,
 ) -> None:
-
     logger.info("Generate redshift stats.")
+
+    if stats_items is None:
+        raise ValueError("No stats items are not selected. Please specify any value to `stats_items` param.")
+
+    logger.info("The following values will be aggregated. {stats_items}".format(stats_items=stats_items))
     redshift_table_stats(
         conn=conn,
         qdc_client=qdc_client,
         tenant_id=tenant_id,
+        stats_items=stats_items,
     )
 
     logger.info("Stats data is successfully loaded.")
@@ -116,7 +123,6 @@ def load_sqllineage(
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
 ) -> None:
-
     logger.info("Generate Redshift sqllineage.")
     redshift_table_level_sqllineage(
         conn=conn,
@@ -261,6 +267,19 @@ if __name__ == "__main__":
         required=False,
         help="The client secrete that is created on Quollio console to let clients access Quollio External API",
     )
+
+    stats_items = get_column_stats_items()
+    parser.add_argument(
+        "--target_stats_items",
+        type=str,
+        nargs="*",
+        choices=stats_items,
+        default=stats_items,
+        action=env_default("REDSHIFT_STATS_ITEMS"),
+        required=False,
+        help="The items for stats values. \
+              You can choose the items to be aggregated for stats. All items are selected by default.",
+    )
     args = parser.parse_args()
     set_log_level(level=args.log_level)
 
@@ -306,6 +325,7 @@ if __name__ == "__main__":
             conn=conn,
             qdc_client=qdc_client,
             tenant_id=args.tenant_id,
+            stats_items=args.target_stats_items,
         )
     if "load_sqllineage" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(

quollio_core/repository/bigquery.py

@@ -1,41 +1,75 @@
-import logging
+from typing import Any, Dict, List
 
 from google.cloud.bigquery import Client
 from google.cloud.datacatalog_lineage_v1 import EntityReference, LineageClient, SearchLinksRequest
 from google.oauth2.service_account import Credentials
 from googleapiclient.discovery import build
 
-logger = logging.getLogger(__name__)
+from quollio_core.helper.log_utils import logger  # Importing the logger from logging_utils
 
 
 class BigQueryClient:
-    def __init__(self, credentials: Credentials) -> None:
-        self.client = self.__initialze(credentials=credentials)
+    """Client to interact with the BigQuery API."""
 
-    def __initialze(self, credentials: Credentials) -> Client:
-        client = Client(credentials=credentials)
-        return client
+    def __init__(self, credentials: Credentials, project_id: str) -> None:
+        """Initialize the BigQuery client with provided credentials."""
+        self.client = self.__initialize(credentials=credentials, project_id=project_id)
 
-    def list_datasets(self, project_id) -> list:
-        datasets = list(self.client.list_datasets(project_id))
-        logger.debug("Found %s datasets in project %s", len(datasets), project_id)
-        return datasets
+    def __initialize(self, credentials: Credentials, project_id: str) -> Client:
+        return Client(credentials=credentials, project=project_id)
 
-    def list_tables(self, dataset_id) -> list:
+    def list_dataset_ids(self) -> List[str]:
+        """List all dataset ids in the project."""
+        datasets = list(self.client.list_datasets())
+        logger.debug("Found %s datasets in project %s", len(datasets), self.client.project)
+        return [dataset.dataset_id for dataset in datasets]
+
+    def list_tables(self, dataset_id: str) -> List[Dict[str, str]]:
+        """List all tables in the dataset."""
         tables = list(self.client.list_tables(dataset_id))
         logger.debug("Found %s tables in dataset %s", len(tables), dataset_id)
-        return list(self.client.list_tables(dataset_id))
+        return [
+            {
+                "table_id": table.table_id,
+                "table_type": table.table_type,
+                "project": table.project,
+                "dataset_id": table.dataset_id,
+            }
+            for table in tables
+        ]
+
+    def get_columns(self, table_id: str, dataset_id: str) -> List[Dict[str, str]]:
+        """Get the columns of the table."""
+        table = self.client.get_table(f"{self.client.project}.{dataset_id}.{table_id}")
+        return [{"name": field.name, "type": field.field_type} for field in table.schema]
+
+    def get_all_columns(self) -> Dict[str, Dict[str, List[Dict[str, Any]]]]:
+        """Get all columns in the project."""
+        all_columns = {}
+        datasets = self.list_dataset_ids()
+        for dataset_id in datasets:
+            all_columns[dataset_id] = {}
+            tables = self.list_tables(dataset_id)
+            for table_info in tables:
+                table_id = table_info["table_id"]
+                table_type = table_info["table_type"]
+                columns = self.get_columns(table_id, dataset_id)
+                all_columns[dataset_id][table_id] = {"columns": columns, "table_type": table_type}
+        return all_columns
 
 
 class GCPLineageClient:
+    """Client to interact with the GCP Lineage API."""
+
     def __init__(self, credentials: Credentials) -> None:
+        """Initialize the GCP Lineage client with provided credentials."""
         self.client = self.__initialze(credentials=credentials)
 
     def __initialze(self, credentials: Credentials) -> LineageClient:
-        client = LineageClient(credentials=credentials)
-        return client
+        return LineageClient(credentials=credentials)
 
     def get_links(self, request: SearchLinksRequest) -> list:
+        """Search for links between entities (tables)."""
         response = self.client.search_links(request)
         return response.links
 
@@ -57,5 +91,4 @@ def get_org_id(credentials_json: dict) -> str:
     crm_service = build("cloudresourcemanager", "v1", credentials=credentials)
     project_id = credentials_json["project_id"]
     project = crm_service.projects().get(projectId=project_id).execute()
-    org_id = project["parent"]["id"]
-    return org_id
+    return project["parent"]["id"]

quollio_core/repository/qdc.py

@@ -100,3 +100,7 @@ class QDCExternalAPIClient:
             logger.error(f"Error: {re} downstream_global_id: {global_id}.")
         else:
             return res.status_code
+
+
+def initialize_qdc_client(api_url: str, client_id: str, client_secret: str) -> QDCExternalAPIClient:
+    return QDCExternalAPIClient(base_url=api_url, client_id=client_id, client_secret=client_secret)

quollio_core/snowflake.py

@@ -11,6 +11,7 @@ from quollio_core.profilers.snowflake import (
     snowflake_table_stats,
     snowflake_table_to_table_lineage,
 )
+from quollio_core.profilers.stats import get_column_stats_items
 from quollio_core.repository import dbt, qdc, snowflake
 
 logger = logging.getLogger(__name__)
@@ -22,7 +23,6 @@ def build_view(
     target_tables: str = "",
     log_level: str = "info",
 ) -> None:
-
     logger.info("Build profiler views using dbt")
     # set parameters
     dbt_client = dbt.DBTClient()
@@ -103,13 +103,19 @@ def load_stats(
     conn: snowflake.SnowflakeConnectionConfig,
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
+    stats_items: str,
 ) -> None:
-
     logger.info("Generate Snowflake stats.")
+
+    if stats_items is None:
+        raise ValueError("No stats items are not selected. Please specify any value to `stats_items` param.")
+
+    logger.info("The following values will be aggregated. {stats_items}".format(stats_items=stats_items))
     snowflake_table_stats(
         conn=conn,
         qdc_client=qdc_client,
         tenant_id=tenant_id,
+        stats_items=stats_items,
     )
 
     logger.info("Stats data is successfully loaded.")
@@ -122,7 +128,6 @@ def load_sqllineage(
     qdc_client: qdc.QDCExternalAPIClient,
     tenant_id: str,
 ) -> None:
-
     logger.info("Generate Snowflake sqllineage.")
     snowflake_table_level_sqllineage(
         conn=conn,
@@ -275,6 +280,19 @@ if __name__ == "__main__":
         required=False,
         help="Whether to ingest column lineage into QDIC or not. Default value is False",
     )
+
+    stats_items = get_column_stats_items()
+    parser.add_argument(
+        "--target_stats_items",
+        type=str,
+        nargs="*",
+        choices=stats_items,
+        default=stats_items,
+        action=env_default("SNOWFLAKE_STATS_ITEMS"),
+        required=False,
+        help="The items for statistic values.\
+              You can choose the items to be aggregated for stats. All items are selected by default.",
+    )
     args = parser.parse_args()
     set_log_level(level=args.log_level)
 
@@ -321,6 +339,7 @@ if __name__ == "__main__":
             conn=conn,
             qdc_client=qdc_client,
             tenant_id=args.tenant_id,
+            stats_items=args.target_stats_items,
         )
     if "load_sqllineage" in args.commands:
         qdc_client = qdc.QDCExternalAPIClient(

{quollio_core-0.4.11.dist-info → quollio_core-0.4.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: quollio-core
-Version: 0.4.11
+Version: 0.4.13
 Summary: Quollio Core
 Author-email: quollio-dev <qt.dev@quollio.com>
 Maintainer-email: RyoAriyama <ryo.arym@gmail.com>, tharuta <35373297+TakumiHaruta@users.noreply.github.com>
@@ -22,6 +22,7 @@ Requires-Dist: dbt-core==1.7.10
 Requires-Dist: dbt-snowflake==1.7.0
 Requires-Dist: dbt-redshift==1.7.1
 Requires-Dist: dbt-databricks==1.7.1
+Requires-Dist: db-dtypes==1.2.0
 Requires-Dist: jinja2==3.1.3
 Requires-Dist: PyYAML==6.0.1
 Requires-Dist: requests==2.31.0

{quollio_core-0.4.11.dist-info → quollio_core-0.4.13.dist-info}/RECORD

@@ -1,8 +1,8 @@
-quollio_core/__init__.py,sha256=_jIDStu07dkGu5ouTAUr-ImjBx7pmvosgMOUjUT25Pc,84
-quollio_core/bigquery.py,sha256=RguUznaY5YjROzJtXimoS8yCNH9jgGphpzd5v_JgSQM,3884
-quollio_core/bricks.py,sha256=Lehv-qsBSMNNE9BGVvidGOXJsxLSSsbNtmiEZH4lSUg,9458
-quollio_core/redshift.py,sha256=1d-mHnalB1jtiGPgzsGd3lRwLHCxaBJlUMEV2dh4f60,9882
-quollio_core/snowflake.py,sha256=G3tImWbZgMlycYuw1b5WnNBp3zWo3hyrbOX5ARLIs7A,10585
+quollio_core/__init__.py,sha256=AUePs5X9J3XSNhx1MlWVacGiCUUUbcMKTZG3Rs0jrNY,84
+quollio_core/bigquery.py,sha256=6Oq4DVGpa3X21Es_nbrsb8pK3vaxwb9Egnvq3huo95k,5894
+quollio_core/bricks.py,sha256=4M0fzxwtFCwAv2Lat9XYdLtoGp27fy-w6a3ty1dExSc,9999
+quollio_core/redshift.py,sha256=x86Fu3QJoJNGKPYbOcqUgQzzj1qNR6I3dd0R9oQClUE,10720
+quollio_core/snowflake.py,sha256=ars0S8sbEcDR74RLrsJX9VWh8fbBGgk2H7G81paCPlk,11426
 quollio_core/dbt_projects/databricks/.gitignore,sha256=1jJAyXSzJ3YUm0nx3i7wUSE4RjQMX3ad6F8O88UbtzI,29
 quollio_core/dbt_projects/databricks/README.md,sha256=ZpRQyhFAODAiS8dc1Kb_ndkul4cu4o4udN_EMa49CU4,440
 quollio_core/dbt_projects/databricks/dbt_project.yml,sha256=3sH98RNk7TnphvI3yEdXDstb92kW5BNxr-cT0tXhwzk,480
@@ -65,22 +65,23 @@ quollio_core/helper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 quollio_core/helper/core.py,sha256=wbu4FWI7YiFEttXGSuj3tMyAhtPAFlHOjDpWJGNXOHA,1202
 quollio_core/helper/env_default.py,sha256=H6gbSGUPrEDZr4YDrL49hbOpw6RntI4U82kX1q6vUnI,2148
 quollio_core/helper/log.py,sha256=flxyZZ44G79l1TaUp3OT58uCHcnE5z_pCduwoeI6IUs,645
+quollio_core/helper/log_utils.py,sha256=w1El5yafNcKgzpiMmspsAjUm3R32ACm5QNj5lNb3xsk,1392
 quollio_core/profilers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-quollio_core/profilers/bigquery.py,sha256=e1Y8cZR-LxI9mSsYb0DurQyy0eCjM_kAKLfvl4IuQLE,3262
-quollio_core/profilers/databricks.py,sha256=pFWV6ecTefOS8evbnnXTcqR1jmbLsqn1V89ISyM2uqU,7743
+quollio_core/profilers/bigquery.py,sha256=LQzDPo-fyTHPc4C-LC59Aby5cJ7m-m4THzl9HUurXm0,5641
+quollio_core/profilers/databricks.py,sha256=ik4RiR_GOeU3S7s6C6Y9SGe1D_Y_f98BDWJVlEJXL4U,7868
 quollio_core/profilers/lineage.py,sha256=4FyxIuPBrUFihqZryqTQBcfB0Z7634lKl_WwkD82vzE,6865
-quollio_core/profilers/redshift.py,sha256=6_4amsBL4QW0ZajWhS-TW3f_cjKKa6TpXClMgBC-fZo,6440
-quollio_core/profilers/snowflake.py,sha256=nitlP5pmDm2RhLGO4f_WTzkw41EmOTY2uWN1HZkCHbI,8465
+quollio_core/profilers/redshift.py,sha256=p6ONDCkhndZAOcKAwEyQ5fsi-jsQrlwHHb7LTI_m1uk,6473
+quollio_core/profilers/snowflake.py,sha256=YdrV82pjJ1BilWQvPES1pz3EmQoBOJEPc6mVlI4FDRg,8311
 quollio_core/profilers/sqllineage.py,sha256=XkF7hwDWIGNtyEP5cv2wETBgMfdQxeHolv7qPIkntSQ,5066
-quollio_core/profilers/stats.py,sha256=PG1NbbUSpc1JuEYvBzD66rd24tp0C13_Y5Y7vRjYG1c,4720
+quollio_core/profilers/stats.py,sha256=OLQrdrh0y64jo9rmzvGlDdxy_c7gMz_GnlXPJzWkBjM,7343
 quollio_core/repository/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-quollio_core/repository/bigquery.py,sha256=KMJTeF4OUxtaJt0ymoJ4tkrMKq8yLyMYaMxNvU5yd_Y,2271
+quollio_core/repository/bigquery.py,sha256=3AyGcJNYGnUyMweyc6lGm4quwrOzd-ZBS2zNnFwafII,3990
 quollio_core/repository/databricks.py,sha256=9Cgdv8qBnVaHqu3RA-IUBieAqb69moQ-KAAMVSf5Ds4,1877
 quollio_core/repository/dbt.py,sha256=cnLwJPywLi8VowVW7zfIBa9jxVwDWO7xzzNRn1vWiuw,659
-quollio_core/repository/qdc.py,sha256=qEpMF6rKdic23dPJoDYmbIcyCKDuSFqbDF2_jqmqoZw,4369
+quollio_core/repository/qdc.py,sha256=hw7L7RdX5srv_MUSxAObq3l9b3IYjzN5lopp6CgPXyY,4572
 quollio_core/repository/redshift.py,sha256=p2ouEuYcDCjx1oBhc6H1ekQsvEqHGd3bFu3PW0ngYBc,2880
 quollio_core/repository/snowflake.py,sha256=J9rHshfWdOSnjQWxwGEYPpAU2lY7Tu5UFB_BNakkAX0,1892
-quollio_core-0.4.11.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
-quollio_core-0.4.11.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-quollio_core-0.4.11.dist-info/METADATA,sha256=0tv9AKZYWKt9yPa9iaziHZyTzhj708v-Aq22MFEbYAs,6804
-quollio_core-0.4.11.dist-info/RECORD,,
+quollio_core-0.4.13.dist-info/LICENSE,sha256=V8j_M8nAz8PvAOZQocyRDX7keai8UJ9skgmnwqETmdY,34520
+quollio_core-0.4.13.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+quollio_core-0.4.13.dist-info/METADATA,sha256=fyVJbVrl739taWh8w9ndVhKI2KWgsVLLZbRmzVF9Yj8,6836
+quollio_core-0.4.13.dist-info/RECORD,,