heurist-api 0.2.0__tar.gz → 0.2.2__tar.gz

{heurist_api-0.2.0 → heurist_api-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: heurist-api
-Version: 0.2.0
+Version: 0.2.2
 Dynamic: Description
 Dynamic: Description-Content-Type
 Summary: API wrapper and CLI for Heurist database.

{heurist_api-0.2.0 → heurist_api-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "heurist-api"
-version = "0.2.0"
+version = "0.2.2"
 description = "API wrapper and CLI for Heurist database."
 keywords = [ "duckdb", "etl" ]
 dynamic = [ "readme" ]

{heurist_api-0.2.0 → heurist_api-0.2.2}/src/heurist/cli/load.py

@@ -7,8 +7,14 @@ from pathlib import Path
 import duckdb
 from heurist.api.connection import HeuristAPIConnection
 from heurist.api.credentials import CredentialHandler
+from heurist.log import log_summary
+from heurist.log.constants import VALIDATION_LOG
 from heurist.utils.constants import DEFAULT_RECORD_GROUPS
 from heurist.workflows import extract_transform_load
+from rich.columns import Columns
+from rich.console import Console, Group
+from rich.padding import Padding
+from rich.panel import Panel
 
 
 def load_command(
@@ -37,10 +43,14 @@ def load_command(
         )
 
     # Show the results of the created DuckDB database
-    with duckdb.connect(duckdb_database_connection_path) as new_conn:
-        tables = new_conn.sql("show tables;")
-        print("\nCreated the following tables")
-        print(tables)
+    with duckdb.connect(duckdb_database_connection_path, read_only=True) as new_conn:
+        tables = [t[0] for t in new_conn.sql("show tables;").fetchall()]
+        if VALIDATION_LOG.is_file():
+            with open(VALIDATION_LOG) as f:
+                log = f.readlines()
+        else:
+            log = []
+        show_summary_in_console(tables=tables, log_lines=log)
 
         # If writing to CSV files, write only tables of record types
         if outdir:
@@ -53,3 +63,15 @@ def load_command(
                     continue
                 fp = outdir.joinpath(f"{table_name}.csv")
                 new_conn.table(table_name).sort("H-ID").write_csv(str(fp))
+
+
+def show_summary_in_console(tables: list[str], log_lines: list):
+    console = Console()
+    t0 = Panel(
+        Columns(tables, equal=True, expand=True),
+        title="SQL Tables",
+        subtitle="Saved in DuckDB database file.",
+    )
+    t1, t2 = log_summary(lines=log_lines)
+    panel_group = Group(Padding(t0, 1), t1, Padding(t2, 1))
+    console.print(panel_group)

heurist_api-0.2.2/src/heurist/cli/parse_log.py

@@ -0,0 +1,27 @@
+import csv
+from pathlib import Path
+
+import click
+from heurist.log import LogDetail, yield_log_blocks
+from heurist.log.constants import VALIDATION_LOG
+
+log_detail_fieldnames = list(LogDetail.__annotations__.keys())
+
+
+@click.command()
+@click.option("-l", "--log-file", required=None, default=VALIDATION_LOG)
+@click.option("-o", "--outfile", required=None, default="invalid_records.csv")
+def cli(log_file, outfile):
+    logfile = Path(log_file)
+    if not logfile.is_file():
+        raise FileNotFoundError(log_file)
+    with open(logfile) as f, open(outfile, "w") as of:
+        writer = csv.DictWriter(of, fieldnames=log_detail_fieldnames)
+        writer.writeheader()
+        lines = f.readlines()
+        for block in yield_log_blocks(lines):
+            writer.writerow(block.__dict__)
+
+
+if __name__ == "__main__":
+    cli()

{heurist_api-0.2.0 → heurist_api-0.2.2}/src/heurist/log/__init__.py

@@ -1,2 +1,3 @@
 from .iterator import yield_log_blocks as yield_log_blocks
 from .model import LogDetail as LogDetail
+from .summary import log_summary as log_summary

heurist_api-0.2.2/src/heurist/log/constants.py

@@ -0,0 +1,3 @@
+from pathlib import Path
+
+VALIDATION_LOG = Path.cwd().joinpath("validation.log")

heurist_api-0.2.2/src/heurist/log/summary.py

@@ -0,0 +1,35 @@
+from collections import Counter
+
+from heurist.log import yield_log_blocks
+from rich.table import Table
+
+
+def log_summary(lines: list[str]) -> tuple[Table, Table]:
+    rectypes = []
+    recs = []
+    for block in yield_log_blocks(lines):
+        rectypes.append(block.recType)
+        recs.append(block.recID)
+
+    rectype_counter = Counter(rectypes)
+    rec_counter = Counter(recs)
+
+    rec_table = Table(
+        title="Most problematic records",
+        caption="Note: Invalid records are not saved in the DuckDB database.",
+    )
+    rec_table.add_column("Record ID", style="red")
+    rec_table.add_column("Number of problems")
+    for rec, count in rec_counter.most_common(10):
+        rec_table.add_row(str(rec), str(count))
+
+    type_table = Table(
+        title="Types of invalid records",
+        caption="Note: Invalid records are not saved in the DuckDB database.",
+    )
+    type_table.add_column("Record Type", style="red")
+    type_table.add_column("Number of records")
+    for rec, count in rectype_counter.items():
+        type_table.add_row(str(rec), str(count))
+
+    return type_table, rec_table

{heurist_api-0.2.0 → heurist_api-0.2.2}/src/heurist/validators/record_validator.py

@@ -1,16 +1,14 @@
 import logging
 import os
-from pathlib import Path
 
+from heurist.log.constants import VALIDATION_LOG
 from heurist.models.dynamic.annotation import PydanticField
 from heurist.models.dynamic.type import FieldType
 from heurist.validators.detail_validator import DetailValidator
 from heurist.validators.exceptions import RepeatedValueInSingularDetailType
 from pydantic import BaseModel
 
-VALIDATION_LOG = Path.cwd().joinpath("validation.log")
-
-handlers = [logging.FileHandler(filename=VALIDATION_LOG, mode="w", delay=True)]
+handlers = [logging.FileHandler(filename=VALIDATION_LOG, mode="w")]
 if os.getenv("HEURIST_STREAM_LOG") == "True":
     handlers.append(logging.StreamHandler())
 

{heurist_api-0.2.0 → heurist_api-0.2.2}/uv.lock

@@ -421,7 +421,7 @@ wheels = [
 
 [[package]]
 name = "heurist-api"
-version = "0.1.4"
+version = "0.2.2"
 source = { editable = "." }
 dependencies = [
     { name = "click" },

heurist_api-0.2.0/src/heurist/cli/parse_log.py

@@ -1,22 +0,0 @@
-import click
-import csv
-from heurist.log import yield_log_blocks, LogDetail
-
-
-log_detail_fieldnames = list(LogDetail.__annotations__.keys())
-
-
-@click.command()
-@click.argument("csvfile", type=click.Path())
-@click.option("-l", "--log-file", required=None, default="validation.log")
-def cli(csvfile, log_file):
-    with open(log_file) as f, open(csvfile, "w") as of:
-        writer = csv.DictWriter(of, fieldnames=log_detail_fieldnames)
-        writer.writeheader()
-        lines = f.readlines()
-        for block in yield_log_blocks(lines):
-            writer.writerow(block.__dict__)
-
-
-if __name__ == "__main__":
-    cli()

{heurist_api-0.2.0 → heurist_api-0.2.2}/src/heurist/log/model.py

@@ -1,5 +1,5 @@
-from dataclasses import dataclass
 import re
+from dataclasses import dataclass
 
 
 @dataclass