heurist-api 0.1.4__py3-none-any.whl → 0.2.1__py3-none-any.whl

heurist/cli/load.py

@@ -7,8 +7,14 @@ from pathlib import Path
 import duckdb
 from heurist.api.connection import HeuristAPIConnection
 from heurist.api.credentials import CredentialHandler
+from heurist.log import log_summary
+from heurist.log.constants import VALIDATION_LOG
 from heurist.utils.constants import DEFAULT_RECORD_GROUPS
 from heurist.workflows import extract_transform_load
+from rich.columns import Columns
+from rich.console import Console, Group
+from rich.padding import Padding
+from rich.panel import Panel
 
 
 def load_command(
@@ -37,10 +43,11 @@ def load_command(
         )
 
     # Show the results of the created DuckDB database
-    with duckdb.connect(duckdb_database_connection_path) as new_conn:
-        tables = new_conn.sql("show tables;")
-        print("\nCreated the following tables")
-        print(tables)
+    with duckdb.connect(duckdb_database_connection_path, read_only=True) as new_conn:
+        tables = [t[0] for t in new_conn.sql("show tables;").fetchall()]
+        with open(VALIDATION_LOG) as f:
+            log = f.readlines()
+        show_summary_in_console(tables=tables, log_lines=log)
 
         # If writing to CSV files, write only tables of record types
         if outdir:
@@ -53,3 +60,15 @@ def load_command(
                     continue
                 fp = outdir.joinpath(f"{table_name}.csv")
                 new_conn.table(table_name).sort("H-ID").write_csv(str(fp))
+
+
+def show_summary_in_console(tables: list[str], log_lines: list[str]):
+    console = Console()
+    t0 = Panel(
+        Columns(tables, equal=True, expand=True),
+        title="SQL Tables",
+        subtitle="Saved in DuckDB database file.",
+    )
+    t1, t2 = log_summary(lines=log_lines)
+    panel_group = Group(Padding(t0, 1), t1, Padding(t2, 1))
+    console.print(panel_group)

heurist/cli/parse_log.py

@@ -0,0 +1,27 @@
+import csv
+from pathlib import Path
+
+import click
+from heurist.log import LogDetail, yield_log_blocks
+from heurist.log.constants import VALIDATION_LOG
+
+log_detail_fieldnames = list(LogDetail.__annotations__.keys())
+
+
+@click.command()
+@click.option("-l", "--log-file", required=None, default=VALIDATION_LOG)
+@click.option("-o", "--outfile", required=None, default="invalid_records.csv")
+def cli(log_file, outfile):
+    logfile = Path(log_file)
+    if not logfile.is_file():
+        raise FileNotFoundError(log_file)
+    with open(logfile) as f, open(outfile, "w") as of:
+        writer = csv.DictWriter(of, fieldnames=log_detail_fieldnames)
+        writer.writeheader()
+        lines = f.readlines()
+        for block in yield_log_blocks(lines):
+            writer.writerow(block.__dict__)
+
+
+if __name__ == "__main__":
+    cli()

heurist/log/__init__.py

@@ -0,0 +1,3 @@
+from .iterator import yield_log_blocks as yield_log_blocks
+from .model import LogDetail as LogDetail
+from .summary import log_summary as log_summary

heurist/log/constants.py

@@ -0,0 +1,4 @@
+from pathlib import Path
+
+VALIDATION_LOG = Path.cwd().joinpath("validation.log")
+VALIDATION_LOG.touch(exist_ok=True)

heurist/log/iterator.py

@@ -0,0 +1,16 @@
+from typing import Generator
+
+from .model import LogDetail
+
+
+def yield_log_blocks(lines: list[str]) -> Generator[LogDetail, None, None]:
+    line_iterator = iter(lines)
+    l1 = next(line_iterator, None)
+    while l1 is not None:
+        if l1 and not l1.startswith("\t"):
+            l2 = next(line_iterator)
+            l3 = next(line_iterator)
+            l4 = next(line_iterator)
+            l5 = next(line_iterator)
+            yield LogDetail.load_lines(l1, l2, l3, l4, l5)
+        l1 = next(line_iterator, None)

heurist/log/model.py

@@ -0,0 +1,42 @@
+import re
+from dataclasses import dataclass
+
+
+@dataclass
+class LogDetail:
+    time: str
+    level: str
+    recType: int
+    recID: int
+    rule: str
+    problem: str
+
+    @classmethod
+    def load_lines(cls, *block_lines) -> "LogDetail":
+        l1, l2, l3, l4, l5 = block_lines
+        for indented_line in [l2, l3, l4, l5]:
+            assert indented_line.startswith("\t")
+        return LogDetail(
+            time=cls.parse_time(l1),
+            level=cls.parse_level(l1),
+            recType=cls.parse_number(l2),
+            recID=cls.parse_number(l3),
+            rule=l4.removeprefix("\t").strip(),
+            problem=l5.removeprefix("\t").strip(),
+        )
+
+    @staticmethod
+    def parse_number(line) -> int:
+        parts = line.split()
+        suffix: str = parts[-1]
+        number = suffix.removesuffix("]")
+        return int(number)
+
+    @staticmethod
+    def parse_time(l1: str) -> str:
+        parts = l1.split(" - ")
+        return parts[0].strip()
+
+    @staticmethod
+    def parse_level(l1: str) -> str:
+        return re.search(r"[A-Z]+", l1).group(0).strip()

heurist/log/summary.py

@@ -0,0 +1,35 @@
+from collections import Counter
+
+from heurist.log import yield_log_blocks
+from rich.table import Table
+
+
+def log_summary(lines: list[str]) -> tuple[Table, Table]:
+    rectypes = []
+    recs = []
+    for block in yield_log_blocks(lines):
+        rectypes.append(block.recType)
+        recs.append(block.recID)
+
+    rectype_counter = Counter(rectypes)
+    rec_counter = Counter(recs)
+
+    rec_table = Table(
+        title="Most problematic records",
+        caption="Note: Invalid records are not saved in the DuckDB database.",
+    )
+    rec_table.add_column("Record ID", style="red")
+    rec_table.add_column("Number of problems")
+    for rec, count in rec_counter.most_common(10):
+        rec_table.add_row(str(rec), str(count))
+
+    type_table = Table(
+        title="Types of invalid records",
+        caption="Note: Invalid records are not saved in the DuckDB database.",
+    )
+    type_table.add_column("Record Type", style="red")
+    type_table.add_column("Number of records")
+    for rec, count in rectype_counter.items():
+        type_table.add_row(str(rec), str(count))
+
+    return type_table, rec_table

heurist/validators/record_validator.py

@@ -1,15 +1,13 @@
 import logging
 import os
-from pathlib import Path
 
+from heurist.log.constants import VALIDATION_LOG
 from heurist.models.dynamic.annotation import PydanticField
 from heurist.models.dynamic.type import FieldType
 from heurist.validators.detail_validator import DetailValidator
 from heurist.validators.exceptions import RepeatedValueInSingularDetailType
 from pydantic import BaseModel
 
-VALIDATION_LOG = Path.cwd().joinpath("validation.log")
-
 handlers = [logging.FileHandler(filename=VALIDATION_LOG, mode="w", delay=True)]
 if os.getenv("HEURIST_STREAM_LOG") == "True":
     handlers.append(logging.StreamHandler())

{heurist_api-0.1.4.dist-info → heurist_api-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: heurist-api
-Version: 0.1.4
+Version: 0.2.1
 Dynamic: Description
 Dynamic: Description-Content-Type
 Summary: API wrapper and CLI for Heurist database.

{heurist_api-0.1.4.dist-info → heurist_api-0.2.1.dist-info}/RECORD

@@ -9,12 +9,18 @@ heurist/api/url_builder.py,sha256=mT1hgZ-T38EPyHCSLfjFecbZVLsiPi18jemDuuMj21I,49
 heurist/api/utils.py,sha256=DT-BrdF7O2lmFSduRYOecx302dAlLWQTYgr2V9JoDYI,643
 heurist/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 heurist/cli/__main__.py,sha256=QG2gJG7zD3YIDRqbIp2MzTqPxqGLZ77RHCRYZ58SSjg,5975
-heurist/cli/load.py,sha256=kU8nTSJMl6ynz6FguhE_xdqJwQFijjsRerQGRtWil5s,1903
+heurist/cli/load.py,sha256=q6QwZRtIQVoII1HHLW0iGklfXJlHEVxrDwb8UeX7la4,2610
+heurist/cli/parse_log.py,sha256=cZU9mpLbx5p-bdCwbII27T-ugTpecTgCFiBoYJip5f4,828
 heurist/cli/records.py,sha256=yKK3XIZrU1szo6X-HmGjZpD0IKk3a3rd3Kkyrg5AYGE,1345
 heurist/cli/schema.py,sha256=Dg8NcSdI7Xb6r-dkUC_razXlZNtxSG0x7awsr1m8Mcw,2766
 heurist/database/__init__.py,sha256=JvQCGCi84AXUSIPCu4cTOqT_yZGDLmZFAUdsHDLbpKI,79
 heurist/database/basedb.py,sha256=0MKfQOKTBDCc6qcl8BGl5OIpSA7SYgmfcKXCBVWBPes,4200
 heurist/database/database.py,sha256=rJPhGA4158eoIwe13QsSGZ8929dkRxMTesRRlMrn1tI,4057
+heurist/log/__init__.py,sha256=ubZl-2yMlvUYJwJtnk6C8AjmnU-9mbp96aHL1NSzbSg,149
+heurist/log/constants.py,sha256=5FaEnyXCEt1gga0pgwq4sfz_dX58BK1NixXQ2CsihNo,117
+heurist/log/iterator.py,sha256=nmpW3QQfwToADD4zffAm5iFyIUMFT56LRORdThYRoqE,519
+heurist/log/model.py,sha256=rOey1V6Vd_ehm8FTXxfi7Q4A4Mjq0V9ITl_GlNdB0y0,1085
+heurist/log/summary.py,sha256=n-skKRiFSPd3DieZMveJLCP96plVdgjW9wgBR0H_2BI,1113
 heurist/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 heurist/models/dynamic/__init__.py,sha256=dFUNGMugaOAkec6uhDxkUVrmGkR5dr1fTd8X_TprRyU,77
 heurist/models/dynamic/annotation.py,sha256=FOkldKT5u2XWBk7DxXbrgRf8X_G3N5FEfr-rOTLq-Eg,4749
@@ -48,7 +54,7 @@ heurist/validators/__init__.py,sha256=Im5z_Yg2VdKSZRPUOPSF5AFZ8AWgnyBZNKhy1o69lC
 heurist/validators/detail_validator.py,sha256=NVItrZ4Ysmgqab-B24hIeimGqpiaiqFiono5qwW20ko,4070
 heurist/validators/exceptions.py,sha256=tgoMn6R2CExT6mFg2qe4vRxP2D61cNdgEqc7YUQCFSE,1108
 heurist/validators/parse_heurist_date.py,sha256=S-NzuLZJwDrxRJkAd-fXpOlENlkc8HXKRILWCODmtIU,2161
-heurist/validators/record_validator.py,sha256=bzSurq0aLN9LLWcAYSE3X63hT00OZUIqyaNCLTgtE08,6297
+heurist/validators/record_validator.py,sha256=37xFdBwfgQY21jCtcm45_8AOu6DwC27AGC4ompykJUE,6265
 heurist/workflows/__init__.py,sha256=aYN0UE_7rD9h0h74DMXkUTIDrxA0WcsWLEXWK123Gzc,81
 heurist/workflows/etl.py,sha256=IA97B1NOg_2btT_pynY64RYvbL4JraAy6wgIKJlNljM,2286
 mock_data/__init__.py,sha256=TGe3NjGTXVRQbakVGEWbDDzIkFQ72fTZhTcAoIH3DGQ,488
@@ -73,8 +79,8 @@ mock_data/geo/single.py,sha256=gbk_gOLfVlJuU3MhjY2Lu14bs9-FbZmNtevpiw9jArk,79867
 mock_data/resource/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 mock_data/resource/repeated.py,sha256=Nf3nIL596pLVbKo20sQm5UITwBzqf-IHFzE_m2UCLbw,788
 mock_data/resource/single.py,sha256=SXVri1MM8UaJw7GejMEJ6seNEnMksdw_WKiOxQOdVFs,411
-heurist_api-0.1.4.dist-info/METADATA,sha256=p9B4L6XZDvMIzNYRrNEFo3qbNS0aYTGoxa7iX7jpWf4,25177
-heurist_api-0.1.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-heurist_api-0.1.4.dist-info/entry_points.txt,sha256=_TzE7JV8uXjiG-Fo9WFNnQtmvrlCYiAZgu8ZoijfeF8,53
-heurist_api-0.1.4.dist-info/licenses/LICENSE,sha256=I-54yLrknPCOovDISUXGa5h-vkUgiB-1Gz2tT7Q9B8I,20137
-heurist_api-0.1.4.dist-info/RECORD,,
+heurist_api-0.2.1.dist-info/METADATA,sha256=INf5LbhdDjblw82g0TfLpwSC9E2oIQ4pRPjeme25sb0,25177
+heurist_api-0.2.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+heurist_api-0.2.1.dist-info/entry_points.txt,sha256=37KBvqofapLaKDPFrcYxv_rfUM9H08Mbe6mxNdd_Xno,93
+heurist_api-0.2.1.dist-info/licenses/LICENSE,sha256=I-54yLrknPCOovDISUXGa5h-vkUgiB-1Gz2tT7Q9B8I,20137
+heurist_api-0.2.1.dist-info/RECORD,,

{heurist_api-0.1.4.dist-info → heurist_api-0.2.1.dist-info}/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
 heurist = heurist.cli.__main__:cli
+heurist-log = heurist.cli.parse_log:cli