heurist-api 0.1.3__tar.gz → 0.2.0__tar.gz

{heurist_api-0.1.3 → heurist_api-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: heurist-api
-Version: 0.1.3
+Version: 0.2.0
 Dynamic: Description
 Dynamic: Description-Content-Type
 Summary: API wrapper and CLI for Heurist database.
@@ -451,3 +451,17 @@ Requires-Dist: python-dotenv>=1.1.0
 Requires-Dist: requests>=2.32.3
 Requires-Dist: rich>=14.0.0
 Requires-Dist: tenacity>=9.1.2
+Provides-Extra: dev
+Requires-Dist: coverage>=7.8.0; extra == 'dev'
+Requires-Dist: genbadge[coverage]>=1.1.2; extra == 'dev'
+Requires-Dist: isort>=6.0.1; extra == 'dev'
+Requires-Dist: mkdocs-gen-files>=0.5.0; extra == 'dev'
+Requires-Dist: mkdocs-literate-nav>=0.6.2; extra == 'dev'
+Requires-Dist: mkdocs-material[imaging]>=9.6.14; extra == 'dev'
+Requires-Dist: mkdocs>=1.6.1; extra == 'dev'
+Requires-Dist: mkdocstrings-python>=1.16.10; extra == 'dev'
+Requires-Dist: pre-commit>=4.2.0; extra == 'dev'
+Requires-Dist: pymdown-extensions>=10.15; extra == 'dev'
+Requires-Dist: pytest>=8.3.5; extra == 'dev'
+Requires-Dist: ruff>=0.11.10; extra == 'dev'
+Requires-Dist: uv>=0.7.5; extra == 'dev'

{heurist_api-0.1.3 → heurist_api-0.2.0}/README.md

@@ -8,11 +8,25 @@
 
 Extract, transform, and load data from your Heurist database into local formats. Great for freeing up your data analysis pipeline!
 
-## Documentation
+## Quick Start
 
-For information about installation, usage, etc., navigate to the documentation site below:
+Install (Python version +3.10) with pip.
 
-- [Documentation](https://lostma-erc.github.io/heurist-etl-pipeline/)
+```shell
+pip install heurist-api
+```
+
+Download your record types in a [DuckDB](https://duckdb.org/) database file.
+
+```shell
+heurist -d 'YOUR.DATABASE' -u 'YOUR.LOGIN' -p 'YOUR.PASSWORD' download -f 'FILE.DB'
+```
+
+## Full Documentation
+
+For detailed information about installation, usage, etc., navigate to the documentation site below:
+
+- [Documentation](https://lostma-erc.github.io/heurist-api/)
 
 ## License
 

{heurist_api-0.1.3 → heurist_api-0.2.0}/docs/index.md

@@ -1,10 +1,10 @@
-# Heurist ETL
+# Heurist API
 
 [![Python package](https://github.com/LostMa-ERC/heurist-etl-pipeline/actions/workflows/python-package.yml/badge.svg)](https://github.com/LostMa-ERC/heurist-etl-pipeline/actions/workflows/python-package.yml) [![License: CC BY-SA 4.0](https://img.shields.io/badge/License-CC_BY--SA_4.0-lightgrey.svg)](https://creativecommons.org/licenses/by-sa/4.0/)
 [![coverage](https://github.com/LostMa-ERC/heurist-etl-pipeline/raw/main/docs/assets/coverage-badge.svg)](https://github.com/LostMa-ERC/heurist-etl-pipeline/raw/main/docs/assets/coverage-badge.svg)
 [![tests](https://github.com/LostMa-ERC/heurist-etl-pipeline/raw/main/docs/assets/tests-badge.svg)](https://github.com/LostMa-ERC/heurist-etl-pipeline/raw/main/docs/assets/tests-badge.svg)
 
-This Python package Extracts, Transforms, and Loads (ETL) data from a Heurist database server into a local [DuckDB](https://duckdb.org) database file.
+This Python package provides an API wrapper for Heurist as well as a command-line interface (CLI) that Extracts, Transforms, and Loads (ETL) data from a Heurist database server into a local [DuckDB](https://duckdb.org) database file.
 
 - [Installation & configuration](usage/index.md#installation)
 - [Basic command-line usage](usage/index.md#cli-commands)
@@ -14,10 +14,7 @@ This Python package Extracts, Transforms, and Loads (ETL) data from a Heurist da
 [![Logo](https://github.com/LostMa-ERC/heurist-etl-pipeline/raw/main/docs/assets/logo-transparent-1.png)](https://github.com/LostMa-ERC/heurist-etl-pipeline/raw/main/docs/assets/logo-transparent-1.png)
 
 ```shell
-$ pip install \
---index-url https://test.pypi.org/simple/ \
---extra-index-url https://pypi.org/simple \
-heurist
+$ pip install heurist-api
 ```
 
 ## Commands

{heurist_api-0.1.3 → heurist_api-0.2.0}/docs/usage/index.md

@@ -9,7 +9,7 @@ Secondarily, you can also exploit certain modules, such as the API client, for y
 ### Requirements
 
 - Python version 3.10 or greater
-- Virtual Python environment, i.e. [`pyenv`](https://github.com/pyenv/pyenv?tab=readme-ov-file#installation).
+- A way to manage your virtual Python environment, i.e. [`pyenv`](https://github.com/pyenv/pyenv?tab=readme-ov-file#installation).
 
 ### Steps
 
@@ -17,14 +17,11 @@ Secondarily, you can also exploit certain modules, such as the API client, for y
     - Need help installing Python? Check out the [Real Python](https://realpython.com/installing-python/) blog's tutorial.
 2. Create a new virtual environment for the package. Then activate it.
     - What's the simplest way? Check out [Real Python](https://realpython.com/python-virtual-environments-a-primer/)'s thorough blog post.
-    - I recommend naming the environment `heurist`.
-3. Use `pip install` to install the `heurist` Python package.
+    - I recommend naming the environment `heurist-api`.
+3. Use `pip install` to install the `heurist-api` Python package.
 
 ```console
-$ pip install \
---index-url https://test.pypi.org/simple/ \
---extra-index-url https://pypi.org/simple \
-heurist
+$ pip install heurist-api
 ```
 
 ## Configure the CLI

{heurist_api-0.1.3 → heurist_api-0.2.0}/mkdocs.yml

@@ -1,4 +1,4 @@
-site_name: Heurist ETL
+site_name: Heurist API
 site_url: https://lostma-erc.github.io/heurist-api/
 
 theme:

{heurist_api-0.1.3 → heurist_api-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "heurist-api"
-version = "0.1.3"
+version = "0.2.0"
 description = "API wrapper and CLI for Heurist database."
 keywords = [ "duckdb", "etl" ]
 dynamic = [ "readme" ]
@@ -37,7 +37,7 @@ build-backend = "hatchling.build"
 [tool.hatch.build.targets.wheel]
     packages = ["src/heurist", "src/mock_data"]
 
-[dependency-groups]
+[project.optional-dependencies]
 dev = [
     "coverage>=7.8.0",
     "genbadge[coverage]>=1.1.2",
@@ -56,6 +56,7 @@ dev = [
 
 [project.scripts]
 heurist = "heurist.cli.__main__:cli"
+heurist-log = "heurist.cli.parse_log:cli"
 
 [tool.isort]
 profile = "black"

heurist_api-0.2.0/src/heurist/cli/parse_log.py

@@ -0,0 +1,22 @@
+import click
+import csv
+from heurist.log import yield_log_blocks, LogDetail
+
+
+log_detail_fieldnames = list(LogDetail.__annotations__.keys())
+
+
+@click.command()
+@click.argument("csvfile", type=click.Path())
+@click.option("-l", "--log-file", required=None, default="validation.log")
+def cli(csvfile, log_file):
+    with open(log_file) as f, open(csvfile, "w") as of:
+        writer = csv.DictWriter(of, fieldnames=log_detail_fieldnames)
+        writer.writeheader()
+        lines = f.readlines()
+        for block in yield_log_blocks(lines):
+            writer.writerow(block.__dict__)
+
+
+if __name__ == "__main__":
+    cli()

{heurist_api-0.1.3 → heurist_api-0.2.0}/src/heurist/database/database.py

@@ -20,6 +20,8 @@ class TransformedDatabase(HeuristDatabase):
     ) -> None:
         super().__init__(hml_xml, conn, db)
 
+        self.conn.execute("SET GLOBAL pandas_analyze_sample=100000")
+
         # Create an empty index of targeted record types' Pydantic models
         self.pydantic_models = {}
 

heurist_api-0.2.0/src/heurist/log/__init__.py

@@ -0,0 +1,2 @@
+from .iterator import yield_log_blocks as yield_log_blocks
+from .model import LogDetail as LogDetail

heurist_api-0.2.0/src/heurist/log/iterator.py

@@ -0,0 +1,16 @@
+from typing import Generator
+
+from .model import LogDetail
+
+
+def yield_log_blocks(lines: list[str]) -> Generator[LogDetail, None, None]:
+    line_iterator = iter(lines)
+    l1 = next(line_iterator, None)
+    while l1 is not None:
+        if l1 and not l1.startswith("\t"):
+            l2 = next(line_iterator)
+            l3 = next(line_iterator)
+            l4 = next(line_iterator)
+            l5 = next(line_iterator)
+            yield LogDetail.load_lines(l1, l2, l3, l4, l5)
+        l1 = next(line_iterator, None)

heurist_api-0.2.0/src/heurist/log/model.py

@@ -0,0 +1,42 @@
+from dataclasses import dataclass
+import re
+
+
+@dataclass
+class LogDetail:
+    time: str
+    level: str
+    recType: int
+    recID: int
+    rule: str
+    problem: str
+
+    @classmethod
+    def load_lines(cls, *block_lines) -> "LogDetail":
+        l1, l2, l3, l4, l5 = block_lines
+        for indented_line in [l2, l3, l4, l5]:
+            assert indented_line.startswith("\t")
+        return LogDetail(
+            time=cls.parse_time(l1),
+            level=cls.parse_level(l1),
+            recType=cls.parse_number(l2),
+            recID=cls.parse_number(l3),
+            rule=l4.removeprefix("\t").strip(),
+            problem=l5.removeprefix("\t").strip(),
+        )
+
+    @staticmethod
+    def parse_number(line) -> int:
+        parts = line.split()
+        suffix: str = parts[-1]
+        number = suffix.removesuffix("]")
+        return int(number)
+
+    @staticmethod
+    def parse_time(l1: str) -> str:
+        parts = l1.split(" - ")
+        return parts[0].strip()
+
+    @staticmethod
+    def parse_level(l1: str) -> str:
+        return re.search(r"[A-Z]+", l1).group(0).strip()

heurist_api-0.2.0/tests/unit/log_test.py

@@ -0,0 +1,44 @@
+import unittest
+from pathlib import Path
+
+from heurist.log import yield_log_blocks, LogDetail
+
+
+LOG_FILE = Path(__file__).parent.joinpath("mock_log.txt")
+
+LINES = [
+    "2025-05-26 18:50 - WARNING - ",
+    "	[rec_Type 104]",
+    "	[rec_ID 834]",
+    "	The detail 'note' is limited to a maximum of 1 values.",
+    "	Count of values = 3.",
+    "2025-05-26 18:50 - WARNING - ",
+    "	[rec_Type 104]",
+    "	[rec_ID 834]",
+    "	The detail 'contam' is limited to a maximum of 1 values.",
+    "	Count of values = 2.",
+]
+
+
+class TestLog(unittest.TestCase):
+    def assess_log_block(self, log: LogDetail):
+        self.assertEqual(log.level, "WARNING")
+        self.assertIsNotNone(log.rule)
+        self.assertIsNotNone(log.problem)
+        self.assertGreater(log.recID, 100)
+        self.assertGreater(log.recType, 99)
+
+    def test_parsed_lines(self):
+        for log in yield_log_blocks(LINES):
+            self.assess_log_block(log)
+
+    def test_log_file(self):
+        print(LOG_FILE)
+        with open(LOG_FILE) as f:
+            lines = f.readlines()
+            for log in yield_log_blocks(lines):
+                self.assess_log_block(log)
+
+
+if __name__ == "__main__":
+    unittest.main()

{heurist_api-0.1.3 → heurist_api-0.2.0}/uv.lock

@@ -421,7 +421,7 @@ wheels = [
 
 [[package]]
 name = "heurist-api"
-version = "0.1.3"
+version = "0.1.4"
 source = { editable = "." }
 dependencies = [
     { name = "click" },
@@ -438,7 +438,7 @@ dependencies = [
     { name = "tenacity" },
 ]
 
-[package.dev-dependencies]
+[package.optional-dependencies]
 dev = [
     { name = "coverage" },
     { name = "genbadge", extra = ["coverage"] },
@@ -458,35 +458,32 @@ dev = [
 [package.metadata]
 requires-dist = [
     { name = "click", specifier = ">=8.2.0" },
+    { name = "coverage", marker = "extra == 'dev'", specifier = ">=7.8.0" },
     { name = "duckdb", specifier = ">=1.2.2" },
+    { name = "genbadge", extras = ["coverage"], marker = "extra == 'dev'", specifier = ">=1.1.2" },
+    { name = "isort", marker = "extra == 'dev'", specifier = ">=6.0.1" },
     { name = "lxml", specifier = ">=5.4.0" },
+    { name = "mkdocs", marker = "extra == 'dev'", specifier = ">=1.6.1" },
+    { name = "mkdocs-gen-files", marker = "extra == 'dev'", specifier = ">=0.5.0" },
+    { name = "mkdocs-literate-nav", marker = "extra == 'dev'", specifier = ">=0.6.2" },
+    { name = "mkdocs-material", extras = ["imaging"], marker = "extra == 'dev'", specifier = ">=9.6.14" },
+    { name = "mkdocstrings-python", marker = "extra == 'dev'", specifier = ">=1.16.10" },
     { name = "pandas", specifier = ">=2.2.3" },
     { name = "polars", specifier = ">=1.29.0" },
+    { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.2.0" },
     { name = "pyarrow", specifier = ">=20.0.0" },
     { name = "pydantic", specifier = ">=2.11.4" },
     { name = "pydantic-xml", specifier = ">=2.16.0" },
+    { name = "pymdown-extensions", marker = "extra == 'dev'", specifier = ">=10.15" },
+    { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.5" },
     { name = "python-dotenv", specifier = ">=1.1.0" },
     { name = "requests", specifier = ">=2.32.3" },
     { name = "rich", specifier = ">=14.0.0" },
+    { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.11.10" },
     { name = "tenacity", specifier = ">=9.1.2" },
+    { name = "uv", marker = "extra == 'dev'", specifier = ">=0.7.5" },
 ]
-
-[package.metadata.requires-dev]
-dev = [
-    { name = "coverage", specifier = ">=7.8.0" },
-    { name = "genbadge", extras = ["coverage"], specifier = ">=1.1.2" },
-    { name = "isort", specifier = ">=6.0.1" },
-    { name = "mkdocs", specifier = ">=1.6.1" },
-    { name = "mkdocs-gen-files", specifier = ">=0.5.0" },
-    { name = "mkdocs-literate-nav", specifier = ">=0.6.2" },
-    { name = "mkdocs-material", extras = ["imaging"], specifier = ">=9.6.14" },
-    { name = "mkdocstrings-python", specifier = ">=1.16.10" },
-    { name = "pre-commit", specifier = ">=4.2.0" },
-    { name = "pymdown-extensions", specifier = ">=10.15" },
-    { name = "pytest", specifier = ">=8.3.5" },
-    { name = "ruff", specifier = ">=0.11.10" },
-    { name = "uv", specifier = ">=0.7.5" },
-]
+provides-extras = ["dev"]
 
 [[package]]
 name = "identify"