heurist-api 0.1.2__py3-none-any.whl

heurist/sql/sql_safety.py

@@ -0,0 +1,101 @@
+import re
+
+import duckdb
+
+KEYWORDS = duckdb.sql("select * from duckdb_keywords()").fetchall()
+
+
+class SafeSQLName:
+    def __init__(self) -> None:
+        self.reserved = [t[0] for t in KEYWORDS if t[1] == "reserved"]
+        self.unreserved = [t[0] for t in KEYWORDS if t[1] == "unreserved"]
+        self.column_name = [t[0] for t in KEYWORDS if t[1] == "column_name"]
+        self.type_function = [t[0] for t in KEYWORDS if t[1] == "type_function"]
+        self.all_keywords = [t[0] for t in KEYWORDS]
+
+    @classmethod
+    def remove_characters(cls, s: str) -> str:
+        """Simplify and remove undesirable characters from a string.
+
+        Examples:
+        >>> s = "Author or Creator (Person, Organization)"
+        >>> SafeSQLName.remove_characters(s)
+        'Author or Creator'
+
+        >>> s = "Status_trad_freetext"
+        >>> SafeSQLName.remove_characters(s)
+        'Status_trad_freetext'
+
+        Args:
+            s (str): Input string.
+
+        Returns:
+            str: Cleaned string.
+        """
+
+        # Remove parentheses
+        s = re.sub(r"\(.+\)", "", s)
+        # Remove non-letters
+        s = re.sub(r"\W", " ", s)
+        # Remove backslashes
+        s = re.sub(r"/", " ", s)
+        # Remove double spaces
+        s = re.sub(r"\s+", " ", s)
+        # Remove double underscores
+        s = re.sub(r"_+", "_", s)
+        # Trim underscores
+        s = s.strip()
+        return s
+
+    @classmethod
+    def to_pascal_case(cls, text: str) -> str:
+        text_string = text.replace("-", " ").replace("_", " ")
+        words = text_string.split()
+        if len(text) == 0:
+            return text
+        capitalized_words = ["".join(w[0].capitalize() + w[1:] for w in words)]
+        return "".join(capitalized_words)
+
+    def create_column_name(self, field_name: str, field_type: str) -> str:
+        """
+        Create an SQL-safe column name for the Pydantic data field.
+
+        Args:
+            field_name (str): Displayed name of the field (detail) in Heurist.
+            field_type (str): Heurist type of the field (detail).
+
+        Returns:
+            str: SQL-safe column name.
+        """
+
+        simplified_name = self.remove_characters(field_name)
+        if field_type == "resource":
+            final_name = f"{simplified_name} H-ID"
+        elif simplified_name.lower() in self.all_keywords:
+            final_name = f"{simplified_name}_COLUMN"
+        else:
+            final_name = simplified_name
+        return final_name
+
+    def create_table_name(self, record_name: str) -> str:
+        """
+        Create SQL-safe table name for the record's data model.
+
+        Examples:
+        >>> heurist_name = "Sequence"
+        >>> SafeSQLName().create_table_name(heurist_name)
+        'SequenceTable'
+
+        Args:
+            record_name (str): Name of the Heurist record type.
+
+        Returns:
+            str: SQL-safe name for the record type's table.
+        """
+
+        camel_case_name = self.to_pascal_case(record_name)
+        if camel_case_name.lower() in self.all_keywords:
+            final_name = f"{camel_case_name}Table"
+        else:
+            final_name = camel_case_name
+        return final_name

heurist/utils/constants.py

@@ -0,0 +1 @@
+DEFAULT_RECORD_GROUPS = ("My record types",)

heurist/utils/rel_to_dict_array.py

@@ -0,0 +1,8 @@
+import duckdb
+
+
+def rel_to_dict_array(rel: duckdb.DuckDBPyRelation) -> list[dict]:
+    output = []
+    for row in rel.fetchall():
+        output.append({k: v for k, v in zip(rel.columns, row)})
+    return output

heurist/validators/__init__.py

@@ -0,0 +1,3 @@
+from heurist.validators.parse_heurist_date import parse_heurist_date
+
+parse_heurist_date

heurist/validators/detail_validator.py

@@ -0,0 +1,142 @@
+"""Class for converting a record's detail before the Pydantic model validation."""
+
+from heurist.models.dynamic.date import TemporalObject
+from heurist.models.dynamic.type import FieldType
+
+
+class DetailValidator:
+    """
+    In Heurist, a record's "detail" is what is more commonly known as an attribute, \
+        dimension, or a data field.
+
+    This class features methods to extract the key value from Heurist's JSON \
+        formatting for all data types in Heurist's system.
+    """
+
+    direct_values = ["freetext", "blocktext", "integer", "boolean", "float"]
+
+    @classmethod
+    def validate_file(cls, detail: dict) -> str:
+        """
+        Extract the value of a file field.
+
+        Args:
+            detail (dict): Record's detail.
+
+        Returns:
+            str: Value of record's detail.
+        """
+
+        return detail.get("value", {}).get("file", {}).get("ulf_ExternalFileReference")
+
+    @classmethod
+    def validate_enum(cls, detail: dict) -> str:
+        """
+        Extract the value of an enum field.
+
+        Args:
+            detail (dict): Record's detail.
+
+        Returns:
+            str: Value of record's detail.
+        """
+
+        return detail["termLabel"]
+
+    @classmethod
+    def validate_geo(cls, detail: dict) -> str:
+        """
+        Extract the value of a geo field.
+
+        Examples:
+            >>> from mock_data.geo.single import DETAIL_POINT
+            >>> DetailValidator.convert(DETAIL_POINT)
+            'POINT(2.19726563 48.57478991)'
+
+        Args:
+            detail (dict): Record's detail.
+
+        Returns:
+            str: Value of record's detail.
+        """
+
+        geo = detail["value"]["geo"]
+        if geo["type"] == "p" or geo["type"] == "pl":
+            return geo["wkt"]
+
+    @classmethod
+    def validate_date(cls, detail: dict) -> dict:
+        """
+        Build the variable date value into a structured dictionary.
+
+        Examples:
+            >>> # Test temporal object
+            >>> from mock_data.date.compound_single import DETAIL
+            >>> value = DetailValidator.convert(DETAIL)
+            >>> value['start']['earliest']
+            datetime.datetime(1180, 1, 1, 0, 0)
+
+            >>> # Test direct date value
+            >>> from mock_data.date.simple_single import DETAIL
+            >>> value = DetailValidator.convert(DETAIL)
+            >>> value['value']
+            datetime.datetime(2024, 3, 19, 0, 0)
+
+        Args:
+            detail (dict): Record's detail.
+
+        Returns:
+            dict: Structured metadata for a Heurist date object.
+        """
+
+        if isinstance(detail.get("value"), dict):
+            model = TemporalObject.model_validate(detail["value"])
+        else:
+            model = TemporalObject.model_validate(detail)
+        return model.model_dump(by_alias=True)
+
+    @classmethod
+    def validate_resource(cls, detail: dict) -> int:
+        """
+        Extract the value of a resource (foreign key) field.
+
+        Args:
+            detail (dict): Record's detail.
+
+        Returns:
+            int: Heurist ID of the referenced record.
+        """
+
+        return int(detail["value"]["id"])
+
+    @classmethod
+    def convert(cls, detail: dict) -> str | int | list | dict | None:
+        """
+        Based on the data type, convert the record's nested detail to a flat value.
+
+        Args:
+            detail (dict): One of the record's details (data fields).
+
+        Returns:
+            str | int | list | dict | None: Flattened value of the data field.
+        """
+
+        fieldtype = FieldType.from_detail(detail)
+
+        if any(ft in fieldtype for ft in cls.direct_values):
+            return detail["value"]
+
+        elif fieldtype == "date":
+            return cls.validate_date(detail)
+
+        elif fieldtype == "enum":
+            return cls.validate_enum(detail)
+
+        elif fieldtype == "file":
+            return cls.validate_file(detail)
+
+        elif fieldtype == "geo":
+            return cls.validate_geo(detail)
+
+        elif fieldtype == "resource":
+            return cls.validate_resource(detail)

heurist/validators/exceptions.py

@@ -0,0 +1,34 @@
+"""
+Exceptions for classes that convert / transform Heurist data.
+"""
+
+
+class RepeatedValueInSingularDetailType(Exception):
+    """The detail type is limited to a maximum of 1 values
+    but the record has more than 1 value for this detail."""
+
+    description = """
+\t[rec_Type {typeID}]
+\t[rec_ID {recID}]
+\tThe detail '{fieldName}' is limited to a maximum of 1 values.
+\tCount of values = {valueCount}."""
+
+    def __init__(self, type_id: int, record_id: int, field_name: str, value_count: int):
+        self.message = self.description.format(
+            typeID=type_id,
+            recID=record_id,
+            fieldName=field_name,
+            valueCount=value_count,
+        )
+        super().__init__(self.message)
+
+
+class DateNotEnteredAsDateObject(Exception):
+    """The date field was not entered as a constructed Heurist date object."""
+
+    description = """The date field was not entered as a compound Heurist date \
+object.\n\tEntered value = {}"""
+
+    def __init__(self, value: int | str | float):
+        self.message = self.description.format(value)
+        super().__init__(self.message)

heurist/validators/parse_heurist_date.py

@@ -0,0 +1,71 @@
+from datetime import datetime
+
+import dateutil.parser
+import dateutil.relativedelta
+
+
+def parse_heurist_date(repr: str | int | float | None) -> datetime | None:
+    """
+    Convert Heurist's partial date representations to an ISO string format.
+
+    Examples:
+        >>> # Test a string representation of a date
+        >>> v = "2024-03-19"
+        >>> parse_heurist_date(v)
+        datetime.datetime(2024, 3, 19, 0, 0)
+
+        >>> # Test an integer representation of a year, i.e. circa 1188
+        >>> v = 1188
+        >>> parse_heurist_date(v)
+        datetime.datetime(1188, 1, 1, 0, 0)
+
+        >>> # Test a float representation of a date
+        >>> v = 1250.1231
+        >>> parse_heurist_date(v)
+        datetime.datetime(1250, 12, 31, 0, 0)
+
+    Args:
+        repr (str | int | float): Heurist representation \
+            of a date.
+
+    Returns:
+        datetime | None: Parsed date.
+    """
+
+    if not repr:
+        return
+
+    # Affirm Heurist's representation of the date is a Python string
+    repr = str(repr)
+
+    # If the Heurist representation is a year, change it to the start of
+    # the year.
+    if len(repr) == 4:
+        iso_str = f"{repr}-01-01"
+        return dateutil.parser.parse(iso_str)
+
+    # If the Heurist representation is a float, parse the month and day
+    # shown after the decimal.
+    elif "." in repr:
+        splits = repr.split(".")
+        year, smaller_than_year = splits[0], splits[1]
+        if len(smaller_than_year) == 2:
+            iso_str = f"{year}-{smaller_than_year}-01"
+        elif len(smaller_than_year) == 4:
+            iso_str = f"{year}-{smaller_than_year[:2]}-{smaller_than_year[2:]}"
+        else:
+            raise ValueError(repr)
+        return dateutil.parser.parse(iso_str)
+
+    # If the Heurist representation is a year and month, add the day
+    # (first of the month)
+    parts = repr.split("-")
+    if len(parts) == 2:
+        iso_str = f"{repr}-01"
+        return dateutil.parser.parser(iso_str)
+
+    # If no other conditions have been met, the representation is already in
+    # ISO format YYYY-MM-DD.
+    else:
+        iso_str = repr
+        return dateutil.parser.parse(iso_str)

heurist/validators/record_validator.py

@@ -0,0 +1,156 @@
+import logging
+import os
+from pathlib import Path
+
+from heurist.models.dynamic.annotation import PydanticField
+from heurist.models.dynamic.type import FieldType
+from heurist.validators.detail_validator import DetailValidator
+from heurist.validators.exceptions import RepeatedValueInSingularDetailType
+from pydantic import BaseModel
+
+VALIDATION_LOG = Path.cwd().joinpath("validation.log")
+
+handlers = [logging.FileHandler(filename=VALIDATION_LOG, mode="w", delay=True)]
+if os.getenv("HEURIST_STREAM_LOG") == "True":
+    handlers.append(logging.StreamHandler())
+
+logging.basicConfig(
+    encoding="utf-8",
+    format="{asctime} - {levelname} - {message}",
+    style="{",
+    datefmt="%Y-%m-%d %H:%M",
+    handlers=handlers,
+)
+
+
+def list_plural_fields(pydantic_model: BaseModel) -> list:
+    return [
+        v.description
+        for v in pydantic_model.model_fields.values()
+        if repr(v.annotation).startswith("list")
+    ]
+
+
+class RecordValidator:
+    def __init__(
+        self, pydantic_model: BaseModel, records: list[dict], rty_ID: int
+    ) -> None:
+        self.pydantic_model = pydantic_model
+        self._rty_ID = rty_ID
+        self._records = records
+        self._index = 0
+        self._plural_fields = list_plural_fields(pydantic_model=self.pydantic_model)
+
+    def is_plural(self, dty_ID: int) -> bool:
+        if dty_ID in self._plural_fields:
+            return True
+
+    def __iter__(self):
+        return self
+
+    def __next__(self) -> BaseModel:
+        if self._index < len(self._records):
+            record = self._records[self._index]
+            self._index += 1
+            # If the record isn't of the record type for this model, skip it.
+            if record["rec_RecTypeID"] != self._rty_ID:
+                pass
+            # Otherwise, process the record's details into key-value pairs that
+            # will be loaded into the Pydantic model.
+            kwargs = self.flatten_details_to_dynamic_pydantic_fields(record)
+            # Return a validated Pydantic model.
+            return self.pydantic_model.model_validate(kwargs)
+        else:
+            raise StopIteration
+
+    @classmethod
+    def aggregate_details_by_type(cls, details: list[dict]) -> dict:
+        # Set up an index for all the types of details in this record's
+        # sequence of details.
+        index = {d["dty_ID"]: [] for d in details}
+        # According to its type, add each detail to its respective list in the index.
+        [index[d["dty_ID"]].append(d) for d in details]
+        # Return the index of aggregated details.
+        return index
+
+    def flatten_details_to_dynamic_pydantic_fields(self, record: dict) -> dict:
+        detail_type_index = self.aggregate_details_by_type(record["details"])
+        # To the list of key-value pairs, add the record's H-ID and its type ID
+        record_id = record["rec_ID"]
+        kwargs = {
+            "rec_ID": record_id,
+            "rec_RecTypeID": record["rec_RecTypeID"],
+        }
+        for dty_ID, details in detail_type_index.items():
+            # Determine if this detail type is allowed to have multiple values.
+            repeats = self.is_plural(dty_ID=dty_ID)
+
+            # If this detail is not supposed to be repeateable but Heurist allowed more
+            # than 1 value to be saved in the field, raise an error.
+            if not repeats and len(details) > 1:
+                warning = RepeatedValueInSingularDetailType(
+                    type_id=record["rec_RecTypeID"],
+                    record_id=record_id,
+                    field_name=details[0]["fieldName"],
+                    value_count=len(details),
+                )
+                logging.warning(warning)
+                continue
+
+            # Get the validation alias for this kwarg's key
+            key = PydanticField._get_validation_alias(dty_ID=dty_ID)
+
+            # Convert the detail's metadata to a flat value.
+            values = []
+            for detail in details:
+                v = DetailValidator.convert(detail=detail)
+                values.append(v)
+
+            # Check the number of validated metadata against what is permissible for
+            # this detail type according to the Heurist schema.
+            value = self.validate_for_repeatable_values(repeats=repeats, values=values)
+
+            # If the validation failed, do not add this detail type to the set of
+            # kwargs for the Pydantic model. Let the model's default value be used
+            # for this missing / invalid metadata.
+            if not value:
+                continue
+
+            # Add this detail type's alias and validated value(s) to the set of kwargs.
+            kwargs.update({key: value})
+
+            # If the detail is a Term, add an additional field for the foreign key.
+            if FieldType.from_detail(details[0]) == "enum":
+                # To this detail type's validation alias, which is associated with the
+                # term's label, append a suffix to distinguish it as a supplemental
+                # field to hold the foreign key.
+                key += PydanticField.trm_validation_alias_suffix
+                # Into a list, extract each detail's foreign key, which is in "value."
+                values = []
+                for detail in details:
+                    values.append(detail["value"])
+
+                value = self.validate_for_repeatable_values(
+                    repeats=repeats, values=values
+                )
+
+                # The previous if-condition should have already confirmed that this
+                # group of deatils are valid. Therefore, they can be added directly
+                # to the kwargs.
+                kwargs.update({key: value})
+
+        # Return the flat key-value pairs for the Pydantic model's fields.
+        return kwargs
+
+    @classmethod
+    def validate_for_repeatable_values(
+        cls, repeats: bool, values: list
+    ) -> list | dict | None:
+        # If the detail type is not repeatable, extract the first dictionary.
+        if not repeats and len(values) > 0:
+            return values[0]
+        # If the detail type is repeatable, send the list of values, which can
+        # be an empty list--as this should be the default value for this field
+        # annotation.
+        elif repeats:
+            return values

heurist/workflows/__init__.py

@@ -0,0 +1,3 @@
+from heurist.workflows.etl import extract_transform_load
+
+extract_transform_load

heurist/workflows/etl.py

@@ -0,0 +1,66 @@
+import duckdb
+from heurist.api.connection import HeuristAPIConnection
+from heurist.database import TransformedDatabase
+from heurist.utils.constants import DEFAULT_RECORD_GROUPS
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    Progress,
+    SpinnerColumn,
+    TextColumn,
+    TimeElapsedColumn,
+)
+
+
+def extract_transform_load(
+    client: HeuristAPIConnection,
+    duckdb_connection: duckdb.DuckDBPyConnection,
+    user: tuple = (),
+    record_group_names: tuple = DEFAULT_RECORD_GROUPS,
+) -> None:
+    """
+    Workflow for (1) extracting, transforming, and loading the Heurist database \
+        architecture into a DuckDB database and (2) extracting, transforming, \
+        and loading record types' records into the created DuckDB database.
+
+    Args:
+        client (HeuristAPIConnection): Context of a Heurist API connection.
+        duckdb_connection (duckdb.DuckDBPyConnection): Connection to a DuckDB database.
+        user (tuple): IDs (integers) of targeted users.
+        record_group_names (tuple): Names of the record group types. Must include at \
+            least 1. Defaults to ("My record types").
+
+    Returns:
+        duckdb.DuckDBPyConnection: Open connection to the created DuckDB database.
+    """
+
+    # Export the Heurist database's structure
+    with Progress(
+        TextColumn("{task.description}"), SpinnerColumn(), TimeElapsedColumn()
+    ) as p:
+        _ = p.add_task("Get DB Structure")
+        xml = client.get_structure()
+
+    # Export individual record sets and insert into the DuckDB database
+    with (
+        Progress(
+            TextColumn("{task.description}"),
+            BarColumn(),
+            MofNCompleteColumn(),
+            TimeElapsedColumn(),
+        ) as p,
+    ):
+        database = TransformedDatabase(
+            conn=duckdb_connection,
+            hml_xml=xml,
+            record_type_groups=record_group_names,
+        )
+        t = p.add_task(
+            "Get Records",
+            total=len(database.pydantic_models.keys()),
+        )
+        for record_type in database.pydantic_models.values():
+            rty_ID = record_type.rty_ID
+            records = client.get_records(rty_ID, users=user)
+            p.advance(t)
+            database.insert_records(record_type_id=rty_ID, records=records)