graflo 1.1.0__py3-none-any.whl

graflo/util/merge.py

@@ -0,0 +1,148 @@
+"""Document merging and discrimination utilities.
+
+This module provides functions for merging and discriminating between documents
+based on various criteria. It supports merging documents with common keys,
+discriminating based on specific values, and handling different document structures.
+
+Key Functions:
+    - discriminate_by_key: Filter documents based on index fields and key presence
+    - merge_doc_basis: Merge documents based on common index keys
+
+"""
+
+from graflo.architecture.onto import VertexRep
+
+
+def discriminate_by_key(items, indexes, discriminant_key, fast=False):
+    """Filter documents based on index fields and key presence.
+
+    This function filters a list of documents based on the presence of index fields
+    and a specific key. It can operate in fast mode to return after finding the
+    first match.
+
+    Args:
+        items: List of documents (dictionaries) to filter
+        indexes: List of index field names to check for presence
+        discriminant_key: Key to check for presence
+        fast: Whether to return after first match (default: False)
+
+    Returns:
+        list[dict]: Filtered list of documents
+    """
+    # pick items that have any of index field present
+    _items = [item for item in items if any(k in item for k in indexes)]
+
+    if discriminant_key is not None:
+        result = []
+        for item in _items:
+            if discriminant_key in item:
+                result += [item]
+                if fast:
+                    break
+        return result
+    return _items
+
+
+def merge_doc_basis(
+    docs: list[dict],
+    index_keys: tuple[str, ...],
+    discriminant_key=None,
+) -> list[dict]:
+    """Merge documents based on common index keys.
+
+    This function merges documents that share common index key-value combinations.
+    Documents without index keys are merged with the first relevant document that
+    has the discriminant key.
+
+    Note:
+        Currently works best with two groups of documents: those with and without
+        the discriminant key. Future versions will support multiple discriminant
+        value groups.
+
+    Args:
+        docs: List of documents to merge
+        index_keys: Tuple of key names to use for merging
+        discriminant_key: Optional key to use for merging documents without index keys
+
+    Returns:
+        list[dict]: Merged documents
+    """
+    docs_tuplezied = [
+        tuple(sorted((k, v) for k, v in item.items() if k in index_keys))
+        for item in docs
+    ]
+
+    # pick bearing docs : those that differ by index_keys
+    bearing_docs: dict[tuple, dict] = {q: dict() for q in set(docs_tuplezied)}
+
+    # merge docs with respect to unique index key-value combinations
+    for doc, doc_tuple in zip(docs, docs_tuplezied):
+        bearing_docs[doc_tuple].update(doc)
+
+    # merge docs without any index keys onto the first relevant doc
+    if () in docs_tuplezied:
+        relevant_docs = discriminate_by_key(
+            docs, index_keys, discriminant_key, fast=True
+        )
+        if relevant_docs:
+            tuple_ix = tuple(
+                sorted((k, v) for k, v in relevant_docs[0].items() if k in index_keys)
+            )
+            bearing_docs[tuple_ix].update(bearing_docs.pop(()))
+
+    return list(bearing_docs.values())
+
+
+def merge_doc_basis_closest_preceding(
+    docs: list[VertexRep],
+    index_keys: tuple[str, ...],
+) -> list[VertexRep]:
+    """Merge VertexRep documents based on index_keys.
+
+    Leading non-ID VertexReps are merged into the first ID VertexRep.
+    Remaining non-ID VertexReps are merged into the closest preceding ID VertexRep.
+    The merge is performed on the `vertex` attribute, and `ctx` dicts are merged among merged VertexReps.
+
+    Args:
+        docs: List of VertexRep to merge
+        index_keys: Tuple of key names to use for merging
+
+    Returns:
+        list[VertexRep]: Merged VertexReps
+    """
+    merged_docs: list[VertexRep] = []
+    pending_non_ids: list[VertexRep] = []
+
+    def merge_vertex_ctx(target: VertexRep, sources: list[VertexRep]):
+        # Merge vertex dicts
+        for src in sources:
+            target.vertex.update(src.vertex)
+            target.ctx.update(src.ctx)
+        return target
+
+    for doc in docs:
+        if any(k in doc.vertex for k in index_keys):
+            # This is an ID VertexRep
+            # First, handle any accumulated non-ID VertexReps
+            if pending_non_ids:
+                if not merged_docs:
+                    # No previous ID doc, create new one with accumulated non-IDs
+                    merged_doc = VertexRep(vertex={}, ctx={})
+                    merged_doc = merge_vertex_ctx(merged_doc, pending_non_ids)
+                    merged_docs.append(merged_doc)
+                else:
+                    # Merge accumulated non-IDs into the last ID doc
+                    merged_docs[-1] = merge_vertex_ctx(merged_docs[-1], pending_non_ids)
+                pending_non_ids.clear()
+
+            # Add the current ID VertexRep (make a copy to avoid mutating input)
+            merged_docs.append(VertexRep(vertex=doc.vertex.copy(), ctx=doc.ctx.copy()))
+        else:
+            # This is a non-ID VertexRep, accumulate it
+            pending_non_ids.append(doc)
+
+    # Handle any remaining non-ID VertexReps at the end
+    if pending_non_ids and merged_docs:
+        merged_docs[-1] = merge_vertex_ctx(merged_docs[-1], pending_non_ids)
+
+    return merged_docs

graflo/util/misc.py

@@ -0,0 +1,37 @@
+"""Miscellaneous utility functions.
+
+This module provides various utility functions for data manipulation and processing.
+
+Key Functions:
+    - sorted_dicts: Recursively sort dictionaries and lists for consistent ordering
+"""
+
+
+def sorted_dicts(d):
+    """Recursively sort dictionaries and lists for consistent ordering.
+
+    This function recursively sorts dictionaries and lists to ensure consistent
+    ordering of data structures. It handles nested structures and preserves
+    non-collection values.
+
+    Args:
+        d: Data structure to sort (dict, list, tuple, or other)
+
+    Returns:
+        The sorted data structure with consistent ordering
+
+    Example:
+        >>> data = {"b": 2, "a": 1, "c": [3, 1, 2]}
+        >>> sorted_dicts(data)
+        {"a": 1, "b": 2, "c": [1, 2, 3]}
+    """
+    if isinstance(d, (tuple, list)):
+        if d and all([not isinstance(dd, (list, tuple, dict)) for dd in d[0].values()]):
+            return sorted(d, key=lambda x: tuple(x.items()))
+    elif isinstance(d, dict):
+        return {
+            k: v if not isinstance(v, (list, tuple, dict)) else sorted_dicts(v)
+            for k, v in d.items()
+        }
+
+    return d

graflo/util/onto.py

@@ -0,0 +1,63 @@
+"""Utility ontology classes for file patterns and configurations.
+
+This module provides data classes for managing file patterns and configurations
+used throughout the system. These classes support file discovery, pattern matching,
+and configuration management.
+
+Key Components:
+    - FilePattern: Configuration for file pattern matching
+    - Patterns: Collection of named file patterns
+"""
+
+import dataclasses
+import pathlib
+
+from graflo.onto import BaseDataclass
+
+
+@dataclasses.dataclass
+class FilePattern(BaseDataclass):
+    """Configuration for file pattern matching.
+
+    This class defines a pattern for matching files, including a regular expression
+    for matching filenames and a subdirectory path to search in.
+
+    Args:
+        regex: Regular expression pattern for matching filenames
+        sub_path: Path to search for matching files (default: "./")
+
+    Attributes:
+        regex: Regular expression pattern
+        sub_path: Path to search in
+    """
+
+    regex: str | None = None
+    sub_path: None | pathlib.Path = dataclasses.field(
+        default_factory=lambda: pathlib.Path("./")
+    )
+
+    def __post_init__(self):
+        """Initialize and validate the file pattern.
+
+        Ensures that sub_path is a Path object and is not None.
+        """
+        if not isinstance(self.sub_path, pathlib.Path):
+            self.sub_path = pathlib.Path(self.sub_path)
+        assert self.sub_path is not None
+
+
+@dataclasses.dataclass
+class Patterns(BaseDataclass):
+    """Collection of named file patterns.
+
+    This class manages a collection of file patterns, each associated with a name.
+    It provides a way to organize and access multiple file patterns.
+
+    Args:
+        patterns: Dictionary mapping names to FilePattern instances
+
+    Attributes:
+        patterns: Dictionary of named file patterns
+    """
+
+    patterns: dict[str, FilePattern] = dataclasses.field(default_factory=dict)

graflo/util/transform.py

@@ -0,0 +1,406 @@
+"""Data transformation utilities for graph operations.
+
+This module provides utility functions for transforming and standardizing data
+in various formats, particularly for graph database operations. It includes
+functions for date parsing, string standardization, and data cleaning.
+
+Key Functions:
+    - standardize: Standardize string keys and names
+    - parse_date_*: Various date parsing functions for different formats
+    - cast_ibes_analyst: Parse and standardize analyst names
+    - clear_first_level_nones: Clean dictionaries by removing None values
+    - parse_multi_item: Parse complex multi-item strings
+    - pick_unique_dict: Remove duplicate dictionaries
+
+Example:
+    >>> name = standardize("John. Doe, Smith")
+    >>> date = parse_date_standard("2023-01-01")
+    >>> analyst = cast_ibes_analyst("ADKINS/NARRA")
+"""
+
+import json
+import logging
+import re
+import time
+from collections import defaultdict
+from datetime import datetime
+
+ORDINAL_SUFFIX = ["st", "nd", "rd", "th"]
+
+logger = logging.getLogger(__name__)
+
+
+def standardize(k):
+    """Standardizes a string key by removing periods and splitting.
+
+    Handles comma and space-separated strings, normalizing their format.
+
+    Args:
+        k (str): Input string to be standardized.
+
+    Returns:
+        str: Cleaned and standardized string.
+
+    Example:
+        >>> standardize("John. Doe, Smith")
+        'John,Doe,Smith'
+        >>> standardize("John Doe Smith")
+        'John,Doe,Smith'
+    """
+    k = k.translate(str.maketrans({".": ""}))
+    # try to split by ", "
+    k = k.split(", ")
+    if len(k) < 2:
+        k = k[0].split(" ")
+    else:
+        k[1] = k[1].translate(str.maketrans({" ": ""}))
+    return ",".join(k)
+
+
+def parse_date_standard(input_str):
+    """Parse a date string in YYYY-MM-DD format.
+
+    Args:
+        input_str (str): Date string in YYYY-MM-DD format.
+
+    Returns:
+        tuple: (year, month, day) as integers.
+
+    Example:
+        >>> parse_date_standard("2023-01-01")
+        (2023, 1, 1)
+    """
+    dt = datetime.strptime(input_str, "%Y-%m-%d")
+    return dt.year, dt.month, dt.day
+
+
+def parse_date_conf(input_str):
+    """Parse a date string in YYYYMMDD format.
+
+    Args:
+        input_str (str): Date string in YYYYMMDD format.
+
+    Returns:
+        tuple: (year, month, day) as integers.
+
+    Example:
+        >>> parse_date_conf("20230101")
+        (2023, 1, 1)
+    """
+    dt = datetime.strptime(input_str, "%Y%m%d")
+    return dt.year, dt.month, dt.day
+
+
+def parse_date_ibes(date0, time0):
+    """Converts IBES date and time to ISO 8601 format datetime.
+
+    Args:
+        date0 (str/int): Date in YYYYMMDD format.
+        time0 (str): Time in HH:MM:SS format.
+
+    Returns:
+        str: Datetime in ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ).
+
+    Example:
+        >>> parse_date_ibes(20160126, "9:35:52")
+        '2016-01-26T09:35:52Z'
+    """
+    date0 = str(date0)
+    year, month, day = date0[:4], date0[4:6], date0[6:]
+    full_datetime = f"{year}-{month}-{day}T{time0}Z"
+
+    return full_datetime
+
+
+def parse_date_yahoo(date0):
+    """Convert Yahoo Finance date to ISO 8601 format.
+
+    Args:
+        date0 (str): Date in YYYY-MM-DD format.
+
+    Returns:
+        str: Datetime in ISO 8601 format with noon time.
+
+    Example:
+        >>> parse_date_yahoo("2023-01-01")
+        '2023-01-01T12:00:00Z'
+    """
+    full_datetime = f"{date0}T12:00:00Z"
+    return full_datetime
+
+
+def round_str(x, **kwargs):
+    """Round a string number to specified precision.
+
+    Args:
+        x (str): String representation of a number.
+        **kwargs: Additional arguments for round() function.
+
+    Returns:
+        float: Rounded number.
+
+    Example:
+        >>> round_str("3.14159", ndigits=2)
+        3.14
+    """
+    return round(float(x), **kwargs)
+
+
+def parse_date_standard_to_epoch(input_str):
+    """Convert standard date string to Unix epoch timestamp.
+
+    Args:
+        input_str (str): Date string in YYYY-MM-DD format.
+
+    Returns:
+        float: Unix epoch timestamp.
+
+    Example:
+        >>> parse_date_standard_to_epoch("2023-01-01")
+        1672531200.0
+    """
+    dt = datetime.strptime(input_str, "%Y-%m-%d").timetuple()
+    timestamp = time.mktime(dt)
+    return timestamp
+
+
+def cast_ibes_analyst(s):
+    """Splits and normalizes analyst name strings.
+
+    Handles various name formats like 'ADKINS/NARRA' or 'ARFSTROM      J'.
+
+    Args:
+        s (str): Analyst name string.
+
+    Returns:
+        tuple: (last_name, first_initial)
+
+    Examples:
+        >>> cast_ibes_analyst('ADKINS/NARRA')
+        ('ADKINS', 'N')
+        >>> cast_ibes_analyst('ARFSTROM      J')
+        ('ARFSTROM', 'J')
+    """
+    if " " in s or "\t" in s:
+        r = s.split()[:2]
+        if len(r) < 2:
+            return r[0], ""
+        else:
+            return r[0], r[1][:1]
+    else:
+        r = s.split("/")
+        if s.startswith("/"):
+            r = r[1:3]
+        else:
+            r = r[:2]
+        if len(r) < 2:
+            return r[0], ""
+        else:
+            return r[0], r[1][:1]
+
+
+def parse_date_reference(input_str):
+    """Extract year from a date reference string.
+
+    Args:
+        input_str (str): Date reference string.
+
+    Returns:
+        int: Year from the date reference.
+
+    Example:
+        >>> parse_date_reference("1923, May 10")
+        1923
+    """
+    return _parse_date_reference(input_str)["year"]
+
+
+def _parse_date_reference(input_str):
+    """Parse complex, human-written date references.
+
+    Handles various date formats like:
+    - "1923, May 10"
+    - "1923, July"
+    - "1921, Sept"
+    - "1935-36"
+    - "1926, December 24th"
+
+    Args:
+        input_str (str): Date string in various formats.
+
+    Returns:
+        dict: Parsed date information with keys 'year', optional 'month', 'day'.
+
+    Example:
+        >>> _parse_date_reference("1923, May 10")
+        {'year': 1923, 'month': 5, 'day': 10}
+    """
+    if "," in input_str:
+        if len(input_str.split(" ")) == 3:
+            if input_str[-2:] in ORDINAL_SUFFIX:
+                input_str = input_str[:-2]
+            try:
+                dt = datetime.strptime(input_str, "%Y, %B %d")
+                return {"year": dt.year, "month": dt.month, "day": dt.day}
+            except:
+                try:
+                    aux = input_str.split(" ")
+                    input_str = " ".join([aux[0]] + [aux[1][:3]] + [aux[2]])
+                    dt = datetime.strptime(input_str, "%Y, %b %d")
+                    return {"year": dt.year, "month": dt.month, "day": dt.day}
+                except:
+                    return {"year": input_str}
+        else:
+            try:
+                dt = datetime.strptime(input_str, "%Y, %B")
+                return {"year": dt.year, "month": dt.month}
+            except:
+                try:
+                    aux = input_str.split(" ")
+                    input_str = " ".join([aux[0]] + [aux[1][:3]])
+                    dt = datetime.strptime(input_str, "%Y, %b")
+                    return {"year": dt.year, "month": dt.month}
+                except:
+                    return {"year": input_str}
+    else:
+        try:
+            dt = datetime.strptime(input_str[:4], "%Y")
+            return {"year": dt.year}
+        except:
+            return {"year": input_str}
+
+
+def try_int(x):
+    """Attempt to convert a value to integer.
+
+    Args:
+        x: Value to convert.
+
+    Returns:
+        int or original value: Integer if conversion successful, original value otherwise.
+
+    Example:
+        >>> try_int("123")
+        123
+        >>> try_int("abc")
+        'abc'
+    """
+    try:
+        x = int(x)
+        return x
+    except:
+        return x
+
+
+def clear_first_level_nones(docs, keys_keep_nones=None):
+    """Removes None values from dictionaries, with optional key exceptions.
+
+    Args:
+        docs (list): List of dictionaries to clean.
+        keys_keep_nones (list, optional): Keys to keep even if their value is None.
+
+    Returns:
+        list: Cleaned list of dictionaries.
+
+    Example:
+        >>> docs = [{"a": 1, "b": None}, {"a": None, "b": 2}]
+        >>> clear_first_level_nones(docs, keys_keep_nones=["a"])
+        [{"a": 1}, {"a": None, "b": 2}]
+    """
+    docs = [
+        {k: v for k, v in tdict.items() if v or k in keys_keep_nones} for tdict in docs
+    ]
+    return docs
+
+
+def parse_multi_item(s, mapper: dict, direct: list):
+    """Parses complex multi-item strings into structured data.
+
+    Supports parsing strings with quoted or bracketed items.
+
+    Args:
+        s (str): Input string to parse.
+        mapper (dict): Mapping of input keys to output keys.
+        direct (list): Direct keys to extract.
+
+    Returns:
+        defaultdict: Parsed items with lists as values.
+
+    Example:
+        >>> s = '[name: John, age: 30] [name: Jane, age: 25]'
+        >>> mapper = {"name": "full_name"}
+        >>> direct = ["age"]
+        >>> parse_multi_item(s, mapper, direct)
+        defaultdict(list, {'full_name': ['John', 'Jane'], 'age': ['30', '25']})
+    """
+    if "'" in s:
+        items_str = re.findall(r"\"(.*?)\"", s) + re.findall(r"\'(.*?)\'", s)
+    else:
+        # remove brackets
+        items_str = re.findall(r"\[([^]]+)", s)[0].split()
+    r: defaultdict[str, list] = defaultdict(list)
+    for item in items_str:
+        doc0 = [ss.strip().split(":") for ss in item.split(",")]
+        if all([len(x) == 2 for x in doc0]):
+            doc0_dict = dict(doc0)
+            for n_init, n_final in mapper.items():
+                try:
+                    r[n_final] += [doc0_dict[n_init]]
+                except KeyError:
+                    r[n_final] += [None]
+
+            for n_final in direct:
+                try:
+                    r[n_final] += [doc0_dict[n_final]]
+                except KeyError:
+                    r[n_final] += [None]
+        else:
+            for key, value in zip(direct, doc0):
+                r[key] += [value]
+
+    return r
+
+
+def pick_unique_dict(docs):
+    """Removes duplicate dictionaries from a list.
+
+    Uses JSON serialization to identify unique dictionaries.
+
+    Args:
+        docs (list): List of dictionaries.
+
+    Returns:
+        list: List of unique dictionaries.
+
+    Example:
+        >>> docs = [{"a": 1}, {"a": 1}, {"b": 2}]
+        >>> pick_unique_dict(docs)
+        [{"a": 1}, {"b": 2}]
+    """
+    docs = {json.dumps(d, sort_keys=True) for d in docs}
+    docs = [json.loads(t) for t in docs]
+    return docs
+
+
+def split_keep_part(s: str, sep="/", keep=-1) -> str:
+    """Split a string and keep specified parts.
+
+    Args:
+        s (str): String to split.
+        sep (str): Separator to split on.
+        keep (int or list): Index or indices to keep.
+
+    Returns:
+        str: Joined string of kept parts.
+
+    Example:
+        >>> split_keep_part("a/b/c", keep=0)
+        'a'
+        >>> split_keep_part("a/b/c", keep=[0, 2])
+        'a/c'
+    """
+    if isinstance(keep, list):
+        items = s.split(sep)
+        return sep.join(items[k] for k in keep)
+    else:
+        return s.split(sep)[keep]