graflo 1.3.7__py3-none-any.whl

graflo/util/transform.py

@@ -0,0 +1,454 @@
+"""Data transformation utilities for graph operations.
+
+This module provides utility functions for transforming and standardizing data
+in various formats, particularly for graph database operations. It includes
+functions for date parsing, string standardization, and data cleaning.
+
+Key Functions:
+    - standardize: Standardize string keys and names
+    - parse_date_*: Various date parsing functions for different formats
+    - cast_ibes_analyst: Parse and standardize analyst names
+    - clear_first_level_nones: Clean dictionaries by removing None values
+    - parse_multi_item: Parse complex multi-item strings
+    - pick_unique_dict: Remove duplicate dictionaries
+
+Example:
+    >>> name = standardize("John. Doe, Smith")
+    >>> date = parse_date_standard("2023-01-01")
+    >>> analyst = cast_ibes_analyst("ADKINS/NARRA")
+"""
+
+import logging
+import re
+import time
+from collections import defaultdict
+from datetime import datetime
+
+ORDINAL_SUFFIX = ["st", "nd", "rd", "th"]
+
+logger = logging.getLogger(__name__)
+
+
+def standardize(k):
+    """Standardizes a string key by removing periods and splitting.
+
+    Handles comma and space-separated strings, normalizing their format.
+
+    Args:
+        k (str): Input string to be standardized.
+
+    Returns:
+        str: Cleaned and standardized string.
+
+    Example:
+        >>> standardize("John. Doe, Smith")
+        'John,Doe,Smith'
+        >>> standardize("John Doe Smith")
+        'John,Doe,Smith'
+    """
+    k = k.translate(str.maketrans({".": ""}))
+    # try to split by ", "
+    k = k.split(", ")
+    if len(k) < 2:
+        k = k[0].split(" ")
+    else:
+        k[1] = k[1].translate(str.maketrans({" ": ""}))
+    return ",".join(k)
+
+
+def parse_date_standard(input_str):
+    """Parse a date string in YYYY-MM-DD format.
+
+    Args:
+        input_str (str): Date string in YYYY-MM-DD format.
+
+    Returns:
+        tuple: (year, month, day) as integers.
+
+    Example:
+        >>> parse_date_standard("2023-01-01")
+        (2023, 1, 1)
+    """
+    dt = datetime.strptime(input_str, "%Y-%m-%d")
+    return dt.year, dt.month, dt.day
+
+
+def parse_date_conf(input_str):
+    """Parse a date string in YYYYMMDD format.
+
+    Args:
+        input_str (str): Date string in YYYYMMDD format.
+
+    Returns:
+        tuple: (year, month, day) as integers.
+
+    Example:
+        >>> parse_date_conf("20230101")
+        (2023, 1, 1)
+    """
+    dt = datetime.strptime(input_str, "%Y%m%d")
+    return dt.year, dt.month, dt.day
+
+
+def parse_date_ibes(date0, time0):
+    """Converts IBES date and time to ISO 8601 format datetime.
+
+    Args:
+        date0 (str/int): Date in YYYYMMDD format.
+        time0 (str): Time in HH:MM:SS format.
+
+    Returns:
+        str: Datetime in ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ).
+
+    Example:
+        >>> parse_date_ibes(20160126, "9:35:52")
+        '2016-01-26T09:35:52Z'
+    """
+    date0 = str(date0)
+    year, month, day = date0[:4], date0[4:6], date0[6:]
+    full_datetime = f"{year}-{month}-{day}T{time0}Z"
+
+    return full_datetime
+
+
+def parse_date_yahoo(date0):
+    """Convert Yahoo Finance date to ISO 8601 format.
+
+    Args:
+        date0 (str): Date in YYYY-MM-DD format.
+
+    Returns:
+        str: Datetime in ISO 8601 format with noon time.
+
+    Example:
+        >>> parse_date_yahoo("2023-01-01")
+        '2023-01-01T12:00:00Z'
+    """
+    full_datetime = f"{date0}T12:00:00Z"
+    return full_datetime
+
+
+def round_str(x, **kwargs):
+    """Round a string number to specified precision.
+
+    Args:
+        x (str): String representation of a number.
+        **kwargs: Additional arguments for round() function.
+
+    Returns:
+        float: Rounded number.
+
+    Example:
+        >>> round_str("3.14159", ndigits=2)
+        3.14
+    """
+    return round(float(x), **kwargs)
+
+
+def parse_date_standard_to_epoch(input_str):
+    """Convert standard date string to Unix epoch timestamp.
+
+    Args:
+        input_str (str): Date string in YYYY-MM-DD format.
+
+    Returns:
+        float: Unix epoch timestamp.
+
+    Example:
+        >>> parse_date_standard_to_epoch("2023-01-01")
+        1672531200.0
+    """
+    dt = datetime.strptime(input_str, "%Y-%m-%d").timetuple()
+    timestamp = time.mktime(dt)
+    return timestamp
+
+
+def cast_ibes_analyst(s):
+    """Splits and normalizes analyst name strings.
+
+    Handles various name formats like 'ADKINS/NARRA' or 'ARFSTROM      J'.
+
+    Args:
+        s (str): Analyst name string.
+
+    Returns:
+        tuple: (last_name, first_initial)
+
+    Examples:
+        >>> cast_ibes_analyst('ADKINS/NARRA')
+        ('ADKINS', 'N')
+        >>> cast_ibes_analyst('ARFSTROM      J')
+        ('ARFSTROM', 'J')
+    """
+    if " " in s or "\t" in s:
+        r = s.split()[:2]
+        if len(r) < 2:
+            return r[0], ""
+        else:
+            return r[0], r[1][:1]
+    else:
+        r = s.split("/")
+        if s.startswith("/"):
+            r = r[1:3]
+        else:
+            r = r[:2]
+        if len(r) < 2:
+            return r[0], ""
+        else:
+            return r[0], r[1][:1]
+
+
+def parse_date_reference(input_str):
+    """Extract year from a date reference string.
+
+    Args:
+        input_str (str): Date reference string.
+
+    Returns:
+        int: Year from the date reference.
+
+    Example:
+        >>> parse_date_reference("1923, May 10")
+        1923
+    """
+    return _parse_date_reference(input_str)["year"]
+
+
+def _parse_date_reference(input_str):
+    """Parse complex, human-written date references.
+
+    Handles various date formats like:
+    - "1923, May 10"
+    - "1923, July"
+    - "1921, Sept"
+    - "1935-36"
+    - "1926, December 24th"
+
+    Args:
+        input_str (str): Date string in various formats.
+
+    Returns:
+        dict: Parsed date information with keys 'year', optional 'month', 'day'.
+
+    Example:
+        >>> _parse_date_reference("1923, May 10")
+        {'year': 1923, 'month': 5, 'day': 10}
+    """
+    if "," in input_str:
+        if len(input_str.split(" ")) == 3:
+            if input_str[-2:] in ORDINAL_SUFFIX:
+                input_str = input_str[:-2]
+            try:
+                dt = datetime.strptime(input_str, "%Y, %B %d")
+                return {"year": dt.year, "month": dt.month, "day": dt.day}
+            except:
+                try:
+                    aux = input_str.split(" ")
+                    input_str = " ".join([aux[0]] + [aux[1][:3]] + [aux[2]])
+                    dt = datetime.strptime(input_str, "%Y, %b %d")
+                    return {"year": dt.year, "month": dt.month, "day": dt.day}
+                except:
+                    return {"year": input_str}
+        else:
+            try:
+                dt = datetime.strptime(input_str, "%Y, %B")
+                return {"year": dt.year, "month": dt.month}
+            except:
+                try:
+                    aux = input_str.split(" ")
+                    input_str = " ".join([aux[0]] + [aux[1][:3]])
+                    dt = datetime.strptime(input_str, "%Y, %b")
+                    return {"year": dt.year, "month": dt.month}
+                except:
+                    return {"year": input_str}
+    else:
+        try:
+            dt = datetime.strptime(input_str[:4], "%Y")
+            return {"year": dt.year}
+        except:
+            return {"year": input_str}
+
+
+def try_int(x):
+    """Attempt to convert a value to integer.
+
+    Args:
+        x: Value to convert.
+
+    Returns:
+        int or original value: Integer if conversion successful, original value otherwise.
+
+    Example:
+        >>> try_int("123")
+        123
+        >>> try_int("abc")
+        'abc'
+    """
+    try:
+        x = int(x)
+        return x
+    except:
+        return x
+
+
+def clear_first_level_nones(docs, keys_keep_nones: list | None = None):
+    """Removes None values from dictionaries, with optional key exceptions.
+
+    Args:
+        docs (list): List of dictionaries to clean.
+        keys_keep_nones (list, optional): Keys to keep even if their value is None.
+
+    Returns:
+        list: Cleaned list of dictionaries.
+
+    Example:
+        >>> docs = [{"a": 1, "b": None}, {"a": None, "b": 2}]
+        >>> clear_first_level_nones(docs, keys_keep_nones=["a"])
+        [{"a": 1}, {"a": None, "b": 2}]
+    """
+    if keys_keep_nones is not None:
+        docs = [
+            {k: v for k, v in tdict.items() if v or k in keys_keep_nones}
+            for tdict in docs
+        ]
+    return docs
+
+
+def parse_multi_item(s, mapper: dict, direct: list):
+    """Parses complex multi-item strings into structured data.
+
+    Supports parsing strings with quoted or bracketed items.
+
+    Args:
+        s (str): Input string to parse.
+        mapper (dict): Mapping of input keys to output keys.
+        direct (list): Direct keys to extract.
+
+    Returns:
+        defaultdict: Parsed items with lists as values.
+
+    Example:
+        >>> s = '[name: John, age: 30] [name: Jane, age: 25]'
+        >>> mapper = {"name": "full_name"}
+        >>> direct = ["age"]
+        >>> parse_multi_item(s, mapper, direct)
+        defaultdict(list, {'full_name': ['John', 'Jane'], 'age': ['30', '25']})
+    """
+    if "'" in s:
+        items_str = re.findall(r"\"(.*?)\"", s) + re.findall(r"\'(.*?)\'", s)
+    else:
+        # remove brackets
+        items_str = re.findall(r"\[([^]]+)", s)[0].split()
+    r: defaultdict[str, list] = defaultdict(list)
+    for item in items_str:
+        doc0 = [ss.strip().split(":") for ss in item.split(",")]
+        if all([len(x) == 2 for x in doc0]):
+            doc0_dict = dict(doc0)
+            for n_init, n_final in mapper.items():
+                try:
+                    r[n_final] += [doc0_dict[n_init]]
+                except KeyError:
+                    r[n_final] += [None]
+
+            for n_final in direct:
+                # Use field.name for dictionary keys (JSON serialization requires strings)
+                # Handle both Field objects and strings for backward compatibility
+                key = n_final.name if hasattr(n_final, "name") else str(n_final)
+                try:
+                    r[key] += [doc0_dict[key]]
+                except KeyError:
+                    r[key] += [None]
+        else:
+            for key, value in zip(direct, doc0):
+                # Use field.name for dictionary keys (JSON serialization requires strings)
+                # Handle both Field objects and strings for backward compatibility
+                key_str = key.name if hasattr(key, "name") else str(key)
+                r[key_str] += [value]
+
+    return r
+
+
+def pick_unique_dict(docs):
+    """Removes duplicate dictionaries from a list.
+
+    Uses a hash-based approach to identify unique dictionaries, which is more
+    efficient than JSON serialization and preserves original object types.
+
+    Args:
+        docs (list): List of dictionaries.
+
+    Returns:
+        list: List of unique dictionaries (preserving original objects).
+
+    Example:
+        >>> docs = [{"a": 1}, {"a": 1}, {"b": 2}]
+        >>> pick_unique_dict(docs)
+        [{"a": 1}, {"b": 2}]
+    """
+    from datetime import date, datetime, time
+    from decimal import Decimal
+
+    def make_hashable(obj):
+        """Convert an object to a hashable representation.
+
+        Handles nested structures, datetime objects, and Decimal types.
+
+        Args:
+            obj: Object to make hashable
+
+        Returns:
+            Hashable representation of the object
+        """
+        if isinstance(obj, dict):
+            # Sort items by key for consistent hashing
+            return tuple(sorted((k, make_hashable(v)) for k, v in obj.items()))
+        elif isinstance(obj, (list, tuple)):
+            return tuple(make_hashable(item) for item in obj)
+        elif isinstance(obj, (datetime, date, time)):
+            # Convert to ISO format string for hashing
+            return ("__datetime__", obj.isoformat())
+        elif isinstance(obj, Decimal):
+            # Convert to string representation to preserve precision
+            return ("__decimal__", str(obj))
+        elif isinstance(obj, set):
+            # Convert set to sorted tuple for consistent hashing
+            return tuple(sorted(make_hashable(item) for item in obj))
+        else:
+            # Primitive types (int, float, str, bool, None) are already hashable
+            return obj
+
+    # Use a dict to preserve insertion order and original objects
+    seen = {}
+    for doc in docs:
+        # Create hashable representation
+        hashable_repr = make_hashable(doc)
+        # Use hashable representation as key, original doc as value
+        if hashable_repr not in seen:
+            seen[hashable_repr] = doc
+
+    # Return list of unique documents (preserving original objects)
+    return list(seen.values())
+
+
+def split_keep_part(s: str, sep="/", keep=-1) -> str:
+    """Split a string and keep specified parts.
+
+    Args:
+        s (str): String to split.
+        sep (str): Separator to split on.
+        keep (int or list): Index or indices to keep.
+
+    Returns:
+        str: Joined string of kept parts.
+
+    Example:
+        >>> split_keep_part("a/b/c", keep=0)
+        'a'
+        >>> split_keep_part("a/b/c", keep=[0, 2])
+        'a/c'
+    """
+    if isinstance(keep, list):
+        items = s.split(sep)
+        return sep.join(items[k] for k in keep)
+    else:
+        return s.split(sep)[keep]

graflo-1.3.7.dist-info/METADATA

@@ -0,0 +1,243 @@
+Metadata-Version: 2.4
+Name: graflo
+Version: 1.3.7
+Summary: A framework for transforming tabular (CSV, SQL) and hierarchical data (JSON, XML) into property graphs and ingesting them into graph databases (ArangoDB, Neo4j, TigerGraph). Features automatic PostgreSQL schema inference.
+Author-email: Alexander Belikov <alexander@growgraph.dev>
+License-File: LICENSE
+Requires-Python: ~=3.10.0
+Requires-Dist: click<9,>=8.2.0
+Requires-Dist: dataclass-wizard>=0.34.0
+Requires-Dist: ijson<4,>=3.2.3
+Requires-Dist: neo4j<6,>=5.22.0
+Requires-Dist: networkx~=3.3
+Requires-Dist: pandas-stubs==2.3.0.250703
+Requires-Dist: pandas<3,>=2.0.3
+Requires-Dist: psycopg2-binary>=2.9.11
+Requires-Dist: pydantic-settings>=2.12.0
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: python-arango<9,>=8.1.2
+Requires-Dist: pytigergraph>=1.9.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: sqlalchemy>=2.0.0
+Requires-Dist: strenum>=0.4.15
+Requires-Dist: suthing>=0.5.0
+Requires-Dist: urllib3>=2.0.0
+Requires-Dist: xmltodict<0.15,>=0.14.2
+Provides-Extra: plot
+Requires-Dist: pygraphviz>=1.14; extra == 'plot'
+Description-Content-Type: text/markdown
+
+# GraFlo <img src="https://raw.githubusercontent.com/growgraph/graflo/main/docs/assets/favicon.ico" alt="graflo logo" style="height: 32px; width:32px;"/>
+
+A framework for transforming **tabular** (CSV, SQL) and **hierarchical** data (JSON, XML) into property graphs and ingesting them into graph databases (ArangoDB, Neo4j, **TigerGraph**).
+
+> **⚠️ Package Renamed**: This package was formerly known as `graphcast`.
+
+![Python](https://img.shields.io/badge/python-3.10-blue.svg) 
+[![PyPI version](https://badge.fury.io/py/graflo.svg)](https://badge.fury.io/py/graflo)
+[![PyPI Downloads](https://static.pepy.tech/badge/graflo)](https://pepy.tech/projects/graflo)
+[![License: BSL](https://img.shields.io/badge/license-BSL--1.1-green)](https://github.com/growgraph/graflo/blob/main/LICENSE)
+[![pre-commit](https://github.com/growgraph/graflo/actions/workflows/pre-commit.yml/badge.svg)](https://github.com/growgraph/graflo/actions/workflows/pre-commit.yml)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.15446131.svg)]( https://doi.org/10.5281/zenodo.15446131)
+
+## Core Concepts
+
+### Property Graphs
+graflo works with property graphs, which consist of:
+
+- **Vertices**: Nodes with properties and optional unique identifiers
+- **Edges**: Relationships between vertices with their own properties
+- **Properties**: Both vertices and edges may have properties
+
+### Schema
+The Schema defines how your data should be transformed into a graph and contains:
+
+- **Vertex Definitions**: Specify vertex types, their properties, and unique identifiers
+  - Fields can be specified as strings (backward compatible) or typed `Field` objects with types (INT, FLOAT, STRING, DATETIME, BOOL)
+  - Type information enables better validation and database-specific optimizations
+- **Edge Definitions**: Define relationships between vertices and their properties
+  - Weight fields support typed definitions for better type safety
+- **Resource Mapping**: describe how data sources map to vertices and edges
+- **Transforms**: Modify data during the casting process
+- **Automatic Schema Inference**: Generate schemas automatically from PostgreSQL 3NF databases
+
+### Resources
+Resources are your data sources that can be:
+
+- **Table-like**: CSV files, database tables
+- **JSON-like**: JSON files, nested data structures
+
+## Features
+
+- **Graph Transformation Meta-language**: A powerful declarative language to describe how your data becomes a property graph:
+    - Define vertex and edge structures with typed fields
+    - Set compound indexes for vertices and edges
+    - Use blank vertices for complex relationships
+    - Specify edge constraints and properties with typed weight fields
+    - Apply advanced filtering and transformations
+- **Typed Schema Definitions**: Enhanced type support throughout the schema system
+    - Vertex fields support types (INT, FLOAT, STRING, DATETIME, BOOL) for better validation
+    - Edge weight fields can specify types for improved type safety
+    - Backward compatible: fields without types default to None (suitable for databases like ArangoDB)
+- **🚀 PostgreSQL Schema Inference**: **Automatically generate schemas from PostgreSQL 3NF databases** - No manual schema definition needed!
+    - Introspect PostgreSQL schemas to identify vertex-like and edge-like tables
+    - Automatically map PostgreSQL data types to graflo Field types (INT, FLOAT, STRING, DATETIME, BOOL)
+    - Infer vertex configurations from table structures with proper indexes
+    - Infer edge configurations from foreign key relationships
+    - Create Resource mappings from PostgreSQL tables automatically
+    - Direct database access - ingest data without exporting to files first
+- **Parallel processing**: Use as many cores as you have
+- **Database support**: Ingest into ArangoDB, Neo4j, and **TigerGraph** using the same API (database agnostic). Source data from PostgreSQL and other SQL databases.
+- **Server-side filtering**: Efficient querying with server-side filtering support (TigerGraph REST++ API)
+
+## Documentation
+Full documentation is available at: [growgraph.github.io/graflo](https://growgraph.github.io/graflo)
+
+## Installation
+
+```bash
+pip install graflo
+```
+
+## Usage Examples
+
+### Simple ingest
+
+```python
+from suthing import FileHandle
+
+from graflo import Schema, Caster, Patterns
+from graflo.db.connection.onto import ArangoConfig
+
+schema = Schema.from_dict(FileHandle.load("schema.yaml"))
+
+# Option 1: Load config from docker/arango/.env (recommended)
+conn_conf = ArangoConfig.from_docker_env()
+
+# Option 2: Load from environment variables
+# Set: ARANGO_URI, ARANGO_USERNAME, ARANGO_PASSWORD, ARANGO_DATABASE
+conn_conf = ArangoConfig.from_env()
+
+# Option 3: Load with custom prefix (for multiple configs)
+# Set: USER_ARANGO_URI, USER_ARANGO_USERNAME, USER_ARANGO_PASSWORD, USER_ARANGO_DATABASE
+user_conn_conf = ArangoConfig.from_env(prefix="USER")
+
+# Option 4: Create config directly
+# conn_conf = ArangoConfig(
+#     uri="http://localhost:8535",
+#     username="root",
+#     password="123",
+#     database="mygraph",  # For ArangoDB, 'database' maps to schema/graph
+# )
+# Note: If 'database' (or 'schema_name' for TigerGraph) is not set,
+# Caster will automatically use Schema.general.name as fallback
+
+from graflo.util.onto import FilePattern
+import pathlib
+
+# Create Patterns with file patterns
+patterns = Patterns()
+patterns.add_file_pattern(
+    "work",
+    FilePattern(regex="\Sjson$", sub_path=pathlib.Path("./data"), resource_name="work")
+)
+
+# Or use resource_mapping for simpler initialization
+# patterns = Patterns(
+#     _resource_mapping={
+#         "work": "./data/work.json",
+#     }
+# )
+
+schema.fetch_resource()
+
+from graflo.caster import IngestionParams
+
+caster = Caster(schema)
+
+ingestion_params = IngestionParams(
+    clean_start=False,  # Set to True to wipe existing database
+    # max_items=1000,  # Optional: limit number of items to process
+    # batch_size=10000,  # Optional: customize batch size
+)
+
+caster.ingest(
+    output_config=conn_conf,  # Target database config
+    patterns=patterns,  # Source data patterns
+    ingestion_params=ingestion_params,
+)
+```
+
+### PostgreSQL Schema Inference
+
+```python
+from graflo.db.postgres import PostgresConnection
+from graflo.db.postgres.heuristics import infer_schema_from_postgres
+from graflo.db.connection.onto import PostgresConfig
+from graflo import Caster
+from graflo.onto import DBFlavor
+
+# Connect to PostgreSQL
+postgres_config = PostgresConfig.from_docker_env()  # or PostgresConfig.from_env()
+postgres_conn = PostgresConnection(postgres_config)
+
+# Infer schema from PostgreSQL 3NF database
+schema = infer_schema_from_postgres(
+    postgres_conn,
+    schema_name="public",  # PostgreSQL schema name
+    db_flavor=DBFlavor.ARANGO  # Target graph database flavor
+)
+
+# Close PostgreSQL connection
+postgres_conn.close()
+
+# Use the inferred schema with Caster
+caster = Caster(schema)
+# ... continue with ingestion
+```
+
+## Development
+
+To install requirements
+
+```shell
+git clone git@github.com:growgraph/graflo.git && cd graflo
+uv sync --dev
+```
+
+### Tests
+
+#### Test databases
+Spin up Arango from [arango docker folder](./docker/arango) by
+
+```shell
+docker-compose --env-file .env up arango
+```
+
+Neo4j from [neo4j docker folder](./docker/neo4j) by
+
+```shell
+docker-compose --env-file .env up neo4j
+```
+
+and TigerGraph from [tigergraph docker folder](./docker/tigergraph) by
+
+```shell
+docker-compose --env-file .env up tigergraph
+```
+
+To run unit tests
+
+```shell
+pytest test
+```
+
+## Requirements
+
+- Python 3.10+
+- python-arango
+- sqlalchemy>=2.0.0 (for PostgreSQL and SQL data sources)
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.