graflo 1.1.0__py3-none-any.whl

graflo/architecture/edge.py

@@ -0,0 +1,295 @@
+"""Edge configuration and management for graph databases.
+
+This module provides classes and utilities for managing edges in graph databases.
+It handles edge configuration, weight management, indexing, and relationship operations.
+The module supports both ArangoDB and Neo4j through the DBFlavor enum.
+
+Key Components:
+    - Edge: Represents an edge with its source, target, and configuration
+    - EdgeConfig: Manages collections of edges and their configurations
+    - WeightConfig: Configuration for edge weights and relationships
+
+Example:
+    >>> edge = Edge(source="user", target="post")
+    >>> config = EdgeConfig(edges=[edge])
+    >>> edge.finish_init(vertex_config=vertex_config)
+"""
+
+import dataclasses
+from typing import Optional
+
+from graflo.architecture.onto import (
+    BaseDataclass,
+    EdgeId,
+    EdgeType,
+    Index,
+    Weight,
+)
+from graflo.architecture.vertex import VertexConfig
+from graflo.onto import DBFlavor
+
+
+@dataclasses.dataclass
+class WeightConfig(BaseDataclass):
+    """Configuration for edge weights and relationships.
+
+    This class manages the configuration of weights and relationships for edges,
+    including source and target field mappings.
+
+    Attributes:
+        vertices: List of weight configurations
+        direct: List of direct field mappings
+    """
+
+    vertices: list[Weight] = dataclasses.field(default_factory=list)
+    direct: list[str] = dataclasses.field(default_factory=list)
+
+
+@dataclasses.dataclass
+class Edge(BaseDataclass):
+    """Represents an edge in the graph database.
+
+    An edge connects two vertices and can have various configurations for
+    indexing, weights, and relationship types.
+
+    Attributes:
+        source: Source vertex name
+        target: Target vertex name
+        indexes: List of indexes for the edge
+        weights: Optional weight configuration
+        relation: Optional relation name (for Neo4j)
+        purpose: Optional purpose for utility collections
+        match_source: Optional source discriminant field
+        match_target: Optional target discriminant field
+        type: Edge type (DIRECT or INDIRECT)
+        aux: Whether this is an auxiliary edge
+        by: Optional vertex name for indirect edges
+        graph_name: Optional graph name
+        collection_name: Optional collection name
+        db_flavor: Database flavor (ARANGO or NEO4J)
+    """
+
+    source: str
+    target: str
+    indexes: list[Index] = dataclasses.field(default_factory=list)
+    weights: Optional[WeightConfig] = None
+
+    # relation represents Class in neo4j, for arango it becomes a weight
+    relation: str | None = None
+    # field that contains Class or relation
+    relation_field: str | None = None
+    relation_from_key: bool = False
+
+    # used to create extra utility collections between the same type of vertices (A, B)
+    purpose: str | None = None
+
+    match_source: str | None = None
+    match_target: str | None = None
+    match: str | None = None
+
+    type: EdgeType = EdgeType.DIRECT
+
+    aux: bool = False  # aux=True edges are init in the db but not considered by graflo
+
+    by: str | None = None
+    graph_name: str | None = None
+    collection_name: str | None = None
+    db_flavor: DBFlavor = DBFlavor.ARANGO
+
+    def __post_init__(self):
+        """Initialize the edge after dataclass initialization."""
+
+        self._source_collection: str | None = None
+        self._target_collection: str | None = None
+
+    def finish_init(self, vertex_config: VertexConfig):
+        """Complete edge initialization with vertex configuration.
+
+        Sets up edge collections, graph names, and initializes indices based on
+        the vertex configuration.
+
+        Args:
+            vertex_config: Configuration for vertices
+
+        """
+        if self.type == EdgeType.INDIRECT and self.by is not None:
+            self.by = vertex_config.vertex_dbname(self.by)
+
+        self._source_collection = vertex_config.vertex_dbname(self.source)
+        self._target_collection = vertex_config.vertex_dbname(self.target)
+        graph_name = [
+            vertex_config.vertex_dbname(self.source),
+            vertex_config.vertex_dbname(self.target),
+        ]
+        if self.purpose is not None:
+            graph_name += [self.purpose]
+        self.graph_name = "_".join(graph_name + ["graph"])
+        self.collection_name = "_".join(graph_name + ["edges"])
+        self.db_flavor = vertex_config.db_flavor
+        self._init_indices(vertex_config)
+
+    def _init_indices(self, vc: VertexConfig):
+        """Initialize indices for the edge.
+
+        Args:
+            vc: Vertex configuration
+        """
+        self.indexes = [self._init_index(index, vc) for index in self.indexes]
+
+    def _init_index(self, index: Index, vc: VertexConfig) -> Index:
+        """Initialize a single index for the edge.
+
+        Args:
+            index: Index to initialize
+            vc: Vertex configuration
+
+        Returns:
+            Index: Initialized index
+
+        Note:
+            Default behavior for edge indices: adds ["_from", "_to"] for uniqueness
+            in ArangoDB.
+        """
+        index_fields = []
+
+        # "@" is reserved : quick hack - do not reinit the index twice
+        if any("@" in f for f in index.fields):
+            return index
+        if index.name is None:
+            index_fields += index.fields
+        else:
+            # add index over a vertex of index.name
+            if index.fields:
+                fields = index.fields
+            else:
+                fields = vc.index(index.name).fields
+            index_fields += [f"{index.name}@{x}" for x in fields]
+
+        if not index.exclude_edge_endpoints and self.db_flavor == DBFlavor.ARANGO:
+            if all([item not in index_fields for item in ["_from", "_to"]]):
+                index_fields = ["_from", "_to"] + index_fields
+
+        index.fields = index_fields
+        return index
+
+    @property
+    def edge_name_dyad(self):
+        """Get the edge name as a dyad (source, target).
+
+        Returns:
+            tuple[str, str]: Source and target vertex names
+        """
+        return self.source, self.target
+
+    @property
+    def edge_id(self) -> EdgeId:
+        """Get the edge ID.
+
+        Returns:
+            EdgeId: Tuple of (source, target, purpose)
+        """
+        return self.source, self.target, self.purpose
+
+
+@dataclasses.dataclass
+class EdgeConfig(BaseDataclass):
+    """Configuration for managing collections of edges.
+
+    This class manages a collection of edges, providing methods for accessing
+    and manipulating edge configurations.
+
+    Attributes:
+        edges: List of edge configurations
+    """
+
+    edges: list[Edge] = dataclasses.field(default_factory=list)
+
+    def __post_init__(self):
+        """Initialize the edge configuration.
+
+        Creates internal mapping of edge IDs to edge configurations.
+        """
+        self._edges_map: dict[EdgeId, Edge] = {e.edge_id: e for e in self.edges}
+
+    def finish_init(self, vc: VertexConfig):
+        """Complete initialization of all edges with vertex configuration.
+
+        Args:
+            vc: Vertex configuration
+        """
+        for _, e in self._edges_map.items():
+            e.finish_init(vc)
+
+    def edges_list(self, include_aux=False):
+        """Get list of edges.
+
+        Args:
+            include_aux: Whether to include auxiliary edges
+
+        Returns:
+            generator: Generator yielding edge configurations
+        """
+        return (e for e in self._edges_map.values() if include_aux or not e.aux)
+
+    def edges_items(self, include_aux=False):
+        """Get items of edges.
+
+        Args:
+            include_aux: Whether to include auxiliary edges
+
+        Returns:
+            generator: Generator yielding (edge_id, edge) tuples
+        """
+        return (
+            (eid, e) for eid, e in self._edges_map.items() if include_aux or not e.aux
+        )
+
+    def __contains__(self, item: EdgeId | Edge):
+        """Check if edge exists in configuration.
+
+        Args:
+            item: Edge ID or Edge instance to check
+
+        Returns:
+            bool: True if edge exists, False otherwise
+        """
+        if isinstance(item, Edge):
+            eid = item.edge_id
+        else:
+            eid = item
+
+        if eid in self._edges_map:
+            return True
+        else:
+            return False
+
+    def update_edges(self, edge: Edge, vertex_config: VertexConfig):
+        """Update edge configuration.
+
+        Args:
+            edge: Edge configuration to update
+            vertex_config: Vertex configuration
+        """
+        if edge.edge_id in self._edges_map:
+            self._edges_map[edge.edge_id].update(edge)
+        else:
+            self._edges_map[edge.edge_id] = edge
+        self._edges_map[edge.edge_id].finish_init(vertex_config=vertex_config)
+
+    @property
+    def vertices(self):
+        """Get set of vertex names involved in edges.
+
+        Returns:
+            set[str]: Set of vertex names
+        """
+        return {e.source for e in self.edges} | {e.target for e in self.edges}
+
+    # def __getitem__(self, key: EdgeId):
+    #     if key in self._reset_edges():
+    #         return self._edges_map[key]
+    #     else:
+    #         raise KeyError(f"Vertex {key} absent")
+    #
+    # def __setitem__(self, key: EdgeId, value: Edge):
+    #     self._edges_map[key] = value

graflo/architecture/onto.py

@@ -0,0 +1,374 @@
+"""Core ontology and data structures for graph database operations.
+
+This module defines the fundamental data structures and types used throughout the graflo
+package for working with graph databases. It provides:
+
+- Core data types for vertices and edges
+- Database index configurations
+- Graph container implementations
+- Edge mapping and casting utilities
+- Action context for graph transformations
+
+The module is designed to be database-agnostic, supporting both ArangoDB and Neo4j through
+the DBFlavor enum. It provides a unified interface for working with graph data structures
+while allowing for database-specific optimizations and features.
+
+Key Components:
+    - EdgeMapping: Defines how edges are mapped between vertices
+    - IndexType: Supported database index types
+    - EdgeType: Types of edge handling in the graph database
+    - GraphContainer: Main container for graph data
+    - ActionContext: Context for graph transformation operations
+
+Example:
+    >>> container = GraphContainer(vertices={}, edges={}, linear=[])
+    >>> index = Index(fields=["name", "age"], type=IndexType.PERSISTENT)
+    >>> context = ActionContext()
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import logging
+from abc import ABCMeta
+from collections import defaultdict
+from typing import Any, Optional, TypeAlias, Union
+
+from dataclass_wizard import JSONWizard, YAMLWizard
+
+from graflo.onto import BaseDataclass, BaseEnum, DBFlavor
+from graflo.util.transform import pick_unique_dict
+
+# type for vertex or edge name (index)
+EdgeId: TypeAlias = tuple[str, str, Optional[str]]
+GraphEntity: TypeAlias = Union[str, EdgeId]
+
+logger = logging.getLogger(__name__)
+
+
+class EdgeMapping(BaseEnum):
+    """Defines how edges are mapped between vertices.
+
+    ALL: Maps all vertices to all vertices
+    ONE_N: Maps one vertex to many vertices
+    """
+
+    ALL = "all"
+    ONE_N = "1-n"
+
+
+class EncodingType(BaseEnum):
+    """Supported character encodings for data input/output."""
+
+    ISO_8859 = "ISO-8859-1"
+    UTF_8 = "utf-8"
+
+
+class IndexType(BaseEnum):
+    """Types of database indexes supported.
+
+    PERSISTENT: Standard persistent index
+    HASH: Hash-based index for fast lookups
+    SKIPLIST: Sorted index using skip list data structure
+    FULLTEXT: Index optimized for text search
+    """
+
+    PERSISTENT = "persistent"
+    HASH = "hash"
+    SKIPLIST = "skiplist"
+    FULLTEXT = "fulltext"
+
+
+class EdgeType(BaseEnum):
+    """Defines how edges are handled in the graph database.
+
+    INDIRECT: Defined as a collection with indexes, may be used after data ingestion
+    DIRECT: In addition to indexes, these edges are generated during ingestion
+    """
+
+    INDIRECT = "indirect"
+    DIRECT = "direct"
+
+
+@dataclasses.dataclass
+class ABCFields(BaseDataclass, metaclass=ABCMeta):
+    """Abstract base class for entities that have fields.
+
+    Attributes:
+        name: Optional name of the entity
+        fields: List of field names
+    """
+
+    name: Optional[str] = None
+    fields: list[str] = dataclasses.field(default_factory=list)
+    keep_vertex_name: bool = True
+
+    def cfield(self, x: str) -> str:
+        """Creates a composite field name by combining the entity name with a field name.
+
+        Args:
+            x: Field name to combine with entity name
+
+        Returns:
+            Composite field name in format "entity@field"
+        """
+        return f"{self.name}@{x}" if self.keep_vertex_name else x
+
+
+@dataclasses.dataclass
+class Weight(ABCFields):
+    """Defines weight configuration for edges.
+
+    Attributes:
+        map: Dictionary mapping field values to weights
+        filter: Dictionary of filter conditions for weights
+    """
+
+    map: dict = dataclasses.field(default_factory=dict)
+    filter: dict = dataclasses.field(default_factory=dict)
+
+
+@dataclasses.dataclass
+class Index(BaseDataclass):
+    """Configuration for database indexes.
+
+    Attributes:
+        name: Optional name of the index
+        fields: List of fields to index
+        unique: Whether the index enforces uniqueness
+        type: Type of index to create
+        deduplicate: Whether to deduplicate index entries
+        sparse: Whether to create a sparse index
+        exclude_edge_endpoints: Whether to exclude edge endpoints from index
+    """
+
+    name: str | None = None
+    fields: list[str] = dataclasses.field(default_factory=list)
+    unique: bool = True
+    type: IndexType = IndexType.PERSISTENT
+    deduplicate: bool = True
+    sparse: bool = False
+    exclude_edge_endpoints: bool = False
+
+    def __iter__(self):
+        """Iterate over the indexed fields."""
+        return iter(self.fields)
+
+    def db_form(self, db_type: DBFlavor) -> dict:
+        """Convert index configuration to database-specific format.
+
+        Args:
+            db_type: Type of database (ARANGO or NEO4J)
+
+        Returns:
+            Dictionary of index configuration in database-specific format
+
+        Raises:
+            ValueError: If db_type is not supported
+        """
+        r = self.to_dict()
+        if db_type == DBFlavor.ARANGO:
+            _ = r.pop("name")
+            _ = r.pop("exclude_edge_endpoints")
+        elif db_type == DBFlavor.NEO4J:
+            pass
+        else:
+            raise ValueError(f"Unknown db_type {db_type}")
+
+        return r
+
+
+class ItemsView:
+    """View class for iterating over vertices and edges in a GraphContainer."""
+
+    def __init__(self, gc: GraphContainer):
+        self._dictlike = gc
+
+    def __iter__(self):
+        """Iterate over vertices and edges in the container."""
+        for key in self._dictlike.vertices:
+            yield key, self._dictlike.vertices[key]
+        for key in self._dictlike.edges:
+            yield key, self._dictlike.edges[key]
+
+
+@dataclasses.dataclass
+class GraphContainer(BaseDataclass):
+    """Container for graph data including vertices and edges.
+
+    Attributes:
+        vertices: Dictionary mapping vertex names to lists of vertex data
+        edges: Dictionary mapping edge IDs to lists of edge data
+        linear: List of default dictionaries containing linear data
+    """
+
+    vertices: dict[str, list]
+    edges: dict[tuple[str, str, str | None], list]
+    linear: list[defaultdict[str | tuple[str, str, str | None], list[Any]]]
+
+    def __post_init__(self):
+        pass
+
+    def items(self):
+        """Get an ItemsView of the container's contents."""
+        return ItemsView(self)
+
+    def pick_unique(self):
+        """Remove duplicate entries from vertices and edges."""
+        for k, v in self.vertices.items():
+            self.vertices[k] = pick_unique_dict(v)
+        for k, v in self.edges.items():
+            self.edges[k] = pick_unique_dict(v)
+
+    def loop_over_relations(self, edge_def: tuple[str, str, str | None]):
+        """Iterate over edges matching the given edge definition.
+
+        Args:
+            edge_def: Tuple of (source, target, optional_purpose)
+
+        Returns:
+            Generator yielding matching edge IDs
+        """
+        source, target, _ = edge_def
+        return (ed for ed in self.edges if source == ed[0] and target == ed[1])
+
+    @classmethod
+    def from_docs_list(
+        cls, list_default_dicts: list[defaultdict[GraphEntity, list]]
+    ) -> GraphContainer:
+        """Create a GraphContainer from a list of default dictionaries.
+
+        Args:
+            list_default_dicts: List of default dictionaries containing vertex and edge data
+
+        Returns:
+            New GraphContainer instance
+
+        Raises:
+            AssertionError: If edge IDs are not properly formatted
+        """
+        vdict: defaultdict[str, list] = defaultdict(list)
+        edict: defaultdict[tuple[str, str, str | None], list] = defaultdict(list)
+
+        for d in list_default_dicts:
+            for k, v in d.items():
+                if isinstance(k, str):
+                    vdict[k].extend(v)
+                elif isinstance(k, tuple):
+                    assert (
+                        len(k) == 3
+                        and all(isinstance(item, str) for item in k[:-1])
+                        and isinstance(k[-1], (str, type(None)))
+                    )
+                    edict[k].extend(v)
+        return GraphContainer(
+            vertices=dict(vdict.items()),
+            edges=dict(edict.items()),
+            linear=list_default_dicts,
+        )
+
+
+class EdgeCastingType(BaseEnum):
+    """Types of edge casting supported.
+
+    PAIR: Edges are cast as pairs of vertices
+    PRODUCT: Edges are cast as combinations of vertex sets
+    """
+
+    PAIR = "pair"
+    PRODUCT = "product"
+    COMBINATIONS = "combinations"
+
+
+def inner_factory_vertex() -> defaultdict[LocationIndex, list]:
+    """Create a default dictionary for vertex data."""
+    return defaultdict(list)
+
+
+def outer_factory() -> defaultdict[str, defaultdict[LocationIndex, list]]:
+    """Create a nested default dictionary for vertex data."""
+    return defaultdict(inner_factory_vertex)
+
+
+def dd_factory() -> defaultdict[GraphEntity, list]:
+    """Create a default dictionary for graph entity data."""
+    return defaultdict(list)
+
+
+@dataclasses.dataclass(kw_only=True)
+class VertexRep(BaseDataclass):
+    """Context for graph transformation actions.
+
+    Attributes:
+        vertex: doc representing a vertex
+        ctx: context (for edge definition upstream
+    """
+
+    vertex: dict
+    ctx: dict
+
+
+@dataclasses.dataclass(frozen=True, eq=True)
+class LocationIndex(JSONWizard, YAMLWizard):
+    path: tuple[str | int | None, ...] = dataclasses.field(default_factory=tuple)
+
+    def extend(self, extension: tuple[str | int | None, ...]) -> LocationIndex:
+        return LocationIndex((*self.path, *extension))
+
+    def depth(self):
+        return len(self.path)
+
+    def congruence_measure(self, other: LocationIndex):
+        neq_position = 0
+        for step_a, step_b in zip(self.path, other.path):
+            if step_a != step_b:
+                break
+            neq_position += 1
+        return neq_position
+
+    def filter(self, lindex_list: list[LocationIndex]) -> list[LocationIndex]:
+        return [
+            t
+            for t in lindex_list
+            if t.depth() >= self.depth() and t.path[: self.depth()] == self.path
+        ]
+
+    def __lt__(self, other: LocationIndex):
+        return len(self.path) < len(other.path)
+
+    def __contains__(self, item):
+        return item in self.path
+
+    def __len__(self):
+        return len(self.path)
+
+    def __iter__(self):
+        return iter(self.path)
+
+    def __getitem__(self, item):
+        return self.path[item]
+
+
+@dataclasses.dataclass(kw_only=True)
+class ActionContext(BaseDataclass):
+    """Context for graph transformation actions.
+
+    Attributes:
+        acc_vertex: Local accumulation of vertices
+        acc_global: Global accumulation of graph entities
+        buffer_vertex: Buffer for vertex data
+        buffer_transforms: Buffer for transforms data
+    """
+
+    acc_vertex: defaultdict[str, defaultdict[LocationIndex, list]] = dataclasses.field(
+        default_factory=outer_factory
+    )
+    acc_global: defaultdict[GraphEntity, list] = dataclasses.field(
+        default_factory=dd_factory
+    )
+    buffer_vertex: defaultdict[GraphEntity, list] = dataclasses.field(
+        default_factory=lambda: defaultdict(list)
+    )
+    buffer_transforms: defaultdict[LocationIndex, list[dict]] = dataclasses.field(
+        default_factory=lambda: defaultdict(list)
+    )