graph-seeder 1.0.0.dev0__py3-none-any.whl

graph_seeder/utils/utils.py

@@ -0,0 +1,217 @@
+import networkx as nx
+import json
+from importlib import resources
+
+BUILTIN_CONFIGS = [
+    "dbpedia_default",
+    "wikidata_default",
+    "pgxlod_default",
+    "europeana_default",
+    "default",
+]
+
+OVERRIDE_MAP = {
+    # Data section
+    "input_path": ("data", str, "Path to the input csv file containing seed nodes."),
+    "output_format": (
+        "data",
+        str,
+        "Format of the extracted graph output ('csv', 'json', 'ttl').",
+    ),
+    "output_path": (
+        "data",
+        str,
+        "Destination path and base filename for the extracted graph (e.g., 'output/result').",
+    ),
+    "turtle_path": (
+        "data",
+        str,
+        "Path to a local Turtle file (if using local extraction instead of a SPARQL endpoint).",
+    ),
+    # Client section
+    "endpoint": (
+        "client",
+        str,
+        "URL of the SPARQL endpoint to query (e.g., 'https://dbpedia.org/sparql').",
+    ),
+    "user_agent": (
+        "client",
+        str,
+        "HTTP User-Agent header to identify your requests to the server.",
+    ),
+    "request_delay": (
+        "client",
+        float,
+        "Delay in seconds between consecutive requests to avoid overloading the server.",
+    ),
+    "retry_attempts": (
+        "client",
+        int,
+        "Number of times to retry a failed HTTP request.",
+    ),
+    "retry_delay": (
+        "client",
+        float,
+        "Delay in seconds before retrying a failed request.",
+    ),
+    "rate_limit_wait": (
+        "client",
+        float,
+        "Time to wait in seconds when a rate limit (HTTP 429) is encountered.",
+    ),
+    "timeout": (
+        "client",
+        float,
+        "Maximum time in seconds to wait for a server response.",
+    ),
+    # Extraction section
+    "strategy": (
+        "extraction",
+        str,
+        "Graph extraction algorithm to use ('bfs', 'hop').",
+    ),
+    "batch_size": (
+        "extraction",
+        int,
+        "Number of entities to process in a single SPARQL query.",
+    ),
+    "max_hops": (
+        "extraction",
+        int,
+        "Maximum depth or distance from the seed nodes to explore.",
+    ),
+    "hub_pagination_threshold": (
+        "extraction",
+        int,
+        "Number of neighbors at which the extractor will start paginating queries for a node.",
+    ),
+    "max_neighbors_threshold": (
+        "extraction",
+        int,
+        "Maximum number of neighbors allowed before a node is considered a massive hub.",
+    ),
+    "min_triplets_per_property": (
+        "extraction",
+        int,
+        "Minimum number of triplets required per property to be kept when paginating.",
+    ),
+    "check_seeds_validity": (
+        "extraction",
+        bool,
+        "Verify if seed nodes have valid URIs.",
+    ),
+    "check_hub_seeds": (
+        "extraction",
+        bool,
+        "Check the degree of seed nodes beforehand to identify massive hubs.",
+    ),
+    "keep_hub_seeds": (
+        "extraction",
+        bool,
+        "Whether to keep (True), skip (False), or prompt the user (None) about massive hub seeds.",
+    ),
+    # Densification section
+    "mode": (
+        "densification",
+        str,
+        "Strategy used to connect disconnected components during densification ('most_connected', 'random').",
+    ),
+    "skip_densification": (
+        "densification",
+        bool,
+        "Skip the post-extraction step that attempts to connect isolated subgraphs.",
+    ),
+    # Graph filters
+    "include_uri_prefixes": (
+        "graph_filters",
+        list,
+        "Only explore nodes whose URIs start with one of these prefixes.",
+    ),
+    "exclude_uri_prefixes": (
+        "graph_filters",
+        list,
+        "Ignore nodes whose URIs start with any of these prefixes.",
+    ),
+    "exclude_properties": (
+        "graph_filters",
+        list,
+        "Specific properties (URIs) to completely ignore during extraction.",
+    ),
+    "exclude_nodes": (
+        "graph_filters",
+        list,
+        "Specific nodes (URIs) to completely ignore during extraction.",
+    ),
+    "namespaces": (
+        "graph_filters",
+        (dict, list),
+        "Custom namespaces in 'prefix=URI' format (e.g., ex=http://example.com/).",
+    ),
+    # Debug section
+    "debug_enabled": ("debug", bool, "Enable verbose debug-level logging."),
+    "request_logging": (
+        "debug",
+        bool,
+        "Log details of all SPARQL queries and HTTP requests.",
+    ),
+}
+
+
+def load_config(config_path: str | None, overrides: dict) -> dict:
+    """Load configuration from a JSON file and apply overrides."""
+    cfg: dict = {}
+
+    if config_path is not None:
+        config_name = config_path.lower().replace(".json", "")
+
+        if config_name in BUILTIN_CONFIGS:
+            template_path = resources.files("graph_seeder.configs").joinpath(
+                f"{config_name}.json"
+            )
+            with template_path.open("r", encoding="utf-8") as f:
+                cfg = json.load(f)
+        else:
+            with open(config_path, "r", encoding="utf-8") as f:
+                cfg = json.load(f)
+
+    for key, value in overrides.items():
+        if key not in OVERRIDE_MAP:
+            raise ValueError(
+                f"Unknown override key: '{key}'. Valid keys: {list(OVERRIDE_MAP)}"
+            )
+
+        mapping = OVERRIDE_MAP[key]
+        section = mapping[0]
+        expected_type = mapping[1]
+
+        if expected_type == (dict, list):
+            expected_type = dict
+
+            # If the value is already a dict (e.g., from a config file), we keep it as is. If it's a list (from command-line), we parse it into a dict.
+            if isinstance(value, list):
+                parsed_dict = {}
+                for item in value:
+                    if "=" not in item:
+                        raise ValueError(
+                            f"Namespace override '{item}' is invalid. Use 'prefix=URI' format."
+                        )
+                    pref, uri = item.split("=", 1)
+                    parsed_dict[pref] = uri
+                value = parsed_dict
+
+        if not isinstance(value, expected_type):
+            raise TypeError(
+                f"Override '{key}' must be {expected_type}, got {type(value).__name__}"
+            )
+
+        target_section = cfg.setdefault(section, {})
+
+        target_section[key] = value
+
+    return cfg
+
+
+def get_connected_components(triplets: list[tuple[str, str, str]]) -> list[set[str]]:
+    graph: nx.Graph = nx.Graph()
+    graph.add_edges_from((s, o) for s, p, o in triplets)
+    return list(nx.connected_components(graph))

graph_seeder/wrapper/NeighborhoodWrapper.py

@@ -0,0 +1,47 @@
+from abc import ABC, abstractmethod
+from collections.abc import Generator
+from graph_seeder.utils.URIManager import URIManager
+
+
+class NeighborhoodWrapper(ABC):
+    def __init__(self, uri_manager: URIManager, config: dict):
+        self.uri_manager = uri_manager
+        self.cfg = config
+
+        skip_val = config["extraction"].get("max_neighbors_threshold")
+        self.max_neighbors_threshold = (
+            skip_val if skip_val is not None else float("inf")
+        )
+
+        self.excluded_nodes = {
+            n for n in config.get("graph_filters", {}).get("exclude_nodes", [])
+        }
+        self.excluded_properties = {
+            p for p in config.get("graph_filters", {}).get("exclude_properties", [])
+        }
+
+        self.included_uri_prefixes = {
+            p for p in config.get("graph_filters", {}).get("include_uri_prefixes", [])
+        }
+        self.excluded_uri_prefixes = {
+            p for p in config.get("graph_filters", {}).get("exclude_uri_prefixes", [])
+        }
+
+        self.forced_hubs: set[str] = set()
+
+    @abstractmethod
+    def check_seeds_validity(self, seeds: list[str]) -> dict[str, bool]:
+        """Check if the given seeds are valid and return a dict mapping each seed to a boolean indicating if it's valid."""
+
+    @abstractmethod
+    def count_neighbors(self, seeds: list[str]) -> dict[str, int]:
+        """Return a dict mapping each seed to its total number of neighbors."""
+
+    @abstractmethod
+    def get_neighborhood(
+        self, nodes: list[str]
+    ) -> Generator[list[tuple[str, str, str]], None, None]:
+        """
+        Yields the neighborhood of a list of nodes in batches.
+        Allows the consumer to break the loop to stop early.
+        """

graph_seeder/wrapper/hashmap/HashMapWrapper.py

@@ -0,0 +1,124 @@
+from collections.abc import Generator
+import lmdb
+import logging
+from graph_seeder.wrapper.NeighborhoodWrapper import NeighborhoodWrapper
+from graph_seeder.utils.URIManager import URIManager
+import json
+
+logger = logging.getLogger("subgraph")
+
+
+class HashMapWrapper(NeighborhoodWrapper):
+    """
+    Local graph wrapper using an LMDB hashmap.
+    dictionaries of properties and neighbor lists.
+    """
+
+    def __init__(self, uri_manager: URIManager, config: dict):
+        super().__init__(uri_manager, config)
+
+        db_path = config.get("data", {}).get("hashmap_path", "data")
+        self.env = lmdb.open(db_path, readonly=True, lock=False)
+
+        self._included_prefixes_tuple = tuple(self.included_uri_prefixes)
+        self._excluded_prefixes_tuple = tuple(self.excluded_uri_prefixes)
+
+    def check_seeds_validity(self, seeds: list[str]) -> dict[str, bool]:
+        """Return a dict mapping each seed to a boolean indicating if it's valid."""
+        results = {}
+        with self.env.begin() as txn:
+            for seed in seeds:
+                val = txn.get(seed.encode("utf-8"))
+                results[seed] = val is not None
+        return results
+
+    def count_neighbors(self, seeds: list[str]) -> dict[str, int]:
+        """Return a dict mapping each seed to its total number of neighbors."""
+        seed_totals = {}
+        with self.env.begin() as txn:
+            for node in seeds:
+                val = txn.get(node.encode("utf-8"))
+                if val:
+                    try:
+                        data = json.loads(val.decode("utf-8"))
+                        seed_totals[node] = sum(len(n) for n in data.values())
+                    except Exception:
+                        pass
+        return seed_totals
+
+    def get_neighborhood(
+        self, nodes: list[str]
+    ) -> Generator[list[tuple[str, str, str]], None, None]:
+        triplets: list[tuple[str, str, str]] = []
+        skipped_nodes: set[str] = set()
+
+        with self.env.begin() as txn:
+            for original_node in nodes:
+                value = txn.get(original_node.encode("utf-8"))
+
+                if value is None:
+                    logger.warning(f"Node {original_node} not found in hashmap.")
+                    continue
+
+                try:
+                    neighbors_data = value.decode("utf-8")
+                except UnicodeDecodeError:
+                    logger.error(
+                        f"Failed to decode value for node {original_node}. Skipping."
+                    )
+
+                try:
+                    neighbors_data: dict[str, list[str]] = json.loads(neighbors_data)
+                except json.JSONDecodeError:
+                    logger.error(
+                        f"Failed to parse JSON for node {original_node}. Skipping."
+                    )
+                    continue
+
+                total_neighbors = sum(
+                    len(neighbors) for neighbors in neighbors_data.values()
+                )
+                if (
+                    total_neighbors >= self.max_neighbors_threshold
+                    and original_node not in self.forced_hubs
+                ):
+                    logger.warning(
+                        f"Node {original_node} has {total_neighbors} neighbors, which exceeds the skip threshold of {self.max_neighbors_threshold}. Skipping."
+                    )
+                    skipped_nodes.add(original_node)
+                    continue
+
+                for prop, neighbors in neighbors_data.items():
+                    is_inverse = prop.startswith("(-)")
+                    clean_prop = prop[3:] if is_inverse else prop
+
+                    if clean_prop in self.excluded_properties:
+                        continue
+
+                    for neighbor in neighbors:
+                        if neighbor in self.excluded_nodes or neighbor in skipped_nodes:
+                            continue
+
+                        if self._included_prefixes_tuple and not neighbor.startswith(
+                            self._included_prefixes_tuple
+                        ):
+                            continue
+
+                        if self._excluded_prefixes_tuple and neighbor.startswith(
+                            self._excluded_prefixes_tuple
+                        ):
+                            continue
+
+                        if is_inverse:
+                            triplets.append((neighbor, clean_prop, original_node))
+                        else:
+                            triplets.append((original_node, clean_prop, neighbor))
+
+        if triplets:
+            yield triplets
+
+    def __del__(self):
+        try:
+            self.env.close()
+        except Exception:
+            pass

graph_seeder/wrapper/sparql/BaseClient.py

@@ -0,0 +1,23 @@
+from abc import ABC, abstractmethod
+
+
+class BaseClient(ABC):
+    """Abstract base class for Knowledge Graph querying clients."""
+
+    def __init__(self, config: dict):
+        self.hub_pairs_batch_size = config.get("extraction", {}).get(
+            "hub_pairs_batch_size", 128
+        )
+
+    @abstractmethod
+    def query(
+        self, query_string: str, silent: bool = False, retries: int = None
+    ) -> list[dict]:
+        """
+        Execute a query and return results in the standard SPARQL JSON bindings format:
+        [
+            {"subject": {"value": "http://..."}, "property": {"value": "..."}},
+            ...
+        ]
+        """
+        pass

graph_seeder/wrapper/sparql/GraphWrapper.py

@@ -0,0 +1,269 @@
+from collections.abc import Generator
+
+from graph_seeder.wrapper.NeighborhoodWrapper import NeighborhoodWrapper
+import logging
+from graph_seeder.wrapper.sparql.SparqlQueryBuilder import SparqlQueryBuilder
+from graph_seeder.utils.URIManager import URIManager
+from graph_seeder.wrapper.sparql.BaseClient import BaseClient
+
+logger = logging.getLogger("subgraph")
+
+
+class GraphWrapper(NeighborhoodWrapper):
+    def __init__(self, uri_manager: URIManager, config: dict, client: BaseClient):
+        super().__init__(uri_manager, config)
+
+        self.client = client
+
+        extraction_config: dict = config.get("extraction", {})
+
+        max_neighbors_val = extraction_config.get("max_neighbors_threshold")
+        self.max_neighbors_threshold = (
+            max_neighbors_val if max_neighbors_val is not None else float("inf")
+        )
+
+        pagination_val = extraction_config.get("hub_pagination_threshold")
+        self.hub_pagination_threshold = (
+            pagination_val if pagination_val is not None else float("inf")
+        )
+
+        prop_skip_val = extraction_config.get("min_triplets_per_property")
+        self.min_triplets_per_property = (
+            prop_skip_val if prop_skip_val is not None else 0
+        )
+
+        self.request_logging = config.get("debug", {}).get("request_logging", False)
+
+        self.query_builder = SparqlQueryBuilder(
+            self.uri_manager,
+            config["graph_filters"],
+        )
+
+        self.batch_size = extraction_config.get("batch_size", 20)
+
+    def _execute_with_dichotomy(
+        self, items, build_query
+    ) -> Generator[list[dict], None, None]:
+        """Recursively execute a query with dichotomy splitting on failure.
+
+        Args:
+            items: List of items to include in the query (e.g., nodes or node-property pairs).
+            build_query: Function that takes a list of items and returns a SPARQL query string
+
+        Yields:
+            Lists of (subject, property, object) tuples from successful query executions.
+        """
+        if not items:
+            return
+
+        query = build_query(items)
+        if self.request_logging:
+            logger.info(f"Query for {len(items)} items:\n{query.strip()}")
+
+        try:
+            rows = self.client.query(query)
+            if rows:
+                yield rows
+        except RuntimeError:
+            if len(items) <= 1:
+                logger.error(f"Cannot split further ({len(items)} item(s)), skipping.")
+                return
+            mid = len(items) // 2
+            logger.warning(
+                f"Query failed for {len(items)} items, splitting into halves."
+            )
+            yield from self._execute_with_dichotomy(items[:mid], build_query)
+            yield from self._execute_with_dichotomy(items[mid:], build_query)
+
+    def check_seeds_validity(self, seeds: list[str]) -> dict[str, bool]:
+        """Return a dict mapping each seed to a boolean indicating if it's valid."""
+        results = {}
+
+        for i in range(0, len(seeds), self.batch_size):
+            batch = seeds[i : i + self.batch_size]
+
+            batch_results = self._check_batch_strict(batch)
+            results.update(batch_results)
+
+        return results
+
+    def _check_batch_strict(self, batch: list[str]) -> dict[str, bool]:
+        if not batch:
+            return {}
+
+        query = self.query_builder.build_checking_validity_query(batch)
+        if self.request_logging:
+            logger.info(f"Checking seeds with query:\n{query.strip()}")
+
+        try:
+            rows = self.client.query(query, silent=True, retries=1)
+            results = {r["node"]["value"]: True for r in rows}
+            return results
+
+        except RuntimeError:
+            if len(batch) == 1:
+                return {batch[0]: False}
+
+            mid = len(batch) // 2
+            left_results = self._check_batch_strict(batch[mid:])
+            right_results = self._check_batch_strict(batch[:mid])
+
+            left_results.update(right_results)
+
+            return left_results
+
+    def count_neighbors(self, seeds: list[str]) -> dict[str, int]:
+        """Return a dict mapping each seed to its total number of neighbors."""
+        results = {}
+
+        for i in range(0, len(seeds), self.batch_size):
+            batch = seeds[i : i + self.batch_size]
+
+            batch_results = self._get_hub_seeds(batch)
+            for r in batch_results:
+                results.update(r)
+
+        return results
+
+    def _get_hub_seeds(self, seeds: list[str]) -> Generator[dict[str, int], None, None]:
+        """Return the list of hub seeds among the given seeds."""
+        for raw_rows in self._execute_with_dichotomy(
+            seeds, self.query_builder.build_total_neighbors_query
+        ):
+            for r in raw_rows:
+                yield {r["node"]["value"]: int(r["total"]["value"])}
+
+    def get_neighborhood(
+        self, nodes: list[str]
+    ) -> Generator[list[tuple[str, str, str]], None, None]:
+        """Fetch one-hop neighbors using property occurrence to decide strategy."""
+        if not nodes:
+            return None
+
+        for i in range(0, len(nodes), self.batch_size):
+            batch_nodes = nodes[i : i + self.batch_size]
+
+            prop_data = self._get_properties_statistics(batch_nodes)
+
+            node_totals: dict[str, int] = {}
+            node_to_props: dict[str, list[tuple[str, int]]] = {}
+
+            for node, prop, count in prop_data:
+                node_totals[node] = node_totals.get(node, 0) + count
+                if node not in node_to_props:
+                    node_to_props[node] = []
+                node_to_props[node].append((prop, count))
+
+            safe_nodes = []
+            hub_pairs_to_chunk: list[tuple[str, str, int]] = []
+
+            for node in batch_nodes:
+                total = node_totals.get(node, 0)
+                is_force_paginate = node in self.forced_hubs
+
+                if total > self.max_neighbors_threshold and not is_force_paginate:
+                    logger.warning(
+                        f"Node {node} has total {total} neighbors, which exceeds the skip threshold of {self.max_neighbors_threshold}. Skipping."
+                    )
+                elif total > self.hub_pagination_threshold or is_force_paginate:
+                    for prop, count in node_to_props.get(node, []):
+                        if count >= self.min_triplets_per_property:
+                            hub_pairs_to_chunk.append((node, prop, count))
+                else:
+                    safe_nodes.append(node)
+
+            if safe_nodes:
+                for raw_rows in self._execute_with_dichotomy(
+                    safe_nodes, self.query_builder.build_neighborhood_query
+                ):
+                    yield [
+                        (
+                            r["subject"]["value"],
+                            r["property"]["value"],
+                            r["object"]["value"],
+                        )
+                        for r in raw_rows
+                    ]
+
+            if hub_pairs_to_chunk:
+                for batch in self._chunk_hub_properties(hub_pairs_to_chunk):
+                    for raw_rows in self._execute_with_dichotomy(
+                        batch, self.query_builder.build_hub_neighborhood_query
+                    ):
+                        yield [
+                            (
+                                r["subject"]["value"],
+                                r["property"]["value"],
+                                r["object"]["value"],
+                            )
+                            for r in raw_rows
+                        ]
+        return None
+
+    def _get_properties_statistics(
+        self, nodes: list[str]
+    ) -> list[tuple[str, str, int]]:
+        """Execute the prop occurrence query and return raw results.
+
+        Args:
+            nodes: List of node IDs to analyze.
+        Returns:
+            List of (node, property, count) tuples from the query results.
+        """
+        all_stats = []
+        for raw_rows in self._execute_with_dichotomy(
+            nodes, self.query_builder.build_prop_occurrence_query
+        ):
+            all_stats.extend(
+                [
+                    (
+                        r["entity"]["value"],
+                        r["property"]["value"],
+                        int(r["count"]["value"]),
+                    )
+                    for r in raw_rows
+                ]
+            )
+        return all_stats
+
+    def _chunk_hub_properties(
+        self, prop_counts: list[tuple[str, str, int]]
+    ) -> list[list[tuple[str, str]]]:
+        """
+        Group (node, prop) pairs into batches using Largest-Smallest strategy.
+
+        Args:
+            prop_counts: List of (node, property, count) tuples sorted by count descending.
+        Returns:
+            List of batches, where each batch is a list of (node, property) pairs.
+        """
+        if not prop_counts:
+            return []
+
+        max_pairs_per_batch = self.client.hub_pairs_batch_size
+
+        sorted_props = sorted(prop_counts, key=lambda x: x[2], reverse=True)
+        batches = []
+
+        left = 0
+        right = len(sorted_props) - 1
+
+        while left <= right:
+            current_batch = []
+            node_l, prop_l, count_l = sorted_props[left]
+            current_batch.append((node_l, prop_l))
+            current_sum = count_l
+            left += 1
+
+            while left <= right and len(current_batch) < max_pairs_per_batch:
+                node_r, prop_r, count_r = sorted_props[right]
+                if current_sum + count_r <= self.max_neighbors_threshold:
+                    current_batch.append((node_r, prop_r))
+                    current_sum += count_r
+                    right -= 1
+                else:
+                    break
+
+            batches.append(current_batch)
+
+        return batches