graph-seeder 1.0.0.dev0__py3-none-any.whl

graph_seeder/wrapper/sparql/SparqlQueryBuilder.py

@@ -0,0 +1,175 @@
+import re
+from graph_seeder.utils.URIManager import URIManager
+
+
+class SparqlQueryBuilder:
+    def __init__(
+        self,
+        uri_manager: URIManager,
+        graph_filters: dict,
+        context: dict = None,
+    ):
+        self.uri_manager = uri_manager
+        self.context = context or {}
+        self.include_uri_prefixes = graph_filters.get("include_uri_prefixes", [])
+        self.exclude_uri_prefixes = graph_filters.get("exclude_uri_prefixes", [])
+
+        self.prop_filter = self.format_prop_filter(
+            set(graph_filters.get("exclude_properties", []))
+        )
+        self.node_filter_obj = self.format_node_filter(
+            set(graph_filters.get("exclude_nodes", [])), "object"
+        )
+        self.node_filter_subj = self.format_node_filter(
+            set(graph_filters.get("exclude_nodes", [])), "subject"
+        )
+
+    def build_checking_validity_query(self, seeds: list[str]) -> str:
+        values_str = self.format_values(seeds)
+        return f"""
+        {self.format_namespaces()}
+        SELECT DISTINCT ?node 
+        WHERE {{
+            VALUES ?node {{ {values_str} }}
+            ?node ?p ?o .
+        }}
+        """
+
+    def build_total_neighbors_query(self, nodes: list[str]) -> str:
+        """Build a SPARQL query to count total neighbors for each node."""
+        values_str = self.format_values(nodes)
+        return f"""
+        {self.format_namespaces()}
+        SELECT ?node (COUNT(?property) AS ?total) WHERE {{
+            VALUES ?node {{ {values_str} }}
+            {{
+                ?node ?property ?object .
+            }}
+            UNION
+            {{
+                ?subject ?property ?node .
+            }}
+        }} GROUP BY ?node
+        """
+
+    def _build_prop_occurrence_query(self, values_str: str) -> str:
+        return f"""
+        {self.format_namespaces()}
+        SELECT ?entity ?property (COUNT(?property) AS ?count) WHERE {{
+            VALUES ?entity {{ {values_str} }}
+            {{ ?entity ?property ?object . }}
+            UNION 
+            {{ ?subject ?property ?entity . }}
+        }} 
+        GROUP BY ?entity ?property
+        ORDER BY ?entity DESC(?count)
+        """
+
+    def _build_neighborhood_query(self, values_str: str) -> str:
+        return f"""
+        {self.format_namespaces()}
+        SELECT ?subject ?property ?object WHERE {{
+            VALUES ?node {{ {values_str} }}
+          {{
+            ?node ?property ?object .
+            BIND (?node AS ?subject)
+            {self.node_filter_obj}
+            {self.prop_filter}
+            {self.format_entity_filter("object")}
+          }}
+          UNION
+          {{
+            ?subject ?property ?node .                                                                                                        
+            BIND (?node AS ?object)
+            {self.node_filter_subj}
+            {self.prop_filter}
+            {self.format_entity_filter("subject")}
+          }}
+        }}
+        """
+
+    def _build_hub_neighborhood_query(self, values_pairs_str: str) -> str:
+        return f"""
+        {self.format_namespaces()}
+        SELECT ?subject ?property ?object WHERE {{
+            VALUES (?node ?property) {{ {values_pairs_str} }}
+          {{
+            ?node ?property ?object .
+            BIND (?node AS ?subject)
+            {self.node_filter_obj} {self.prop_filter} {self.format_entity_filter("object")}
+          }}
+          UNION
+          {{
+            ?subject ?property ?node .
+            BIND (?node AS ?object)
+            {self.node_filter_subj} {self.prop_filter} {self.format_entity_filter("subject")}
+          }}
+        }}
+        """
+
+    def build_neighborhood_query(self, nodes: list[str]) -> str:
+        return self._build_neighborhood_query(self.format_values(nodes))
+
+    def build_prop_occurrence_query(self, nodes: list[str]) -> str:
+        return self._build_prop_occurrence_query(self.format_values(nodes))
+
+    def build_hub_neighborhood_query(self, pairs: list[tuple[str, str]]) -> str:
+        return self._build_hub_neighborhood_query(self.format_values_pairs(pairs))
+
+    def format_namespaces(self) -> str:
+        return "\n".join(
+            f"PREFIX {p}: <{uri}>" for p, uri in self.uri_manager.namespaces.items()
+        )
+
+    def format_node(self, node_id: str) -> str:
+        if node_id.startswith("<") and node_id.endswith(">"):
+            return node_id
+
+        clean_node = node_id.replace("\\'", "'").replace('\\"', '"')
+        compressed_uri = self.uri_manager.compress_uri(clean_node)
+
+        if compressed_uri.startswith("<"):
+            return compressed_uri
+
+        if ":" in compressed_uri and not compressed_uri.startswith("http"):
+            prefix, local_name = compressed_uri.split(":", 1)
+            if re.match(r"^[A-Za-z0-9_-]+$", local_name):
+                return compressed_uri
+
+        if clean_node.startswith("http://") or clean_node.startswith("https://"):
+            return f"<{clean_node}>"
+
+        return clean_node
+
+    def format_values_pairs(self, pairs: list[tuple[str, str]]) -> str:
+        return " ".join(
+            f"({self.format_node(n)} {self.format_node(p)})" for n, p in pairs
+        )
+
+    def format_values(self, entity_ids: list[str]) -> str:
+        return " ".join(self.format_node(e) for e in entity_ids)
+
+    def format_prop_filter(self, properties: set[str]) -> str:
+        if not properties:
+            return ""
+        return f"FILTER (?property NOT IN ({', '.join(self.format_node(p) for p in properties)}))"
+
+    def format_node_filter(self, nodes: set[str], var_name: str) -> str:
+        if not nodes:
+            return ""
+        return f"FILTER (?{var_name} NOT IN ({', '.join(self.format_node(n) for n in nodes)}))"
+
+    def format_entity_filter(self, var_name: str) -> str:
+        conditions = [f"isIRI(?{var_name})", f"!isBlank(?{var_name})"]
+        if self.include_uri_prefixes:
+            checks = [
+                f'STRSTARTS(STR(?{var_name}), "{b}")' for b in self.include_uri_prefixes
+            ]
+            conditions.append(f"({' || '.join(checks)})")
+        if self.exclude_uri_prefixes:
+            exclusions = [
+                f'!STRSTARTS(STR(?{var_name}), "{b}")'
+                for b in self.exclude_uri_prefixes
+            ]
+            conditions.append(f"({' && '.join(exclusions)})")
+        return f"FILTER({' && '.join(conditions)})"

graph_seeder/wrapper/sparql/client/SparqlClient.py

@@ -0,0 +1,118 @@
+import logging
+import time
+from urllib.error import HTTPError
+from SPARQLWrapper import JSON, SPARQLWrapper
+from SPARQLWrapper.SPARQLExceptions import QueryBadFormed, URITooLong
+from graph_seeder.wrapper.sparql.BaseClient import BaseClient
+
+logger = logging.getLogger("subgraph")
+
+
+class SparqlClient(BaseClient):
+    def __init__(self, cfg: dict) -> None:
+        super().__init__(cfg)
+        client_cfg = cfg.get("client", {})
+
+        endpoint = client_cfg.get("endpoint")
+        if not endpoint:
+            raise ValueError("SPARQL endpoint URL must be provided in config.")
+
+        user_agent = client_cfg.get("user_agent")
+        if not user_agent:
+            self.sparql = SPARQLWrapper(endpoint)
+        else:
+            self.sparql = SPARQLWrapper(endpoint, agent=user_agent)
+        self.sparql.setTimeout(client_cfg.get("timeout", 50))
+        self.sparql.setReturnFormat(JSON)
+
+        self.retry_attempts = client_cfg.get("retry_attempts", 3)
+        self.retry_delay = client_cfg.get("retry_delay", 3)
+        self.request_delay = client_cfg.get("request_delay", 1)
+        self.rate_limit_wait = client_cfg.get("rate_limit_wait", 60)
+
+    def _log_failed_query(self, query: str, error: Exception) -> None:
+        err_str = str(error)
+
+        error_lines = err_str.split("\n")
+        short_msg = next(
+            (line for line in error_lines if "Virtuoso" in line), error_lines[0]
+        )
+
+        logger.warning(
+            f"SPARQL Error: {short_msg}\n See failed_queries.log for more details."
+        )
+        logger.debug(f"Full error response:\n{error}")
+        try:
+            with open("failed_queries.log", "a", encoding="utf-8") as f:
+                timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
+                f.write(
+                    f"=== {timestamp} | {type(error).__name__} ===\n{error}\n{query}\n{'=' * 40}\n\n"
+                )
+        except Exception as file_err:
+            logger.error(f"Could not write to failed_queries.log: {file_err}")
+
+    def query(
+        self, sparql_query: str, silent: bool = False, retries: int = None
+    ) -> list[dict]:
+        """Execute a SPARQL query with retries.
+
+        Args:
+            sparql_query: Full SPARQL query string.
+            silent: If True, suppress logging of failed queries.
+            retries: Number of retry attempts.
+
+        Returns:
+            List of binding dicts from the ``results.bindings`` array.
+
+        Raises:
+            RuntimeError: When all retry attempts are exhausted.
+        """
+        retries = retries if retries is not None else self.retry_attempts
+        self.sparql.setQuery(sparql_query)
+
+        for attempt in range(1, retries + 1):
+            try:
+                results = self.sparql.query().convert()
+                if self.request_delay > 0:
+                    time.sleep(self.request_delay)
+                return results["results"]["bindings"]
+
+            except (URITooLong, QueryBadFormed) as e:
+                if not silent:
+                    self._log_failed_query(sparql_query, e)
+                raise RuntimeError(
+                    f"Query rejected by endpoint ({type(e).__name__}): {e}. "
+                    f"See failed_queries.log for the full query."
+                ) from e
+
+            except HTTPError as e:
+                if e.code == 429:
+                    logger.warning(
+                        f"Rate limited (429). Waiting {self.rate_limit_wait}s…"
+                    )
+                    time.sleep(self.rate_limit_wait)
+                    continue
+                logger.warning(
+                    f"HTTP {e.code} on attempt {attempt}/{retries}: {e}. "
+                    f"Retrying in {self.retry_delay}s…"
+                )
+                time.sleep(self.retry_delay)
+
+            except Exception as e:
+                if attempt < retries:
+                    logger.warning(
+                        f"Attempt {attempt}/{retries} failed: {e}. Retrying…"
+                    )
+                    time.sleep(self.retry_delay)
+                else:
+                    if not silent:
+                        self._log_failed_query(sparql_query, e)
+                    raise RuntimeError(
+                        f"SPARQL query failed after {retries} attempts: {e}. "
+                        f"See failed_queries.log for the full query."
+                    ) from e
+
+        raise RuntimeError(
+            f"SPARQL query failed after {retries} attempts (retries exhausted). "
+            f"See failed_queries.log for details."
+        )

graph_seeder/wrapper/sparql/client/TurtleClient.py

@@ -0,0 +1,47 @@
+import logging
+import rdflib
+from graph_seeder.wrapper.sparql.BaseClient import BaseClient
+
+logger = logging.getLogger("subgraph")
+
+
+class TurtleClient(BaseClient):
+    """Client that loads a local Turtle file into memory and allows SPARQL querying over it."""
+
+    def __init__(self, cfg: dict) -> None:
+        super().__init__(cfg)
+        data: dict = cfg.get("data", {})
+        self.file_path = data.get("turtle_path")
+
+        if not self.file_path:
+            raise ValueError("file_path must be provided in config for TurtleClient.")
+
+        logger.info(
+            f"Loading Turtle graph from {self.file_path}. This may take a moment..."
+        )
+        self.graph = rdflib.Graph()
+        self.graph.parse(self.file_path, format="turtle")
+        logger.info(f"Successfully loaded {len(self.graph)} triples.")
+
+        self.optimal_batch_size = 500
+
+    def query(
+        self, sparql_query: str, silent: bool = False, retries: int = None
+    ) -> list[dict]:
+        """Execute the SPARQL query on the local rdflib graph and format the output."""
+        try:
+            results = self.graph.query(sparql_query)
+            bindings = []
+
+            for row in results:
+                binding = {}
+                for var in results.vars:
+                    val = row[var]
+                    if val is not None:
+                        binding[str(var)] = {"value": str(val)}
+                bindings.append(binding)
+
+            return bindings
+        except Exception as e:
+            logger.error(f"Failed to execute local Turtle query: {e}")
+            raise RuntimeError(f"Turtle query failed: {e}") from e

graph_seeder-1.0.0.dev0.dist-info/METADATA

@@ -0,0 +1,191 @@
+Metadata-Version: 2.4
+Name: graph-seeder
+Version: 1.0.0.dev0
+Summary: A powerful tool to extract and densify subgraphs from Knowledge Graphs via SPARQL or LMDB, with different extraction strategies.
+Requires-Python: >=3.9
+Requires-Dist: lmdb>=2.2.0
+Requires-Dist: networkx<4.0.0,>=3.2.1
+Requires-Dist: pandas<3.0.0,>=2.3.3
+Requires-Dist: rdflib>=7.6.0
+Requires-Dist: requests>=2.32.5
+Requires-Dist: rich>=15.0.0
+Requires-Dist: sparqlwrapper>=2.0.0
+Requires-Dist: urllib3>=2.6.3
+Description-Content-Type: text/markdown
+
+# Graph densifier
+
+Graph Densifier is a collection of tools that can be used for enriching, analyzing, and extracting subgraphs from
+knowledge graphs represented as triplet datasets.
+
+It offers the following functions:
+
+- **Densify graphs** by enriching an existing knowledge graph with additional Wikidata triplets between known entities.
+- **Compute statistics** to analyze the graph's composition and connectivity.
+- **Extract paths from Wikidata** dynamically by finding connections between pairs of entities.
+- **Extract local subgraphs** from an existing dataset using shortest paths or neighborhood expansion around seed
+  entities.
+
+## Installation
+
+Follow these steps to set up the project locally:
+
+### 1. Clone the repository
+
+```bash
+git clone https://github.com/Wimmics/graph-densifier.git
+cd graph-densifier
+```
+
+### 2. Install dependencies
+
+#### Option A (recommended): using uv
+
+We recommend using uv for fast and reliable dependency management.
+
+- [Install uv](https://docs.astral.sh/uv/#installation) by following the official guide
+- Then run:
+
+```bash
+uv sync
+```
+
+#### Option B: using pip
+
+If you prefer not to use uv, you can install dependencies with pip:
+
+```bash
+pip install -r requirements.txt
+```
+
+### 3. Environment configuration
+
+> [!note]
+> To avoid being rate-limited or blocked when querying Wikidata, you should configure a user identity.
+> - Create a `.env` file at the root of the project
+> - Add the following line to the file with your Wikidata username:
+> ```bash
+> USER_AGENT="graph_densify/1.0 (contact: wikidata_username)"
+> ```
+> While this step is not strictly required to run the project, it is **recommended**. Without it, requests to Wikidata
+> may be throttled or blocked during large runs, which can interrupt the graph densification and path extraction
+> processes.
+
+## Usage
+
+The project provides four main scripts:
+
+1. **graph_densify.py** – enrich a local graph with additional Wikidata triplets.
+2. **statistics.py** – compute statistics for a triplet dataset.
+3. **subgraph_extract.py** – query Wikidata to find paths between entity pairs.
+4. **hashmap_extract_subgraph.py** – extract relevant subgraphs from a local CSV graph.
+
+## 1. Graph Densification (`graph_densify.py`)
+
+This script enriches the input graph by querying Wikidata for additional relationships between entities already present
+in the graph. It identifies all unique entities in the `subject` and `object` columns and adds any newly discovered
+direct relations to the dataset.
+
+### Command
+
+```bash
+python src/graph_densify.py --input path/to/input.csv --output path/to/output.csv
+```
+
+## 2. Graph Statistics (`statistics.py`)
+
+This script computes descriptive statistics for a triplet dataset and generates a summary CSV file in the `stat/`
+directory.
+
+### Command
+
+```bash
+python src/statistics.py --input path/to/graph.csv
+```
+
+### Computed Statistics
+
+| Metric                        | Description                                        |
+|-------------------------------|----------------------------------------------------|
+| `total_triplets`              | Total number of triplets                           |
+| `unique_subjects`             | Number of unique subjects                          |
+| `unique_predicates`           | Number of unique predicates                        |
+| `unique_objects`              | Number of unique objects                           |
+| `unique_entities`             | Unique entities across subjects and objects        |
+| `unique_subject_object_pairs` | Distinct `(subject, object)` pairs                 |
+| `connected_components`        | Number of weakly connected components in the graph |
+
+## 3. Wikidata Path Extraction (`subgraph_extract.py`)
+
+This script takes a list of entity pairs and dynamically queries Wikidata to find a short path (not necessarily the
+shortest) between them. It outputs the discovered path triplets as a CSV and saves the explored network as a `.gpickle`
+graph file.
+
+### Command
+
+```bash
+python src/subgraph_extract.py --input path/to/pairs.csv --output path/to/extracted_paths.csv
+```
+
+---
+
+## 4. Local Subgraph Extraction (`hashmap_extract_subgraph.py`)
+
+This script extracts subgraphs from a **local** graph dataset (CSV) using one of the two modes:
+
+### Mode A — Shortest paths between seed/target pairs
+
+Extracts all shortest paths between specified source-target entity pairs.
+
+```bash
+python src/hashmap_extract_subgraph.py \
+    --sub_graph path/to/main_graph.csv \
+    --seed_target_pairs path/to/pairs.csv
+```
+
+### Mode B — Radius around seed nodes
+
+Extracts all nodes within a specified number of hops (default: 2) from a list of seed entities.
+
+```bash
+python src/hashmap_extract_subgraph.py \
+    --sub_graph path/to/main_graph.csv \
+    --seeds_only path/to/seeds.csv \
+    --max_length 2
+```
+
+**Output:** The extracted subgraph is saved by default to `data/extracted_subgraph.csv`.
+
+---
+
+## Dataset Structure
+
+All datasets are expected to be provided as CSV files.
+
+### Main Graph Dataset
+
+Must contain three columns representing a knowledge graph triplet:
+
+| subject | predicate | object |
+|---------|-----------|--------|
+| Q937    | P36       | Q90    |
+| Q90     | P17       | Q142   |
+
+### Seed-Target Pair Dataset
+
+Used for finding paths between specific entities. Must contain two columns:
+
+| subject | object |
+|---------|--------|
+| Q937    | Q304   |
+| Q90     | Q183   |
+
+### Seed-Only Dataset
+
+Used for neighborhood expansion. Must contain one column representing the seed entity (the column can be named `seed` or
+be the first column):
+
+| seed |
+|------|
+| Q937 |
+| Q90  |

graph_seeder-1.0.0.dev0.dist-info/RECORD

@@ -0,0 +1,28 @@
+graph_seeder/GraphSeeder.py,sha256=3ODCN8IyUZt3NB04n7FUKTgNu9_56IMomuqYLLHzRSA,1457
+graph_seeder/SubgraphExtractor.py,sha256=v5ajBQldM9u2BChTPKpeqqO61OH4fuLhRL9A3p36nJQ,14539
+graph_seeder/configs/dbpedia_default.json,sha256=Ds-42oRLvWW2f0LP0m9yf3n_XvDwym4ol9_2BxMQ3sI,1802
+graph_seeder/configs/default.json,sha256=7d2DUfcMvodMcxEIoI0CBLZareJ3aSet9pakT0NHV4Y,1214
+graph_seeder/configs/europeana_default.json,sha256=mS_c9CBz1CDvSUZt_N4pwLs-M7CWoX9PqrKZMl_6hSU,1501
+graph_seeder/configs/pgxlod_default.json,sha256=YGTGXM02uw87LR85JMW1uDfoQMgh7KyjWzXpOSLs_xc,1339
+graph_seeder/configs/wikidata_default.json,sha256=lZ4q5_RPnwK_dduuYSCvm4E4_yjynPJOjwHGl0occAY,2474
+graph_seeder/densification/GraphConnector.py,sha256=1SsXuKNHRZbIOixerEHgPTREWvuy8_OC9EPuQwExzsY,4067
+graph_seeder/extraction/ExtractionStrategy.py,sha256=IIlnk_7RxLdquyNQSJIhuqL_P2BDSQfBp5WDLiXq21k,2409
+graph_seeder/extraction/BFS/BFS.py,sha256=r-QidFmV9oR1phYnVqDgim4KS8RcnrteQa6c6N80oJw,7301
+graph_seeder/extraction/Hop/HopExpansion.py,sha256=bKikLNwrL0sseXcPPI8brb5p64yN7u3exEhg7Nw2Q8I,3514
+graph_seeder/utils/ConsoleUI.py,sha256=FIggOlXAecfZvSM361VlqHpQTD_YtXrK0Pgz_iyOPhg,10153
+graph_seeder/utils/Factory.py,sha256=iz1TI0YHkvKz9VRhR4cfpxNZAVzjCvbJNZBM_0yZfgY,2549
+graph_seeder/utils/GraphExporter.py,sha256=GAF_MLv3AnGNZEQyMjLXpE4qVGaufnHanjAE5etFHaY,3016
+graph_seeder/utils/GraphStatistics.py,sha256=rczbakVcSjt9itTBq6WnYDIv9OhfFiqCiqAfK954cSk,1023
+graph_seeder/utils/URIManager.py,sha256=j-5HD1oMDyrS7dtdXTWWHyo-M-iETp3hRYBTTa6R198,3445
+graph_seeder/utils/utils.py,sha256=AQXXCcUGSP_4eLYUU6u_EKsFZhoQh7y22TsA-IM4OF8,6759
+graph_seeder/wrapper/NeighborhoodWrapper.py,sha256=VYrJHQOhXchgO0YqK3zldHeM3KJUNxR_CYNMCuI9zxU,1766
+graph_seeder/wrapper/hashmap/HashMapWrapper.py,sha256=9KOZ61XmNv0XC_hn9VcdhBXmRL4iU1aw4LZG33pJyms,4823
+graph_seeder/wrapper/sparql/BaseClient.py,sha256=3HShDXLEYwk7QARVWVQwFA7kbVuakKN1_96Yj-3UzII,684
+graph_seeder/wrapper/sparql/GraphWrapper.py,sha256=6_FLk8nDpQBNxZ7AY-U39OvuAViZl5K4IBNFOF5yUQM,10136
+graph_seeder/wrapper/sparql/SparqlQueryBuilder.py,sha256=L7mLnq1k5hV8xIfqt5NlgCHP6dY5p3lWgN0G3dRIGgk,6542
+graph_seeder/wrapper/sparql/client/SparqlClient.py,sha256=HMUBwIbMrHj-bdVW8zFi9hwsJcZP8-P_PuK9t3YpzT0,4670
+graph_seeder/wrapper/sparql/client/TurtleClient.py,sha256=x7RXsn0zrC9sagZGIUTM97SorX5ELqRHIF-BXfrpuVs,1685
+graph_seeder-1.0.0.dev0.dist-info/METADATA,sha256=E9IKc2FwiqGrx8KmSfq4n8NDf55qDZT7KIF46hbXs44,6133
+graph_seeder-1.0.0.dev0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+graph_seeder-1.0.0.dev0.dist-info/entry_points.txt,sha256=eM5WRu0uS9fNN_nmZ09zHX1p5UbsHrFi8A9yE-B3X9w,63
+graph_seeder-1.0.0.dev0.dist-info/RECORD,,

graph_seeder-1.0.0.dev0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

graph_seeder-1.0.0.dev0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+graph-seeder = graph_seeder.GraphSeeder:main