pyomnigraph 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/PKG-INFO +1 -1
  2. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/docs/index.md +1 -1
  3. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/mkdocs.yml +1 -1
  4. pyomnigraph-0.2.2/omnigraph/__init__.py +1 -0
  5. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/omnigraph_cmd.py +2 -1
  6. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/omniserver.py +1 -0
  7. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/rdf_dataset.py +45 -3
  8. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/rdfdump_cmd.py +14 -0
  9. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/resources/examples/datasets.yaml +37 -1
  10. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/blazegraph.py +64 -0
  11. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/jena.py +62 -2
  12. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/qlever.py +66 -0
  13. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/sparql_server.py +35 -4
  14. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/scripts/quickstart +6 -3
  15. pyomnigraph-0.2.2/tests/test_rdf_dataset_params.py +58 -0
  16. pyomnigraph-0.2.2/tests/test_upload.py +98 -0
  17. pyomnigraph-0.2.1/omnigraph/__init__.py +0 -1
  18. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/.github/workflows/build.yml +0 -0
  19. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/.github/workflows/upload-to-pypi.yml +0 -0
  20. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/.gitignore +0 -0
  21. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/.project +0 -0
  22. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/.pydevproject +0 -0
  23. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/AGENTS.md +0 -0
  24. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/LICENSE +0 -0
  25. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/README.md +0 -0
  26. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/basecmd.py +0 -0
  27. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/ominigraph_paths.py +0 -0
  28. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/rdfdump.py +0 -0
  29. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/resources/__init__.py +0 -0
  30. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/resources/examples/endpoints.yaml +0 -0
  31. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/resources/examples/prefixes.yaml +0 -0
  32. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/resources/examples/queries.yaml +0 -0
  33. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/resources/examples/royals.ttl +0 -0
  34. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/resources/examples/servers.yaml +0 -0
  35. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/server_config.py +0 -0
  36. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/__init__.py +0 -0
  37. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/graphdb.py +0 -0
  38. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/millenniumdb.py +0 -0
  39. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/oxigraph.py +0 -0
  40. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/stardog.py +0 -0
  41. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/virtuoso.py +0 -0
  42. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/software.py +0 -0
  43. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/version.py +0 -0
  44. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/pyproject.toml +0 -0
  45. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/scripts/blackisort +0 -0
  46. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/scripts/doc +0 -0
  47. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/scripts/gov-quickstart +0 -0
  48. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/scripts/install +0 -0
  49. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/scripts/release +0 -0
  50. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/scripts/test +0 -0
  51. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/tests/__init__.py +0 -0
  52. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/tests/basetest.py +0 -0
  53. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/tests/test_omniserver.py +0 -0
  54. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/tests/test_qlever.py +0 -0
  55. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/tests/test_rdfdump.py +0 -0
  56. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/tests/test_software_list.py +0 -0
  57. {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/tests/test_sparql_server.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pyomnigraph
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: pyomnigraph: Unified Python interface for multiple graph databases (SPARQL, Neo4j, Weaviate)
5
5
  Project-URL: Home, https://github.com/WolfgangFahl/omnigraph
6
6
  Project-URL: Documentation, https://wiki.bitplan.com/index.php/omnigraph
@@ -1,4 +1,4 @@
1
- # omnigraph API Documentation
1
+ # pyomnigraph API Documentation
2
2
 
3
3
  ::: omnigraph
4
4
  options:
@@ -1,4 +1,4 @@
1
- site_name: omnigraph API Documentation
1
+ site_name: pyomnigraph API Documentation
2
2
  theme:
3
3
  name: material
4
4
  plugins:
@@ -0,0 +1 @@
1
+ __version__ = "0.2.2"
@@ -171,7 +171,8 @@ class OmnigraphCmd(BaseCmd):
171
171
  handled = False
172
172
  if cmds:
173
173
  for cmd in cmds:
174
- if cmd == "load":
174
+ # issue #38: upload needs the per-dataset dumps_dir just like load
175
+ if cmd in ("load", "upload"):
175
176
  cmd_iterator = self.load_iterator(server)
176
177
  else:
177
178
  cmd_iterator = iter([None]) # Single iteration
@@ -70,6 +70,7 @@ class OmniServer:
70
70
  "start": lambda s: ServerCmd(title("start", s), s.start),
71
71
  "status": lambda s: ServerCmd(title("status", s), s.status_info),
72
72
  "stop": lambda s: ServerCmd(title("stop", s), s.stop),
73
+ "upload": lambda s: ServerCmd(title("native bulk upload dumps", s), s.upload_dump_files),
73
74
  "webui": lambda s: ServerCmd(title("webui", s), s.webui),
74
75
  }
75
76
  return server_cmds
@@ -8,6 +8,7 @@ from dataclasses import dataclass, field
8
8
  from typing import Dict, Optional
9
9
 
10
10
  from basemkit.yamlable import lod_storable
11
+ from lodstorage.params import Params
11
12
  from lodstorage.query import Query
12
13
  from lodstorage.sparql import SPARQL
13
14
 
@@ -26,6 +27,7 @@ class RdfDataset:
26
27
  expected_solutions: Optional[int] = None # Expected number of solutions
27
28
  select_pattern: str = "?s ?p ?o" # Basic Graph Pattern for queries
28
29
  construct_template: Optional[str] = field(default="?s ?p ?o")
30
+ params: Optional[Dict[str, str]] = None # npq parameters for {{ name }} templates (issue #36)
29
31
  prefix_sets: Optional[list] = field(default_factory=list)
30
32
  active: Optional[bool] = False
31
33
  rdf_file: Optional[str] = None # Path to local RDF file (alternative to endpoint_url)
@@ -35,10 +37,41 @@ class RdfDataset:
35
37
  select_query: Optional[Query] = field(default=None)
36
38
  sparql: Optional[SPARQL] = field(default=None)
37
39
 
38
- def __post_init__(self):
40
+ def apply_params(self, params_dict: Optional[Dict[str, str]] = None) -> None:
41
+ """
42
+ Apply npq parameters to select_pattern and construct_template.
43
+
44
+ {{ name }} templates are replaced with the values of the merged
45
+ params: the dataset's own params overridden by the given ones.
46
+ Substitution always starts from the raw templates, so CLI overrides
47
+ can be re-applied after YAML defaults. The blanket Params audit is
48
+ off - it rejects quoted VALUES lists (see issue #36); values stem
49
+ from local configuration and CLI.
50
+
51
+ Args:
52
+ params_dict: additional parameter values overriding self.params
39
53
  """
40
- Generate count_query and construct_pattern from select_pattern.
41
- Only initialize SPARQL-related fields if endpoint_url is provided.
54
+ # preserve the raw templates on first call
55
+ if not hasattr(self, "raw_templates"):
56
+ self.raw_templates = {
57
+ "select_pattern": self.select_pattern,
58
+ "construct_template": self.construct_template,
59
+ }
60
+ merged = dict(self.params) if self.params else {}
61
+ if params_dict:
62
+ merged.update(params_dict)
63
+ if merged:
64
+ for attr, raw_template in self.raw_templates.items():
65
+ if raw_template:
66
+ params = Params(raw_template, with_audit=False)
67
+ if params.has_params:
68
+ params.set(merged)
69
+ setattr(self, attr, params.apply_parameters())
70
+
71
+ def build_queries(self) -> None:
72
+ """
73
+ (Re)build count_query and select_query from the current select_pattern.
74
+ Only initializes SPARQL-related fields if endpoint_url is provided.
42
75
  """
43
76
  if self.endpoint_url:
44
77
  self.count_query = Query(
@@ -55,6 +88,13 @@ class RdfDataset:
55
88
  )
56
89
  self.sparql = SPARQL(self.endpoint_url)
57
90
 
91
+ def __post_init__(self):
92
+ """
93
+ Apply npq params and generate the queries from select_pattern.
94
+ """
95
+ self.apply_params()
96
+ self.build_queries()
97
+
58
98
  @property
59
99
  def full_name(self):
60
100
  ds_id = self.id or "?"
@@ -69,6 +109,8 @@ class RdfDataset:
69
109
  Number of solutions available from the count query
70
110
  """
71
111
  count = self.sparql.getValue(self.count_query.query, "count")
112
+ # issue #37: getValue returns the raw literal which may be a str
113
+ count = int(count)
72
114
  return count
73
115
 
74
116
  def getTryItUrl(self, database: str = "blazegraph") -> str:
@@ -10,6 +10,8 @@ import os
10
10
  import webbrowser
11
11
  from argparse import ArgumentParser, Namespace
12
12
 
13
+ from basemkit.argparse_action import StoreDictKeyPair
14
+
13
15
  from omnigraph.basecmd import BaseCmd
14
16
  from omnigraph.rdf_dataset import RdfDataset, RdfDatasets
15
17
  from omnigraph.rdfdump import RdfDumpDownloader
@@ -61,6 +63,13 @@ class RdfDumpCmd(BaseCmd):
61
63
  default=None,
62
64
  help="Maximum number of solutions/triples to download (uses dataset expected_solutions if not specified)",
63
65
  )
66
+ parser.add_argument(
67
+ "--param",
68
+ action=StoreDictKeyPair,
69
+ dest="params",
70
+ metavar="KEY=VALUE,KEY=VALUE...",
71
+ help="npq parameter values overriding the dataset's params (issue #36)",
72
+ )
64
73
  parser.add_argument("--no-progress", action="store_true", help="Disable progress bar")
65
74
  parser.add_argument("--output-path", default=".", help="Path for dump files")
66
75
  parser.add_argument("--tryit", action="store_true", help="open the try it! URL [default: %(default)s]")
@@ -97,6 +106,11 @@ class RdfDumpCmd(BaseCmd):
97
106
  """
98
107
  super().handle_args(args)
99
108
  datasets = self.datasets
109
+ if getattr(args, "params", None):
110
+ # apply CLI npq parameter overrides and rebuild the queries
111
+ for dataset in datasets.values():
112
+ dataset.apply_params(args.params)
113
+ dataset.build_queries()
100
114
  if self.args.about:
101
115
  self.about()
102
116
 
@@ -65,4 +65,40 @@ datasets:
65
65
  ?pName ?p4Name ?o4Name
66
66
  # Optional dataset description
67
67
  description: "W2306 postal code coordinates"
68
- active: false
68
+ active: false
69
+ papers_of_my_network_wikidata:
70
+ # npq (named parameterized query) example - issue #36
71
+ # ORCID is the join key; {{ orcids }} is the VALUES scaling knob;
72
+ # endpoint is late-bound - own copy answers completely and fastest,
73
+ # see the endpoint matrix in the issue
74
+ name: "Papers of my network (wikidata)"
75
+ base_url: "https://qlever-api.wikidata.dbis.rwth-aachen.de"
76
+ endpoint_url: "https://qlever-api.wikidata.dbis.rwth-aachen.de/"
77
+ database: "qlever"
78
+ select_pattern: |
79
+ VALUES ?orcid { {{ orcids }} }
80
+ ?author <http://www.wikidata.org/prop/direct/P496> ?orcid .
81
+ ?work <http://www.wikidata.org/prop/direct/P50> ?author .
82
+ ?work ?p ?o
83
+ construct_template: "?work ?p ?o"
84
+ params:
85
+ orcids: '"0000-0002-0821-6995" "0000-0003-2533-6363" "0000-0001-9879-3827" "0000-0001-6324-7164"'
86
+ description: "Papers authored by the co-author network, seeded by ORCIDs"
87
+ active: false
88
+ papers_of_my_network_dblp:
89
+ # npq companion - same {{ orcids }} param drives the dblp graph
90
+ name: "Papers of my network (dblp)"
91
+ base_url: "https://sparql.dblp.org"
92
+ endpoint_url: "https://sparql.dblp.org/sparql"
93
+ database: "qlever"
94
+ select_pattern: |
95
+ VALUES ?orcid { {{ orcids }} }
96
+ BIND(IRI(CONCAT("https://orcid.org/", ?orcid)) AS ?orcid_uri)
97
+ ?author <https://dblp.org/rdf/schema#orcid> ?orcid_uri .
98
+ ?paper <https://dblp.org/rdf/schema#authoredBy> ?author .
99
+ ?paper ?p ?o
100
+ construct_template: "?paper ?p ?o"
101
+ params:
102
+ orcids: '"0000-0002-0821-6995" "0000-0003-2533-6363" "0000-0001-9879-3827" "0000-0001-6324-7164"'
103
+ description: "Papers authored by the co-author network in dblp, same ORCID seed"
104
+ active: false
@@ -8,6 +8,8 @@ from dataclasses import dataclass
8
8
  from pathlib import Path
9
9
  import re
10
10
  import os
11
+ import shutil
12
+
11
13
  from omnigraph.server_config import ServerLifecycleState, ServerStatus
12
14
  from omnigraph.sparql_server import ServerConfig, ServerEnv, SparqlServer
13
15
 
@@ -94,6 +96,68 @@ class Blazegraph(SparqlServer):
94
96
  rwstore_path.write_text(props)
95
97
 
96
98
 
99
+ def get_dataloader_xml(self, container_path: str) -> str:
100
+ """
101
+ Build the DataLoader servlet properties XML for the given
102
+ container-side file or directory path.
103
+
104
+ Args:
105
+ container_path: path as seen inside the container (mounted /data)
106
+
107
+ Returns:
108
+ the properties XML document
109
+ """
110
+ xml = f"""<?xml version="1.0" encoding="UTF-8" standalone="no"?>
111
+ <!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
112
+ <properties>
113
+ <entry key="namespace">{self.config.dataset}</entry>
114
+ <entry key="propertyFile">/RWStore.properties</entry>
115
+ <entry key="fileOrDirs">{container_path}</entry>
116
+ <entry key="-durableQueues">true</entry>
117
+ </properties>"""
118
+ return xml
119
+
120
+ def upload_dump_files(self, file_pattern: str = None) -> int:
121
+ """
122
+ Bulk-load dump files via Blazegraph's REST DataLoader servlet, which
123
+ reads the files from the server's own filesystem (the mounted /data
124
+ directory) instead of pushing them through HTTP request bodies.
125
+
126
+ Args:
127
+ file_pattern: Glob pattern for dump files
128
+
129
+ Returns:
130
+ Number of files loaded successfully
131
+ """
132
+ container_name = self.config.container_name
133
+ files = self.get_dump_files(file_pattern)
134
+ loaded_count = 0
135
+ if not files:
136
+ self.log.log("⚠️", container_name, f"No dump files found for pattern: {file_pattern}")
137
+ else:
138
+ # stage the dumps under data_dir so the container sees them at /data/dumps
139
+ stage_dir = Path(self.config.base_data_dir) / "dumps"
140
+ stage_dir.mkdir(parents=True, exist_ok=True)
141
+ for file in files:
142
+ target = stage_dir / file.name
143
+ if not target.exists():
144
+ shutil.copy2(file, target)
145
+ xml = self.get_dataloader_xml("/data/dumps")
146
+ response = self.make_request(
147
+ "POST",
148
+ self.config.dataloader_url,
149
+ headers={"Content-Type": "application/xml"},
150
+ data=xml,
151
+ timeout=self.config.upload_timeout,
152
+ )
153
+ if response.success:
154
+ loaded_count = len(files)
155
+ self.log.log("✅", container_name, f"DataLoader loaded {loaded_count} file(s)")
156
+ else:
157
+ error_msg = str(response.error) if response.error else f"HTTP {response.response.status_code}"
158
+ self.log.log("❌", container_name, f"DataLoader failed: {error_msg}")
159
+ return loaded_count
160
+
97
161
  def status(self) -> ServerStatus:
98
162
  """
99
163
  Get server status information.
@@ -7,6 +7,8 @@ Apache Jena SPARQL support
7
7
  """
8
8
 
9
9
  from dataclasses import dataclass
10
+ from pathlib import Path
11
+ from typing import List
10
12
 
11
13
  from omnigraph.server_config import ServerLifecycleState, ServerStatus
12
14
  from omnigraph.sparql_server import ServerConfig, ServerEnv, SparqlServer
@@ -96,7 +98,65 @@ class Jena(SparqlServer):
96
98
  Returns:
97
99
  Tuple of (response, exception)
98
100
  """
99
- result,error=self.execute_update_query_with_post(update_query)
100
- return result,error
101
+ result, error = self.execute_update_query_with_post(update_query)
102
+ return result, error
103
+
104
+ def get_tdbloader_command(self, files: List[Path]) -> str:
105
+ """
106
+ Build the tdb2.tdbloader docker command for the given dump files.
107
+
108
+ The loader classes ship inside fuseki-server.jar, so the server's own
109
+ image is reused with an overridden entrypoint. The TDB2 dataset is
110
+ single-writer - the server must be stopped while the loader runs.
111
+
112
+ Args:
113
+ files: dump files to load (from the dumps directory)
114
+
115
+ Returns:
116
+ the docker run command string
117
+ """
118
+ loc = f"/fuseki/databases/{self.config.dataset}"
119
+ dumps_dir = Path(self.config.dumps_dir)
120
+ file_args = " ".join(f"/dumps/{file.name}" for file in files)
121
+ command = (
122
+ f"docker run --rm {self.config.docker_user_flag} --entrypoint java "
123
+ f"-v {self.config.base_data_dir}:/fuseki "
124
+ f"-v {dumps_dir}:/dumps "
125
+ f"{self.config.image} "
126
+ f"-cp /jena-fuseki/fuseki-server.jar tdb2.tdbloader "
127
+ f"--loc {loc} {file_args}"
128
+ )
129
+ return command
130
+
131
+ def upload_dump_files(self, file_pattern: str = None) -> int:
132
+ """
133
+ Bulk-load dump files with tdb2.tdbloader directly into the TDB2
134
+ database files - the native path for the file-backed store, avoiding
135
+ the HTTP single-POST transaction that faults the mmap'd node table
136
+ (issue #25).
137
+
138
+ Args:
139
+ file_pattern: Glob pattern for dump files
140
+
141
+ Returns:
142
+ Number of files loaded successfully
143
+ """
144
+ container_name = self.config.container_name
145
+ files = self.get_dump_files(file_pattern)
146
+ loaded_count = 0
147
+ if not files:
148
+ self.log.log("⚠️", container_name, f"No dump files found for pattern: {file_pattern}")
149
+ else:
150
+ self.stop()
151
+ loader_cmd = self.get_tdbloader_command(files)
152
+ shell_result = self.run_shell_command(
153
+ loader_cmd,
154
+ success_msg=f"tdb2.tdbloader loaded {len(files)} file(s)",
155
+ error_msg="tdb2.tdbloader failed",
156
+ )
157
+ if shell_result.success:
158
+ loaded_count = len(files)
159
+ self.start()
160
+ return loaded_count
101
161
 
102
162
 
@@ -5,6 +5,7 @@ Created on 2025-05-28
5
5
  """
6
6
 
7
7
  import os
8
+ import shutil
8
9
  from configparser import ConfigParser, ExtendedInterpolation
9
10
  from dataclasses import dataclass
10
11
  from pathlib import Path
@@ -238,6 +239,71 @@ class QLever(SparqlServer):
238
239
 
239
240
  return None
240
241
 
242
+ def get_index_commands(self, files: List[Path]) -> List[str]:
243
+ """
244
+ Build the qlever CLI commands to (re)index the given dump files.
245
+
246
+ QLever loads bulk data by building its index from files - there is no
247
+ native HTTP bulk-write path; the SPARQL INSERT route is only suitable
248
+ for small increments.
249
+
250
+ Args:
251
+ files: dump files staged in the data directory
252
+
253
+ Returns:
254
+ list of shell commands to run in the data directory
255
+ """
256
+ commands = [
257
+ "qlever stop",
258
+ "qlever index --overwrite-existing",
259
+ f"qlever start --server-container {self.config.container_name}",
260
+ ]
261
+ return commands
262
+
263
+ def upload_dump_files(self, file_pattern: str = None) -> int:
264
+ """
265
+ Bulk-load dump files by rebuilding the QLever index from them -
266
+ the native path; the turtle-to-INSERT conversion of upload_request
267
+ is unsuitable for bulk data.
268
+
269
+ Args:
270
+ file_pattern: Glob pattern for dump files
271
+
272
+ Returns:
273
+ Number of files loaded successfully
274
+ """
275
+ container_name = self.config.container_name
276
+ files = self.get_dump_files(file_pattern)
277
+ loaded_count = 0
278
+ if not files:
279
+ self.log.log("⚠️", container_name, f"No dump files found for pattern: {file_pattern}")
280
+ return loaded_count
281
+ data_dir = Path(self.config.data_dir)
282
+ qleverfile_path = data_dir / "Qleverfile"
283
+ qlever_file = QLeverfile.ofFile(qleverfile_path)
284
+ if qlever_file is None:
285
+ self.log.log("❌", container_name, f"no Qleverfile in {data_dir} - run start (setup-config) first")
286
+ return loaded_count
287
+ # stage the dumps in the data directory and register them as input files
288
+ for file in files:
289
+ target = data_dir / file.name
290
+ if not target.exists():
291
+ shutil.copy2(file, target)
292
+ input_files = " ".join(file.name for file in files)
293
+ qlever_file.set("index", "INPUT_FILES", input_files)
294
+ qlever_file.save()
295
+ ok = True
296
+ for command in self.get_index_commands(files):
297
+ shell_result = self.run_shell_command(f"cd {data_dir};{command}")
298
+ if not shell_result.success:
299
+ self.log.log("❌", container_name, f"failed: {command}")
300
+ ok = False
301
+ break
302
+ if ok:
303
+ loaded_count = len(files)
304
+ self.log.log("✅", container_name, f"index rebuilt from {loaded_count} file(s)")
305
+ return loaded_count
306
+
241
307
  def upload_request(self, file_content: bytes) -> Response:
242
308
  """Upload request for QLever using SPARQL INSERT statements."""
243
309
  turtle_data = file_content.decode("utf-8")
@@ -687,21 +687,52 @@ class SparqlServer:
687
687
 
688
688
  return load_success
689
689
 
690
- def load_dump_files(self, file_pattern: str = None) -> int:
690
+ def get_dump_files(self, file_pattern: str = None) -> List[Path]:
691
691
  """
692
- Load all dump files matching pattern.
692
+ Get the dump files matching the given pattern.
693
693
 
694
694
  Args:
695
695
  file_pattern: Glob pattern for dump files
696
- use_bulk: Use bulk loader if True, individual files if False
697
696
 
698
697
  Returns:
699
- Number of files loaded successfully
698
+ sorted list of matching dump file paths
700
699
  """
701
700
  dump_path: Path = Path(self.config.dumps_dir)
702
701
  if file_pattern is None:
703
702
  file_pattern = f"*{self.rdf_format.extension}"
704
703
  files = sorted(dump_path.glob(file_pattern))
704
+ return files
705
+
706
+ def upload_dump_files(self, file_pattern: str = None) -> int:
707
+ """
708
+ Bulk-upload all dump files matching pattern using the server's
709
+ native bulk load mechanism.
710
+
711
+ Subclasses override this with their native loader (e.g. Jena
712
+ tdb2.tdbloader, Blazegraph REST DataLoader, QLever index build);
713
+ the default falls back to the HTTP load path.
714
+
715
+ Args:
716
+ file_pattern: Glob pattern for dump files
717
+
718
+ Returns:
719
+ Number of files loaded successfully
720
+ """
721
+ loaded_count = self.load_dump_files(file_pattern)
722
+ return loaded_count
723
+
724
+ def load_dump_files(self, file_pattern: str = None) -> int:
725
+ """
726
+ Load all dump files matching pattern.
727
+
728
+ Args:
729
+ file_pattern: Glob pattern for dump files
730
+ use_bulk: Use bulk loader if True, individual files if False
731
+
732
+ Returns:
733
+ Number of files loaded successfully
734
+ """
735
+ files = self.get_dump_files(file_pattern)
705
736
  loaded_count = 0
706
737
  container_name = self.config.container_name
707
738
 
@@ -35,6 +35,7 @@ usage: $0 [options]
35
35
  --triplestore <name> Triple store to use (default: blazegraph)
36
36
  --download Download dataset (for big external datasets)
37
37
  --url <url> Download URL (default: auto-generated from dataset name)
38
+ -q|--query <query> Query name with optional @params (e.g., "count" or "search@term=Berlin")
38
39
  --no-webui Don't open web UI after loading
39
40
  -h|--help Show this help message
40
41
 
@@ -49,11 +50,13 @@ See https://wiki.bitplan.com/index.php/Pyomnigraph#Quickstart for more info.
49
50
  Examples:
50
51
  $0
51
52
  $0 --dataset gov_full --triplestore jena
52
- $0 --dataset gov_full --download --url https://example.com/gov_full.tar.gz
53
+ $0 --dataset gov_full --download -q "count"
54
+ $0 -q "Triplestores"
53
55
  EOF
54
56
  exit 1
55
57
  }
56
58
 
59
+
57
60
  #
58
61
  # download_dataset - download the dataset if not cached
59
62
  # $1: l_tar_file - path to tar file
@@ -112,15 +115,15 @@ load_dataset() {
112
115
  local l_dataset="$2"
113
116
  action "Loading $l_dataset dataset into $l_triplestore"
114
117
  omnigraph -s "$l_triplestore" --cmd load --datasets "$l_dataset" || error "Failed to load dataset"
118
+ success "$l_dataset loaded"
115
119
  }
116
120
 
117
121
  #
118
- # open_webui - open the web UI
122
+ # open_webui - open the web UI with optional query
119
123
  # $1: l_triplestore - name of triplestore
120
124
  #
121
125
  open_webui() {
122
126
  local l_triplestore="$1"
123
- success "Done! Opening web UI"
124
127
  omnigraph -s "$l_triplestore" --cmd webui
125
128
  }
126
129
 
@@ -0,0 +1,58 @@
1
+ """
2
+ Created on 2026-07-22
3
+
4
+ Tests for npq (named parameterized query) support in RdfDataset - issue #36.
5
+
6
+ @author: wf
7
+ """
8
+
9
+ from omnigraph.ominigraph_paths import OmnigraphPaths
10
+ from omnigraph.rdf_dataset import RdfDataset, RdfDatasets
11
+ from tests.basetest import Basetest
12
+
13
+
14
+ class TestRdfDatasetParams(Basetest):
15
+ """
16
+ test npq parameter support in RdfDataset
17
+ """
18
+
19
+ def setUp(self, debug=False, profile=True):
20
+ Basetest.setUp(self, debug=debug, profile=profile)
21
+ self.ogp = OmnigraphPaths()
22
+ self.datasets_yaml_path = self.ogp.examples_dir / "datasets.yaml"
23
+
24
+ def get_npq_dataset(self, orcids: str = '"0000-0002-0821-6995"') -> RdfDataset:
25
+ """create an npq test dataset with the given orcids param"""
26
+ dataset = RdfDataset(
27
+ name="npq-test",
28
+ endpoint_url="https://example.org/sparql",
29
+ select_pattern="VALUES ?orcid { {{ orcids }} } ?author ?p ?orcid",
30
+ construct_template="?author ?p ?orcid",
31
+ params={"orcids": orcids},
32
+ )
33
+ return dataset
34
+
35
+ def test_params_applied_from_yaml_defaults(self):
36
+ """dataset params are substituted into the query templates"""
37
+ dataset = self.get_npq_dataset()
38
+ self.assertNotIn("{{", dataset.select_pattern)
39
+ self.assertIn('VALUES ?orcid { "0000-0002-0821-6995" }', dataset.select_pattern)
40
+ self.assertIn('"0000-0002-0821-6995"', dataset.count_query.query)
41
+
42
+ def test_params_cli_override(self):
43
+ """apply_params re-applies from raw templates so overrides work"""
44
+ dataset = self.get_npq_dataset()
45
+ dataset.apply_params({"orcids": '"0000-0001-6324-7164"'})
46
+ dataset.build_queries()
47
+ self.assertIn('"0000-0001-6324-7164"', dataset.select_pattern)
48
+ self.assertNotIn("0000-0002-0821-6995", dataset.select_pattern)
49
+ self.assertIn('"0000-0001-6324-7164"', dataset.count_query.query)
50
+
51
+ def test_example_npq_datasets(self):
52
+ """the papers_of_my_network example datasets resolve their params"""
53
+ datasets = RdfDatasets.ofYaml(str(self.datasets_yaml_path))
54
+ for ds_id in ["papers_of_my_network_wikidata", "papers_of_my_network_dblp"]:
55
+ dataset = datasets.datasets.get(ds_id)
56
+ self.assertIsNotNone(dataset, ds_id)
57
+ self.assertNotIn("{{", dataset.select_pattern)
58
+ self.assertIn("0000-0002-0821-6995", dataset.select_pattern)
@@ -0,0 +1,98 @@
1
+ """
2
+ Created on 2026-07-22
3
+
4
+ Tests for the upload command - native per-server bulk load (issue #27,
5
+ root cause fix for issue #25).
6
+
7
+ @author: wf
8
+ """
9
+
10
+ import tempfile
11
+ from pathlib import Path
12
+
13
+ from omnigraph.ominigraph_paths import OmnigraphPaths
14
+ from omnigraph.omniserver import OmniServer
15
+ from omnigraph.sparql_server import ServerEnv
16
+ from tests.basetest import Basetest
17
+
18
+
19
+ class TestUpload(Basetest):
20
+ """
21
+ test the native bulk upload command
22
+ """
23
+
24
+ def setUp(self, debug=False, profile=True):
25
+ """
26
+ setUp the test environment
27
+ """
28
+ Basetest.setUp(self, debug=debug, profile=profile)
29
+ self.ogp = OmnigraphPaths()
30
+ self.servers_yaml_path = self.ogp.examples_dir / "servers.yaml"
31
+ self.env = ServerEnv(debug=self.debug)
32
+ self.omni_server = OmniServer(env=self.env)
33
+ self.servers_dict = self.omni_server.servers(self.servers_yaml_path, filter_active=False)
34
+ # dumps_dir is normally set per dataset by the load/upload command
35
+ self.dumps_dir = tempfile.mkdtemp(prefix="omnigraph-dumps-")
36
+ for server in self.servers_dict.values():
37
+ server.config.dumps_dir = self.dumps_dir
38
+
39
+ def test_upload_command_registered(self):
40
+ """
41
+ the upload command must be available in the server command registry
42
+ """
43
+ server_cmds = self.omni_server.get_server_commands()
44
+ self.assertIn("upload", server_cmds)
45
+
46
+ def test_jena_tdbloader_command(self):
47
+ """
48
+ Jena builds a tdb2.tdbloader docker command against the TDB2 location
49
+ """
50
+ jena = self.servers_dict.get("jena")
51
+ self.assertIsNotNone(jena)
52
+ dumps_dir = jena.config.dumps_dir
53
+ files = [Path(dumps_dir) / "test1.ttl", Path(dumps_dir) / "test2.ttl"]
54
+ command = jena.get_tdbloader_command(files)
55
+ if self.debug:
56
+ print(command)
57
+ self.assertIn("tdb2.tdbloader", command)
58
+ self.assertIn("--entrypoint java", command)
59
+ self.assertIn(f"--loc /fuseki/databases/{jena.config.dataset}", command)
60
+ self.assertIn(jena.config.image, command)
61
+ self.assertIn("/dumps/test1.ttl", command)
62
+ self.assertIn("/dumps/test2.ttl", command)
63
+
64
+ def test_blazegraph_dataloader_xml(self):
65
+ """
66
+ Blazegraph builds a DataLoader servlet properties XML
67
+ """
68
+ blazegraph = self.servers_dict.get("blazegraph")
69
+ self.assertIsNotNone(blazegraph)
70
+ xml = blazegraph.get_dataloader_xml("/data/dumps")
71
+ if self.debug:
72
+ print(xml)
73
+ self.assertIn('<entry key="namespace">kb</entry>', xml)
74
+ self.assertIn('<entry key="fileOrDirs">/data/dumps</entry>', xml)
75
+ self.assertIn('<entry key="propertyFile">/RWStore.properties</entry>', xml)
76
+ self.assertIn("dataloader", blazegraph.config.dataloader_url)
77
+
78
+ def test_qlever_index_commands(self):
79
+ """
80
+ QLever rebuilds its index via the qlever CLI instead of INSERT statements
81
+ """
82
+ qlever = self.servers_dict.get("qlever")
83
+ self.assertIsNotNone(qlever)
84
+ commands = qlever.get_index_commands([])
85
+ if self.debug:
86
+ print(commands)
87
+ self.assertTrue(any("qlever index" in command for command in commands))
88
+ self.assertTrue(any("qlever start" in command for command in commands))
89
+
90
+ def test_upload_fallback_no_files(self):
91
+ """
92
+ the base upload falls back to the HTTP path; with no dump files
93
+ matching it must report zero loads without contacting any server
94
+ """
95
+ oxigraph = self.servers_dict.get("oxigraph")
96
+ self.assertIsNotNone(oxigraph)
97
+ loaded_count = oxigraph.upload_dump_files("*.does-not-exist")
98
+ self.assertEqual(0, loaded_count)
@@ -1 +0,0 @@
1
- __version__ = "0.2.1"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes