pyomnigraph 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/PKG-INFO +1 -1
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/docs/index.md +1 -1
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/mkdocs.yml +1 -1
- pyomnigraph-0.2.2/omnigraph/__init__.py +1 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/omnigraph_cmd.py +2 -1
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/omniserver.py +1 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/rdf_dataset.py +45 -3
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/rdfdump_cmd.py +14 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/resources/examples/datasets.yaml +37 -1
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/blazegraph.py +64 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/jena.py +62 -2
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/qlever.py +66 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/sparql_server.py +35 -4
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/scripts/quickstart +6 -3
- pyomnigraph-0.2.2/tests/test_rdf_dataset_params.py +58 -0
- pyomnigraph-0.2.2/tests/test_upload.py +98 -0
- pyomnigraph-0.2.1/omnigraph/__init__.py +0 -1
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/.github/workflows/build.yml +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/.github/workflows/upload-to-pypi.yml +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/.gitignore +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/.project +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/.pydevproject +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/AGENTS.md +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/LICENSE +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/README.md +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/basecmd.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/ominigraph_paths.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/rdfdump.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/resources/__init__.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/resources/examples/endpoints.yaml +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/resources/examples/prefixes.yaml +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/resources/examples/queries.yaml +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/resources/examples/royals.ttl +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/resources/examples/servers.yaml +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/server_config.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/__init__.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/graphdb.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/millenniumdb.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/oxigraph.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/stardog.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/servers/virtuoso.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/software.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/omnigraph/version.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/pyproject.toml +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/scripts/blackisort +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/scripts/doc +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/scripts/gov-quickstart +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/scripts/install +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/scripts/release +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/scripts/test +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/tests/__init__.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/tests/basetest.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/tests/test_omniserver.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/tests/test_qlever.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/tests/test_rdfdump.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/tests/test_software_list.py +0 -0
- {pyomnigraph-0.2.1 → pyomnigraph-0.2.2}/tests/test_sparql_server.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pyomnigraph
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: pyomnigraph: Unified Python interface for multiple graph databases (SPARQL, Neo4j, Weaviate)
|
|
5
5
|
Project-URL: Home, https://github.com/WolfgangFahl/omnigraph
|
|
6
6
|
Project-URL: Documentation, https://wiki.bitplan.com/index.php/omnigraph
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.2.2"
|
|
@@ -171,7 +171,8 @@ class OmnigraphCmd(BaseCmd):
|
|
|
171
171
|
handled = False
|
|
172
172
|
if cmds:
|
|
173
173
|
for cmd in cmds:
|
|
174
|
-
|
|
174
|
+
# issue #38: upload needs the per-dataset dumps_dir just like load
|
|
175
|
+
if cmd in ("load", "upload"):
|
|
175
176
|
cmd_iterator = self.load_iterator(server)
|
|
176
177
|
else:
|
|
177
178
|
cmd_iterator = iter([None]) # Single iteration
|
|
@@ -70,6 +70,7 @@ class OmniServer:
|
|
|
70
70
|
"start": lambda s: ServerCmd(title("start", s), s.start),
|
|
71
71
|
"status": lambda s: ServerCmd(title("status", s), s.status_info),
|
|
72
72
|
"stop": lambda s: ServerCmd(title("stop", s), s.stop),
|
|
73
|
+
"upload": lambda s: ServerCmd(title("native bulk upload dumps", s), s.upload_dump_files),
|
|
73
74
|
"webui": lambda s: ServerCmd(title("webui", s), s.webui),
|
|
74
75
|
}
|
|
75
76
|
return server_cmds
|
|
@@ -8,6 +8,7 @@ from dataclasses import dataclass, field
|
|
|
8
8
|
from typing import Dict, Optional
|
|
9
9
|
|
|
10
10
|
from basemkit.yamlable import lod_storable
|
|
11
|
+
from lodstorage.params import Params
|
|
11
12
|
from lodstorage.query import Query
|
|
12
13
|
from lodstorage.sparql import SPARQL
|
|
13
14
|
|
|
@@ -26,6 +27,7 @@ class RdfDataset:
|
|
|
26
27
|
expected_solutions: Optional[int] = None # Expected number of solutions
|
|
27
28
|
select_pattern: str = "?s ?p ?o" # Basic Graph Pattern for queries
|
|
28
29
|
construct_template: Optional[str] = field(default="?s ?p ?o")
|
|
30
|
+
params: Optional[Dict[str, str]] = None # npq parameters for {{ name }} templates (issue #36)
|
|
29
31
|
prefix_sets: Optional[list] = field(default_factory=list)
|
|
30
32
|
active: Optional[bool] = False
|
|
31
33
|
rdf_file: Optional[str] = None # Path to local RDF file (alternative to endpoint_url)
|
|
@@ -35,10 +37,41 @@ class RdfDataset:
|
|
|
35
37
|
select_query: Optional[Query] = field(default=None)
|
|
36
38
|
sparql: Optional[SPARQL] = field(default=None)
|
|
37
39
|
|
|
38
|
-
def
|
|
40
|
+
def apply_params(self, params_dict: Optional[Dict[str, str]] = None) -> None:
|
|
41
|
+
"""
|
|
42
|
+
Apply npq parameters to select_pattern and construct_template.
|
|
43
|
+
|
|
44
|
+
{{ name }} templates are replaced with the values of the merged
|
|
45
|
+
params: the dataset's own params overridden by the given ones.
|
|
46
|
+
Substitution always starts from the raw templates, so CLI overrides
|
|
47
|
+
can be re-applied after YAML defaults. The blanket Params audit is
|
|
48
|
+
off - it rejects quoted VALUES lists (see issue #36); values stem
|
|
49
|
+
from local configuration and CLI.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
params_dict: additional parameter values overriding self.params
|
|
39
53
|
"""
|
|
40
|
-
|
|
41
|
-
|
|
54
|
+
# preserve the raw templates on first call
|
|
55
|
+
if not hasattr(self, "raw_templates"):
|
|
56
|
+
self.raw_templates = {
|
|
57
|
+
"select_pattern": self.select_pattern,
|
|
58
|
+
"construct_template": self.construct_template,
|
|
59
|
+
}
|
|
60
|
+
merged = dict(self.params) if self.params else {}
|
|
61
|
+
if params_dict:
|
|
62
|
+
merged.update(params_dict)
|
|
63
|
+
if merged:
|
|
64
|
+
for attr, raw_template in self.raw_templates.items():
|
|
65
|
+
if raw_template:
|
|
66
|
+
params = Params(raw_template, with_audit=False)
|
|
67
|
+
if params.has_params:
|
|
68
|
+
params.set(merged)
|
|
69
|
+
setattr(self, attr, params.apply_parameters())
|
|
70
|
+
|
|
71
|
+
def build_queries(self) -> None:
|
|
72
|
+
"""
|
|
73
|
+
(Re)build count_query and select_query from the current select_pattern.
|
|
74
|
+
Only initializes SPARQL-related fields if endpoint_url is provided.
|
|
42
75
|
"""
|
|
43
76
|
if self.endpoint_url:
|
|
44
77
|
self.count_query = Query(
|
|
@@ -55,6 +88,13 @@ class RdfDataset:
|
|
|
55
88
|
)
|
|
56
89
|
self.sparql = SPARQL(self.endpoint_url)
|
|
57
90
|
|
|
91
|
+
def __post_init__(self):
|
|
92
|
+
"""
|
|
93
|
+
Apply npq params and generate the queries from select_pattern.
|
|
94
|
+
"""
|
|
95
|
+
self.apply_params()
|
|
96
|
+
self.build_queries()
|
|
97
|
+
|
|
58
98
|
@property
|
|
59
99
|
def full_name(self):
|
|
60
100
|
ds_id = self.id or "?"
|
|
@@ -69,6 +109,8 @@ class RdfDataset:
|
|
|
69
109
|
Number of solutions available from the count query
|
|
70
110
|
"""
|
|
71
111
|
count = self.sparql.getValue(self.count_query.query, "count")
|
|
112
|
+
# issue #37: getValue returns the raw literal which may be a str
|
|
113
|
+
count = int(count)
|
|
72
114
|
return count
|
|
73
115
|
|
|
74
116
|
def getTryItUrl(self, database: str = "blazegraph") -> str:
|
|
@@ -10,6 +10,8 @@ import os
|
|
|
10
10
|
import webbrowser
|
|
11
11
|
from argparse import ArgumentParser, Namespace
|
|
12
12
|
|
|
13
|
+
from basemkit.argparse_action import StoreDictKeyPair
|
|
14
|
+
|
|
13
15
|
from omnigraph.basecmd import BaseCmd
|
|
14
16
|
from omnigraph.rdf_dataset import RdfDataset, RdfDatasets
|
|
15
17
|
from omnigraph.rdfdump import RdfDumpDownloader
|
|
@@ -61,6 +63,13 @@ class RdfDumpCmd(BaseCmd):
|
|
|
61
63
|
default=None,
|
|
62
64
|
help="Maximum number of solutions/triples to download (uses dataset expected_solutions if not specified)",
|
|
63
65
|
)
|
|
66
|
+
parser.add_argument(
|
|
67
|
+
"--param",
|
|
68
|
+
action=StoreDictKeyPair,
|
|
69
|
+
dest="params",
|
|
70
|
+
metavar="KEY=VALUE,KEY=VALUE...",
|
|
71
|
+
help="npq parameter values overriding the dataset's params (issue #36)",
|
|
72
|
+
)
|
|
64
73
|
parser.add_argument("--no-progress", action="store_true", help="Disable progress bar")
|
|
65
74
|
parser.add_argument("--output-path", default=".", help="Path for dump files")
|
|
66
75
|
parser.add_argument("--tryit", action="store_true", help="open the try it! URL [default: %(default)s]")
|
|
@@ -97,6 +106,11 @@ class RdfDumpCmd(BaseCmd):
|
|
|
97
106
|
"""
|
|
98
107
|
super().handle_args(args)
|
|
99
108
|
datasets = self.datasets
|
|
109
|
+
if getattr(args, "params", None):
|
|
110
|
+
# apply CLI npq parameter overrides and rebuild the queries
|
|
111
|
+
for dataset in datasets.values():
|
|
112
|
+
dataset.apply_params(args.params)
|
|
113
|
+
dataset.build_queries()
|
|
100
114
|
if self.args.about:
|
|
101
115
|
self.about()
|
|
102
116
|
|
|
@@ -65,4 +65,40 @@ datasets:
|
|
|
65
65
|
?pName ?p4Name ?o4Name
|
|
66
66
|
# Optional dataset description
|
|
67
67
|
description: "W2306 postal code coordinates"
|
|
68
|
-
active: false
|
|
68
|
+
active: false
|
|
69
|
+
papers_of_my_network_wikidata:
|
|
70
|
+
# npq (named parameterized query) example - issue #36
|
|
71
|
+
# ORCID is the join key; {{ orcids }} is the VALUES scaling knob;
|
|
72
|
+
# endpoint is late-bound - own copy answers completely and fastest,
|
|
73
|
+
# see the endpoint matrix in the issue
|
|
74
|
+
name: "Papers of my network (wikidata)"
|
|
75
|
+
base_url: "https://qlever-api.wikidata.dbis.rwth-aachen.de"
|
|
76
|
+
endpoint_url: "https://qlever-api.wikidata.dbis.rwth-aachen.de/"
|
|
77
|
+
database: "qlever"
|
|
78
|
+
select_pattern: |
|
|
79
|
+
VALUES ?orcid { {{ orcids }} }
|
|
80
|
+
?author <http://www.wikidata.org/prop/direct/P496> ?orcid .
|
|
81
|
+
?work <http://www.wikidata.org/prop/direct/P50> ?author .
|
|
82
|
+
?work ?p ?o
|
|
83
|
+
construct_template: "?work ?p ?o"
|
|
84
|
+
params:
|
|
85
|
+
orcids: '"0000-0002-0821-6995" "0000-0003-2533-6363" "0000-0001-9879-3827" "0000-0001-6324-7164"'
|
|
86
|
+
description: "Papers authored by the co-author network, seeded by ORCIDs"
|
|
87
|
+
active: false
|
|
88
|
+
papers_of_my_network_dblp:
|
|
89
|
+
# npq companion - same {{ orcids }} param drives the dblp graph
|
|
90
|
+
name: "Papers of my network (dblp)"
|
|
91
|
+
base_url: "https://sparql.dblp.org"
|
|
92
|
+
endpoint_url: "https://sparql.dblp.org/sparql"
|
|
93
|
+
database: "qlever"
|
|
94
|
+
select_pattern: |
|
|
95
|
+
VALUES ?orcid { {{ orcids }} }
|
|
96
|
+
BIND(IRI(CONCAT("https://orcid.org/", ?orcid)) AS ?orcid_uri)
|
|
97
|
+
?author <https://dblp.org/rdf/schema#orcid> ?orcid_uri .
|
|
98
|
+
?paper <https://dblp.org/rdf/schema#authoredBy> ?author .
|
|
99
|
+
?paper ?p ?o
|
|
100
|
+
construct_template: "?paper ?p ?o"
|
|
101
|
+
params:
|
|
102
|
+
orcids: '"0000-0002-0821-6995" "0000-0003-2533-6363" "0000-0001-9879-3827" "0000-0001-6324-7164"'
|
|
103
|
+
description: "Papers authored by the co-author network in dblp, same ORCID seed"
|
|
104
|
+
active: false
|
|
@@ -8,6 +8,8 @@ from dataclasses import dataclass
|
|
|
8
8
|
from pathlib import Path
|
|
9
9
|
import re
|
|
10
10
|
import os
|
|
11
|
+
import shutil
|
|
12
|
+
|
|
11
13
|
from omnigraph.server_config import ServerLifecycleState, ServerStatus
|
|
12
14
|
from omnigraph.sparql_server import ServerConfig, ServerEnv, SparqlServer
|
|
13
15
|
|
|
@@ -94,6 +96,68 @@ class Blazegraph(SparqlServer):
|
|
|
94
96
|
rwstore_path.write_text(props)
|
|
95
97
|
|
|
96
98
|
|
|
99
|
+
def get_dataloader_xml(self, container_path: str) -> str:
|
|
100
|
+
"""
|
|
101
|
+
Build the DataLoader servlet properties XML for the given
|
|
102
|
+
container-side file or directory path.
|
|
103
|
+
|
|
104
|
+
Args:
|
|
105
|
+
container_path: path as seen inside the container (mounted /data)
|
|
106
|
+
|
|
107
|
+
Returns:
|
|
108
|
+
the properties XML document
|
|
109
|
+
"""
|
|
110
|
+
xml = f"""<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
|
111
|
+
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
|
112
|
+
<properties>
|
|
113
|
+
<entry key="namespace">{self.config.dataset}</entry>
|
|
114
|
+
<entry key="propertyFile">/RWStore.properties</entry>
|
|
115
|
+
<entry key="fileOrDirs">{container_path}</entry>
|
|
116
|
+
<entry key="-durableQueues">true</entry>
|
|
117
|
+
</properties>"""
|
|
118
|
+
return xml
|
|
119
|
+
|
|
120
|
+
def upload_dump_files(self, file_pattern: str = None) -> int:
|
|
121
|
+
"""
|
|
122
|
+
Bulk-load dump files via Blazegraph's REST DataLoader servlet, which
|
|
123
|
+
reads the files from the server's own filesystem (the mounted /data
|
|
124
|
+
directory) instead of pushing them through HTTP request bodies.
|
|
125
|
+
|
|
126
|
+
Args:
|
|
127
|
+
file_pattern: Glob pattern for dump files
|
|
128
|
+
|
|
129
|
+
Returns:
|
|
130
|
+
Number of files loaded successfully
|
|
131
|
+
"""
|
|
132
|
+
container_name = self.config.container_name
|
|
133
|
+
files = self.get_dump_files(file_pattern)
|
|
134
|
+
loaded_count = 0
|
|
135
|
+
if not files:
|
|
136
|
+
self.log.log("⚠️", container_name, f"No dump files found for pattern: {file_pattern}")
|
|
137
|
+
else:
|
|
138
|
+
# stage the dumps under data_dir so the container sees them at /data/dumps
|
|
139
|
+
stage_dir = Path(self.config.base_data_dir) / "dumps"
|
|
140
|
+
stage_dir.mkdir(parents=True, exist_ok=True)
|
|
141
|
+
for file in files:
|
|
142
|
+
target = stage_dir / file.name
|
|
143
|
+
if not target.exists():
|
|
144
|
+
shutil.copy2(file, target)
|
|
145
|
+
xml = self.get_dataloader_xml("/data/dumps")
|
|
146
|
+
response = self.make_request(
|
|
147
|
+
"POST",
|
|
148
|
+
self.config.dataloader_url,
|
|
149
|
+
headers={"Content-Type": "application/xml"},
|
|
150
|
+
data=xml,
|
|
151
|
+
timeout=self.config.upload_timeout,
|
|
152
|
+
)
|
|
153
|
+
if response.success:
|
|
154
|
+
loaded_count = len(files)
|
|
155
|
+
self.log.log("✅", container_name, f"DataLoader loaded {loaded_count} file(s)")
|
|
156
|
+
else:
|
|
157
|
+
error_msg = str(response.error) if response.error else f"HTTP {response.response.status_code}"
|
|
158
|
+
self.log.log("❌", container_name, f"DataLoader failed: {error_msg}")
|
|
159
|
+
return loaded_count
|
|
160
|
+
|
|
97
161
|
def status(self) -> ServerStatus:
|
|
98
162
|
"""
|
|
99
163
|
Get server status information.
|
|
@@ -7,6 +7,8 @@ Apache Jena SPARQL support
|
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
9
|
from dataclasses import dataclass
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import List
|
|
10
12
|
|
|
11
13
|
from omnigraph.server_config import ServerLifecycleState, ServerStatus
|
|
12
14
|
from omnigraph.sparql_server import ServerConfig, ServerEnv, SparqlServer
|
|
@@ -96,7 +98,65 @@ class Jena(SparqlServer):
|
|
|
96
98
|
Returns:
|
|
97
99
|
Tuple of (response, exception)
|
|
98
100
|
"""
|
|
99
|
-
result,error=self.execute_update_query_with_post(update_query)
|
|
100
|
-
return result,error
|
|
101
|
+
result, error = self.execute_update_query_with_post(update_query)
|
|
102
|
+
return result, error
|
|
103
|
+
|
|
104
|
+
def get_tdbloader_command(self, files: List[Path]) -> str:
|
|
105
|
+
"""
|
|
106
|
+
Build the tdb2.tdbloader docker command for the given dump files.
|
|
107
|
+
|
|
108
|
+
The loader classes ship inside fuseki-server.jar, so the server's own
|
|
109
|
+
image is reused with an overridden entrypoint. The TDB2 dataset is
|
|
110
|
+
single-writer - the server must be stopped while the loader runs.
|
|
111
|
+
|
|
112
|
+
Args:
|
|
113
|
+
files: dump files to load (from the dumps directory)
|
|
114
|
+
|
|
115
|
+
Returns:
|
|
116
|
+
the docker run command string
|
|
117
|
+
"""
|
|
118
|
+
loc = f"/fuseki/databases/{self.config.dataset}"
|
|
119
|
+
dumps_dir = Path(self.config.dumps_dir)
|
|
120
|
+
file_args = " ".join(f"/dumps/{file.name}" for file in files)
|
|
121
|
+
command = (
|
|
122
|
+
f"docker run --rm {self.config.docker_user_flag} --entrypoint java "
|
|
123
|
+
f"-v {self.config.base_data_dir}:/fuseki "
|
|
124
|
+
f"-v {dumps_dir}:/dumps "
|
|
125
|
+
f"{self.config.image} "
|
|
126
|
+
f"-cp /jena-fuseki/fuseki-server.jar tdb2.tdbloader "
|
|
127
|
+
f"--loc {loc} {file_args}"
|
|
128
|
+
)
|
|
129
|
+
return command
|
|
130
|
+
|
|
131
|
+
def upload_dump_files(self, file_pattern: str = None) -> int:
|
|
132
|
+
"""
|
|
133
|
+
Bulk-load dump files with tdb2.tdbloader directly into the TDB2
|
|
134
|
+
database files - the native path for the file-backed store, avoiding
|
|
135
|
+
the HTTP single-POST transaction that faults the mmap'd node table
|
|
136
|
+
(issue #25).
|
|
137
|
+
|
|
138
|
+
Args:
|
|
139
|
+
file_pattern: Glob pattern for dump files
|
|
140
|
+
|
|
141
|
+
Returns:
|
|
142
|
+
Number of files loaded successfully
|
|
143
|
+
"""
|
|
144
|
+
container_name = self.config.container_name
|
|
145
|
+
files = self.get_dump_files(file_pattern)
|
|
146
|
+
loaded_count = 0
|
|
147
|
+
if not files:
|
|
148
|
+
self.log.log("⚠️", container_name, f"No dump files found for pattern: {file_pattern}")
|
|
149
|
+
else:
|
|
150
|
+
self.stop()
|
|
151
|
+
loader_cmd = self.get_tdbloader_command(files)
|
|
152
|
+
shell_result = self.run_shell_command(
|
|
153
|
+
loader_cmd,
|
|
154
|
+
success_msg=f"tdb2.tdbloader loaded {len(files)} file(s)",
|
|
155
|
+
error_msg="tdb2.tdbloader failed",
|
|
156
|
+
)
|
|
157
|
+
if shell_result.success:
|
|
158
|
+
loaded_count = len(files)
|
|
159
|
+
self.start()
|
|
160
|
+
return loaded_count
|
|
101
161
|
|
|
102
162
|
|
|
@@ -5,6 +5,7 @@ Created on 2025-05-28
|
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
7
|
import os
|
|
8
|
+
import shutil
|
|
8
9
|
from configparser import ConfigParser, ExtendedInterpolation
|
|
9
10
|
from dataclasses import dataclass
|
|
10
11
|
from pathlib import Path
|
|
@@ -238,6 +239,71 @@ class QLever(SparqlServer):
|
|
|
238
239
|
|
|
239
240
|
return None
|
|
240
241
|
|
|
242
|
+
def get_index_commands(self, files: List[Path]) -> List[str]:
|
|
243
|
+
"""
|
|
244
|
+
Build the qlever CLI commands to (re)index the given dump files.
|
|
245
|
+
|
|
246
|
+
QLever loads bulk data by building its index from files - there is no
|
|
247
|
+
native HTTP bulk-write path; the SPARQL INSERT route is only suitable
|
|
248
|
+
for small increments.
|
|
249
|
+
|
|
250
|
+
Args:
|
|
251
|
+
files: dump files staged in the data directory
|
|
252
|
+
|
|
253
|
+
Returns:
|
|
254
|
+
list of shell commands to run in the data directory
|
|
255
|
+
"""
|
|
256
|
+
commands = [
|
|
257
|
+
"qlever stop",
|
|
258
|
+
"qlever index --overwrite-existing",
|
|
259
|
+
f"qlever start --server-container {self.config.container_name}",
|
|
260
|
+
]
|
|
261
|
+
return commands
|
|
262
|
+
|
|
263
|
+
def upload_dump_files(self, file_pattern: str = None) -> int:
|
|
264
|
+
"""
|
|
265
|
+
Bulk-load dump files by rebuilding the QLever index from them -
|
|
266
|
+
the native path; the turtle-to-INSERT conversion of upload_request
|
|
267
|
+
is unsuitable for bulk data.
|
|
268
|
+
|
|
269
|
+
Args:
|
|
270
|
+
file_pattern: Glob pattern for dump files
|
|
271
|
+
|
|
272
|
+
Returns:
|
|
273
|
+
Number of files loaded successfully
|
|
274
|
+
"""
|
|
275
|
+
container_name = self.config.container_name
|
|
276
|
+
files = self.get_dump_files(file_pattern)
|
|
277
|
+
loaded_count = 0
|
|
278
|
+
if not files:
|
|
279
|
+
self.log.log("⚠️", container_name, f"No dump files found for pattern: {file_pattern}")
|
|
280
|
+
return loaded_count
|
|
281
|
+
data_dir = Path(self.config.data_dir)
|
|
282
|
+
qleverfile_path = data_dir / "Qleverfile"
|
|
283
|
+
qlever_file = QLeverfile.ofFile(qleverfile_path)
|
|
284
|
+
if qlever_file is None:
|
|
285
|
+
self.log.log("❌", container_name, f"no Qleverfile in {data_dir} - run start (setup-config) first")
|
|
286
|
+
return loaded_count
|
|
287
|
+
# stage the dumps in the data directory and register them as input files
|
|
288
|
+
for file in files:
|
|
289
|
+
target = data_dir / file.name
|
|
290
|
+
if not target.exists():
|
|
291
|
+
shutil.copy2(file, target)
|
|
292
|
+
input_files = " ".join(file.name for file in files)
|
|
293
|
+
qlever_file.set("index", "INPUT_FILES", input_files)
|
|
294
|
+
qlever_file.save()
|
|
295
|
+
ok = True
|
|
296
|
+
for command in self.get_index_commands(files):
|
|
297
|
+
shell_result = self.run_shell_command(f"cd {data_dir};{command}")
|
|
298
|
+
if not shell_result.success:
|
|
299
|
+
self.log.log("❌", container_name, f"failed: {command}")
|
|
300
|
+
ok = False
|
|
301
|
+
break
|
|
302
|
+
if ok:
|
|
303
|
+
loaded_count = len(files)
|
|
304
|
+
self.log.log("✅", container_name, f"index rebuilt from {loaded_count} file(s)")
|
|
305
|
+
return loaded_count
|
|
306
|
+
|
|
241
307
|
def upload_request(self, file_content: bytes) -> Response:
|
|
242
308
|
"""Upload request for QLever using SPARQL INSERT statements."""
|
|
243
309
|
turtle_data = file_content.decode("utf-8")
|
|
@@ -687,21 +687,52 @@ class SparqlServer:
|
|
|
687
687
|
|
|
688
688
|
return load_success
|
|
689
689
|
|
|
690
|
-
def
|
|
690
|
+
def get_dump_files(self, file_pattern: str = None) -> List[Path]:
|
|
691
691
|
"""
|
|
692
|
-
|
|
692
|
+
Get the dump files matching the given pattern.
|
|
693
693
|
|
|
694
694
|
Args:
|
|
695
695
|
file_pattern: Glob pattern for dump files
|
|
696
|
-
use_bulk: Use bulk loader if True, individual files if False
|
|
697
696
|
|
|
698
697
|
Returns:
|
|
699
|
-
|
|
698
|
+
sorted list of matching dump file paths
|
|
700
699
|
"""
|
|
701
700
|
dump_path: Path = Path(self.config.dumps_dir)
|
|
702
701
|
if file_pattern is None:
|
|
703
702
|
file_pattern = f"*{self.rdf_format.extension}"
|
|
704
703
|
files = sorted(dump_path.glob(file_pattern))
|
|
704
|
+
return files
|
|
705
|
+
|
|
706
|
+
def upload_dump_files(self, file_pattern: str = None) -> int:
|
|
707
|
+
"""
|
|
708
|
+
Bulk-upload all dump files matching pattern using the server's
|
|
709
|
+
native bulk load mechanism.
|
|
710
|
+
|
|
711
|
+
Subclasses override this with their native loader (e.g. Jena
|
|
712
|
+
tdb2.tdbloader, Blazegraph REST DataLoader, QLever index build);
|
|
713
|
+
the default falls back to the HTTP load path.
|
|
714
|
+
|
|
715
|
+
Args:
|
|
716
|
+
file_pattern: Glob pattern for dump files
|
|
717
|
+
|
|
718
|
+
Returns:
|
|
719
|
+
Number of files loaded successfully
|
|
720
|
+
"""
|
|
721
|
+
loaded_count = self.load_dump_files(file_pattern)
|
|
722
|
+
return loaded_count
|
|
723
|
+
|
|
724
|
+
def load_dump_files(self, file_pattern: str = None) -> int:
|
|
725
|
+
"""
|
|
726
|
+
Load all dump files matching pattern.
|
|
727
|
+
|
|
728
|
+
Args:
|
|
729
|
+
file_pattern: Glob pattern for dump files
|
|
730
|
+
use_bulk: Use bulk loader if True, individual files if False
|
|
731
|
+
|
|
732
|
+
Returns:
|
|
733
|
+
Number of files loaded successfully
|
|
734
|
+
"""
|
|
735
|
+
files = self.get_dump_files(file_pattern)
|
|
705
736
|
loaded_count = 0
|
|
706
737
|
container_name = self.config.container_name
|
|
707
738
|
|
|
@@ -35,6 +35,7 @@ usage: $0 [options]
|
|
|
35
35
|
--triplestore <name> Triple store to use (default: blazegraph)
|
|
36
36
|
--download Download dataset (for big external datasets)
|
|
37
37
|
--url <url> Download URL (default: auto-generated from dataset name)
|
|
38
|
+
-q|--query <query> Query name with optional @params (e.g., "count" or "search@term=Berlin")
|
|
38
39
|
--no-webui Don't open web UI after loading
|
|
39
40
|
-h|--help Show this help message
|
|
40
41
|
|
|
@@ -49,11 +50,13 @@ See https://wiki.bitplan.com/index.php/Pyomnigraph#Quickstart for more info.
|
|
|
49
50
|
Examples:
|
|
50
51
|
$0
|
|
51
52
|
$0 --dataset gov_full --triplestore jena
|
|
52
|
-
$0 --dataset gov_full --download
|
|
53
|
+
$0 --dataset gov_full --download -q "count"
|
|
54
|
+
$0 -q "Triplestores"
|
|
53
55
|
EOF
|
|
54
56
|
exit 1
|
|
55
57
|
}
|
|
56
58
|
|
|
59
|
+
|
|
57
60
|
#
|
|
58
61
|
# download_dataset - download the dataset if not cached
|
|
59
62
|
# $1: l_tar_file - path to tar file
|
|
@@ -112,15 +115,15 @@ load_dataset() {
|
|
|
112
115
|
local l_dataset="$2"
|
|
113
116
|
action "Loading $l_dataset dataset into $l_triplestore"
|
|
114
117
|
omnigraph -s "$l_triplestore" --cmd load --datasets "$l_dataset" || error "Failed to load dataset"
|
|
118
|
+
success "$l_dataset loaded"
|
|
115
119
|
}
|
|
116
120
|
|
|
117
121
|
#
|
|
118
|
-
# open_webui - open the web UI
|
|
122
|
+
# open_webui - open the web UI with optional query
|
|
119
123
|
# $1: l_triplestore - name of triplestore
|
|
120
124
|
#
|
|
121
125
|
open_webui() {
|
|
122
126
|
local l_triplestore="$1"
|
|
123
|
-
success "Done! Opening web UI"
|
|
124
127
|
omnigraph -s "$l_triplestore" --cmd webui
|
|
125
128
|
}
|
|
126
129
|
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Created on 2026-07-22
|
|
3
|
+
|
|
4
|
+
Tests for npq (named parameterized query) support in RdfDataset - issue #36.
|
|
5
|
+
|
|
6
|
+
@author: wf
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from omnigraph.ominigraph_paths import OmnigraphPaths
|
|
10
|
+
from omnigraph.rdf_dataset import RdfDataset, RdfDatasets
|
|
11
|
+
from tests.basetest import Basetest
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class TestRdfDatasetParams(Basetest):
|
|
15
|
+
"""
|
|
16
|
+
test npq parameter support in RdfDataset
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def setUp(self, debug=False, profile=True):
|
|
20
|
+
Basetest.setUp(self, debug=debug, profile=profile)
|
|
21
|
+
self.ogp = OmnigraphPaths()
|
|
22
|
+
self.datasets_yaml_path = self.ogp.examples_dir / "datasets.yaml"
|
|
23
|
+
|
|
24
|
+
def get_npq_dataset(self, orcids: str = '"0000-0002-0821-6995"') -> RdfDataset:
|
|
25
|
+
"""create an npq test dataset with the given orcids param"""
|
|
26
|
+
dataset = RdfDataset(
|
|
27
|
+
name="npq-test",
|
|
28
|
+
endpoint_url="https://example.org/sparql",
|
|
29
|
+
select_pattern="VALUES ?orcid { {{ orcids }} } ?author ?p ?orcid",
|
|
30
|
+
construct_template="?author ?p ?orcid",
|
|
31
|
+
params={"orcids": orcids},
|
|
32
|
+
)
|
|
33
|
+
return dataset
|
|
34
|
+
|
|
35
|
+
def test_params_applied_from_yaml_defaults(self):
|
|
36
|
+
"""dataset params are substituted into the query templates"""
|
|
37
|
+
dataset = self.get_npq_dataset()
|
|
38
|
+
self.assertNotIn("{{", dataset.select_pattern)
|
|
39
|
+
self.assertIn('VALUES ?orcid { "0000-0002-0821-6995" }', dataset.select_pattern)
|
|
40
|
+
self.assertIn('"0000-0002-0821-6995"', dataset.count_query.query)
|
|
41
|
+
|
|
42
|
+
def test_params_cli_override(self):
|
|
43
|
+
"""apply_params re-applies from raw templates so overrides work"""
|
|
44
|
+
dataset = self.get_npq_dataset()
|
|
45
|
+
dataset.apply_params({"orcids": '"0000-0001-6324-7164"'})
|
|
46
|
+
dataset.build_queries()
|
|
47
|
+
self.assertIn('"0000-0001-6324-7164"', dataset.select_pattern)
|
|
48
|
+
self.assertNotIn("0000-0002-0821-6995", dataset.select_pattern)
|
|
49
|
+
self.assertIn('"0000-0001-6324-7164"', dataset.count_query.query)
|
|
50
|
+
|
|
51
|
+
def test_example_npq_datasets(self):
|
|
52
|
+
"""the papers_of_my_network example datasets resolve their params"""
|
|
53
|
+
datasets = RdfDatasets.ofYaml(str(self.datasets_yaml_path))
|
|
54
|
+
for ds_id in ["papers_of_my_network_wikidata", "papers_of_my_network_dblp"]:
|
|
55
|
+
dataset = datasets.datasets.get(ds_id)
|
|
56
|
+
self.assertIsNotNone(dataset, ds_id)
|
|
57
|
+
self.assertNotIn("{{", dataset.select_pattern)
|
|
58
|
+
self.assertIn("0000-0002-0821-6995", dataset.select_pattern)
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Created on 2026-07-22
|
|
3
|
+
|
|
4
|
+
Tests for the upload command - native per-server bulk load (issue #27,
|
|
5
|
+
root cause fix for issue #25).
|
|
6
|
+
|
|
7
|
+
@author: wf
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import tempfile
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from omnigraph.ominigraph_paths import OmnigraphPaths
|
|
14
|
+
from omnigraph.omniserver import OmniServer
|
|
15
|
+
from omnigraph.sparql_server import ServerEnv
|
|
16
|
+
from tests.basetest import Basetest
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class TestUpload(Basetest):
|
|
20
|
+
"""
|
|
21
|
+
test the native bulk upload command
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
def setUp(self, debug=False, profile=True):
|
|
25
|
+
"""
|
|
26
|
+
setUp the test environment
|
|
27
|
+
"""
|
|
28
|
+
Basetest.setUp(self, debug=debug, profile=profile)
|
|
29
|
+
self.ogp = OmnigraphPaths()
|
|
30
|
+
self.servers_yaml_path = self.ogp.examples_dir / "servers.yaml"
|
|
31
|
+
self.env = ServerEnv(debug=self.debug)
|
|
32
|
+
self.omni_server = OmniServer(env=self.env)
|
|
33
|
+
self.servers_dict = self.omni_server.servers(self.servers_yaml_path, filter_active=False)
|
|
34
|
+
# dumps_dir is normally set per dataset by the load/upload command
|
|
35
|
+
self.dumps_dir = tempfile.mkdtemp(prefix="omnigraph-dumps-")
|
|
36
|
+
for server in self.servers_dict.values():
|
|
37
|
+
server.config.dumps_dir = self.dumps_dir
|
|
38
|
+
|
|
39
|
+
def test_upload_command_registered(self):
|
|
40
|
+
"""
|
|
41
|
+
the upload command must be available in the server command registry
|
|
42
|
+
"""
|
|
43
|
+
server_cmds = self.omni_server.get_server_commands()
|
|
44
|
+
self.assertIn("upload", server_cmds)
|
|
45
|
+
|
|
46
|
+
def test_jena_tdbloader_command(self):
|
|
47
|
+
"""
|
|
48
|
+
Jena builds a tdb2.tdbloader docker command against the TDB2 location
|
|
49
|
+
"""
|
|
50
|
+
jena = self.servers_dict.get("jena")
|
|
51
|
+
self.assertIsNotNone(jena)
|
|
52
|
+
dumps_dir = jena.config.dumps_dir
|
|
53
|
+
files = [Path(dumps_dir) / "test1.ttl", Path(dumps_dir) / "test2.ttl"]
|
|
54
|
+
command = jena.get_tdbloader_command(files)
|
|
55
|
+
if self.debug:
|
|
56
|
+
print(command)
|
|
57
|
+
self.assertIn("tdb2.tdbloader", command)
|
|
58
|
+
self.assertIn("--entrypoint java", command)
|
|
59
|
+
self.assertIn(f"--loc /fuseki/databases/{jena.config.dataset}", command)
|
|
60
|
+
self.assertIn(jena.config.image, command)
|
|
61
|
+
self.assertIn("/dumps/test1.ttl", command)
|
|
62
|
+
self.assertIn("/dumps/test2.ttl", command)
|
|
63
|
+
|
|
64
|
+
def test_blazegraph_dataloader_xml(self):
|
|
65
|
+
"""
|
|
66
|
+
Blazegraph builds a DataLoader servlet properties XML
|
|
67
|
+
"""
|
|
68
|
+
blazegraph = self.servers_dict.get("blazegraph")
|
|
69
|
+
self.assertIsNotNone(blazegraph)
|
|
70
|
+
xml = blazegraph.get_dataloader_xml("/data/dumps")
|
|
71
|
+
if self.debug:
|
|
72
|
+
print(xml)
|
|
73
|
+
self.assertIn('<entry key="namespace">kb</entry>', xml)
|
|
74
|
+
self.assertIn('<entry key="fileOrDirs">/data/dumps</entry>', xml)
|
|
75
|
+
self.assertIn('<entry key="propertyFile">/RWStore.properties</entry>', xml)
|
|
76
|
+
self.assertIn("dataloader", blazegraph.config.dataloader_url)
|
|
77
|
+
|
|
78
|
+
def test_qlever_index_commands(self):
|
|
79
|
+
"""
|
|
80
|
+
QLever rebuilds its index via the qlever CLI instead of INSERT statements
|
|
81
|
+
"""
|
|
82
|
+
qlever = self.servers_dict.get("qlever")
|
|
83
|
+
self.assertIsNotNone(qlever)
|
|
84
|
+
commands = qlever.get_index_commands([])
|
|
85
|
+
if self.debug:
|
|
86
|
+
print(commands)
|
|
87
|
+
self.assertTrue(any("qlever index" in command for command in commands))
|
|
88
|
+
self.assertTrue(any("qlever start" in command for command in commands))
|
|
89
|
+
|
|
90
|
+
def test_upload_fallback_no_files(self):
|
|
91
|
+
"""
|
|
92
|
+
the base upload falls back to the HTTP path; with no dump files
|
|
93
|
+
matching it must report zero loads without contacting any server
|
|
94
|
+
"""
|
|
95
|
+
oxigraph = self.servers_dict.get("oxigraph")
|
|
96
|
+
self.assertIsNotNone(oxigraph)
|
|
97
|
+
loaded_count = oxigraph.upload_dump_files("*.does-not-exist")
|
|
98
|
+
self.assertEqual(0, loaded_count)
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.2.1"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|