ovos-wikipedia-plugin 1.0.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ovos_wikipedia_plugin-1.0.0a1/PKG-INFO +131 -0
- ovos_wikipedia_plugin-1.0.0a1/README.md +117 -0
- ovos_wikipedia_plugin-1.0.0a1/ovos_wikipedia/__init__.py +390 -0
- ovos_wikipedia_plugin-1.0.0a1/ovos_wikipedia/version.py +8 -0
- ovos_wikipedia_plugin-1.0.0a1/ovos_wikipedia_plugin.egg-info/PKG-INFO +131 -0
- ovos_wikipedia_plugin-1.0.0a1/ovos_wikipedia_plugin.egg-info/SOURCES.txt +11 -0
- ovos_wikipedia_plugin-1.0.0a1/ovos_wikipedia_plugin.egg-info/dependency_links.txt +1 -0
- ovos_wikipedia_plugin-1.0.0a1/ovos_wikipedia_plugin.egg-info/entry_points.txt +5 -0
- ovos_wikipedia_plugin-1.0.0a1/ovos_wikipedia_plugin.egg-info/requires.txt +6 -0
- ovos_wikipedia_plugin-1.0.0a1/ovos_wikipedia_plugin.egg-info/top_level.txt +1 -0
- ovos_wikipedia_plugin-1.0.0a1/pyproject.toml +28 -0
- ovos_wikipedia_plugin-1.0.0a1/setup.cfg +4 -0
- ovos_wikipedia_plugin-1.0.0a1/tests/test_wikipedia_solver.py +435 -0
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ovos-wikipedia-plugin
|
|
3
|
+
Version: 1.0.0a1
|
|
4
|
+
Summary: OVOS Wikipedia plugin
|
|
5
|
+
License: Apache-2.0
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: ovos-utils<1.0.0,>=0.7.0
|
|
9
|
+
Requires-Dist: ovos-plugin-manager<3.0.0,>=2.3.0a1
|
|
10
|
+
Requires-Dist: ovos-bus-client>=1.0.1
|
|
11
|
+
Requires-Dist: rapidfuzz
|
|
12
|
+
Requires-Dist: pydantic
|
|
13
|
+
Requires-Dist: requests
|
|
14
|
+
|
|
15
|
+
# ovos-wikipedia-plugin
|
|
16
|
+
|
|
17
|
+
[](https://pypi.org/project/ovos-wikipedia-plugin/)
|
|
18
|
+
[](LICENSE)
|
|
19
|
+
[](https://www.python.org/)
|
|
20
|
+
|
|
21
|
+
Wikipedia integration for [OpenVoiceOS](https://openvoiceos.org). Provides a **retrieval engine** for RAG pipelines and
|
|
22
|
+
an **agent toolbox** for tool-using agents, both as standard OPM plugins.
|
|
23
|
+
|
|
24
|
+
---
|
|
25
|
+
|
|
26
|
+
## Installation
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install ovos-wikipedia-plugin
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
---
|
|
33
|
+
|
|
34
|
+
## OPM Entry Points
|
|
35
|
+
|
|
36
|
+
| Entry point | Class | Use case |
|
|
37
|
+
|--------------------------------------------------|----------------------------|----------------------------------------------------------|
|
|
38
|
+
| `opm.agents.retrieval` — `ovos-wikipedia-plugin` | `WikipediaRetrievalEngine` | Retrieval — returns ranked `(passage, score)` tuples |
|
|
39
|
+
| `opm.agents.toolbox` — `ovos-wikipedia-tool` | `WikipediaToolbox` | Agent tool use — exposes `search_wikipedia` |
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## Retrieval Engine
|
|
44
|
+
|
|
45
|
+
`WikipediaRetrievalEngine` implements the `RetrievalEngine` OPM interface. It searches Wikipedia, fetches article
|
|
46
|
+
summaries in parallel, scores them for relevance, and optionally applies extractive QA and reranking sub-plugins.
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
from ovos_wikipedia import WikipediaRetrievalEngine
|
|
50
|
+
|
|
51
|
+
engine = WikipediaRetrievalEngine()
|
|
52
|
+
|
|
53
|
+
# Returns List[WikipediaResult] sorted by relevance
|
|
54
|
+
for result in engine.search("Ada Lovelace", lang="en"):
|
|
55
|
+
print(result.title, result.conf)
|
|
56
|
+
print(result.summary)
|
|
57
|
+
if result.image:
|
|
58
|
+
print(result.image)
|
|
59
|
+
|
|
60
|
+
# RAG interface: List[Tuple[str, float]] (passage, score)
|
|
61
|
+
passages = engine.query("who was Ada Lovelace", lang="en", k=3)
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
### Optional sub-plugins
|
|
65
|
+
|
|
66
|
+
| Config key | OPM type | Effect |
|
|
67
|
+
|---------------------|----------------------------|-----------------------------------------------------------|
|
|
68
|
+
| `extractive_qa` | `opm.agents.extractive_qa` | Extract the best passage from each article summary |
|
|
69
|
+
| `reranker` | `opm.agents.reranker` | Re-score results with a cross-encoder |
|
|
70
|
+
| `keyword_extractor` | `opm.plugin.keywords` | Rewrite the query when the initial search returns nothing |
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
engine = WikipediaRetrievalEngine(config={
|
|
74
|
+
"extractive_qa": "ovos-bm25-solver",
|
|
75
|
+
"reranker": "ovos-bm25-reranker",
|
|
76
|
+
"keyword_extractor": "ovos-rake-keywords",
|
|
77
|
+
})
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## Agent Toolbox
|
|
83
|
+
|
|
84
|
+
`WikipediaToolbox` exposes a single `search_wikipedia` tool that any OPM-compatible agent loop (
|
|
85
|
+
e.g. [ovos-agentic-loop](https://github.com/TigreGotico/ovos-agentic-loop)) can discover and call.
|
|
86
|
+
|
|
87
|
+
### Loading via entry point (recommended)
|
|
88
|
+
|
|
89
|
+
Any agent loop that supports `opm.agents.toolbox` entry points will auto-discover this toolbox by name:
|
|
90
|
+
|
|
91
|
+
```json
|
|
92
|
+
{
|
|
93
|
+
"name": "ResearchAgent",
|
|
94
|
+
"solvers": [
|
|
95
|
+
"ovos-react-loop"
|
|
96
|
+
],
|
|
97
|
+
"ovos-react-loop": {
|
|
98
|
+
"brain": "ovos-chat-openai-plugin",
|
|
99
|
+
"ovos-chat-openai-plugin": {
|
|
100
|
+
"api_url": "http://localhost:11434/v1/chat/completions"
|
|
101
|
+
},
|
|
102
|
+
"toolboxes": [
|
|
103
|
+
"ovos-wikipedia-tool"
|
|
104
|
+
]
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
### Direct usage
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
from ovos_wikipedia import WikipediaToolbox, SearchWikipediaArgs
|
|
113
|
+
|
|
114
|
+
tb = WikipediaToolbox()
|
|
115
|
+
|
|
116
|
+
# Discover the tool (used internally by agent loops)
|
|
117
|
+
tools = tb.discover_tools()
|
|
118
|
+
# [AgentTool(name="search_wikipedia", description="Search Wikipedia for information about a topic...")]
|
|
119
|
+
|
|
120
|
+
# Call directly
|
|
121
|
+
output = tb.search_wikipedia(SearchWikipediaArgs(query="Ada Lovelace", lang="en"))
|
|
122
|
+
for title, summary in output.results:
|
|
123
|
+
print(title)
|
|
124
|
+
print(summary)
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
---
|
|
128
|
+
|
|
129
|
+
## License
|
|
130
|
+
|
|
131
|
+
Apache 2.0 — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# ovos-wikipedia-plugin
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/ovos-wikipedia-plugin/)
|
|
4
|
+
[](LICENSE)
|
|
5
|
+
[](https://www.python.org/)
|
|
6
|
+
|
|
7
|
+
Wikipedia integration for [OpenVoiceOS](https://openvoiceos.org). Provides a **retrieval engine** for RAG pipelines and
|
|
8
|
+
an **agent toolbox** for tool-using agents, both as standard OPM plugins.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## Installation
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pip install ovos-wikipedia-plugin
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
## OPM Entry Points
|
|
21
|
+
|
|
22
|
+
| Entry point | Class | Use case |
|
|
23
|
+
|--------------------------------------------------|----------------------------|----------------------------------------------------------|
|
|
24
|
+
| `opm.agents.retrieval` — `ovos-wikipedia-plugin` | `WikipediaRetrievalEngine` | Retrieval — returns ranked `(passage, score)` tuples |
|
|
25
|
+
| `opm.agents.toolbox` — `ovos-wikipedia-tool` | `WikipediaToolbox` | Agent tool use — exposes `search_wikipedia` |
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
## Retrieval Engine
|
|
30
|
+
|
|
31
|
+
`WikipediaRetrievalEngine` implements the `RetrievalEngine` OPM interface. It searches Wikipedia, fetches article
|
|
32
|
+
summaries in parallel, scores them for relevance, and optionally applies extractive QA and reranking sub-plugins.
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from ovos_wikipedia import WikipediaRetrievalEngine
|
|
36
|
+
|
|
37
|
+
engine = WikipediaRetrievalEngine()
|
|
38
|
+
|
|
39
|
+
# Returns List[WikipediaResult] sorted by relevance
|
|
40
|
+
for result in engine.search("Ada Lovelace", lang="en"):
|
|
41
|
+
print(result.title, result.conf)
|
|
42
|
+
print(result.summary)
|
|
43
|
+
if result.image:
|
|
44
|
+
print(result.image)
|
|
45
|
+
|
|
46
|
+
# RAG interface: List[Tuple[str, float]] (passage, score)
|
|
47
|
+
passages = engine.query("who was Ada Lovelace", lang="en", k=3)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
### Optional sub-plugins
|
|
51
|
+
|
|
52
|
+
| Config key | OPM type | Effect |
|
|
53
|
+
|---------------------|----------------------------|-----------------------------------------------------------|
|
|
54
|
+
| `extractive_qa` | `opm.agents.extractive_qa` | Extract the best passage from each article summary |
|
|
55
|
+
| `reranker` | `opm.agents.reranker` | Re-score results with a cross-encoder |
|
|
56
|
+
| `keyword_extractor` | `opm.plugin.keywords` | Rewrite the query when the initial search returns nothing |
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
engine = WikipediaRetrievalEngine(config={
|
|
60
|
+
"extractive_qa": "ovos-bm25-solver",
|
|
61
|
+
"reranker": "ovos-bm25-reranker",
|
|
62
|
+
"keyword_extractor": "ovos-rake-keywords",
|
|
63
|
+
})
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
---
|
|
67
|
+
|
|
68
|
+
## Agent Toolbox
|
|
69
|
+
|
|
70
|
+
`WikipediaToolbox` exposes a single `search_wikipedia` tool that any OPM-compatible agent loop (
|
|
71
|
+
e.g. [ovos-agentic-loop](https://github.com/TigreGotico/ovos-agentic-loop)) can discover and call.
|
|
72
|
+
|
|
73
|
+
### Loading via entry point (recommended)
|
|
74
|
+
|
|
75
|
+
Any agent loop that supports `opm.agents.toolbox` entry points will auto-discover this toolbox by name:
|
|
76
|
+
|
|
77
|
+
```json
|
|
78
|
+
{
|
|
79
|
+
"name": "ResearchAgent",
|
|
80
|
+
"solvers": [
|
|
81
|
+
"ovos-react-loop"
|
|
82
|
+
],
|
|
83
|
+
"ovos-react-loop": {
|
|
84
|
+
"brain": "ovos-chat-openai-plugin",
|
|
85
|
+
"ovos-chat-openai-plugin": {
|
|
86
|
+
"api_url": "http://localhost:11434/v1/chat/completions"
|
|
87
|
+
},
|
|
88
|
+
"toolboxes": [
|
|
89
|
+
"ovos-wikipedia-tool"
|
|
90
|
+
]
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### Direct usage
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from ovos_wikipedia import WikipediaToolbox, SearchWikipediaArgs
|
|
99
|
+
|
|
100
|
+
tb = WikipediaToolbox()
|
|
101
|
+
|
|
102
|
+
# Discover the tool (used internally by agent loops)
|
|
103
|
+
tools = tb.discover_tools()
|
|
104
|
+
# [AgentTool(name="search_wikipedia", description="Search Wikipedia for information about a topic...")]
|
|
105
|
+
|
|
106
|
+
# Call directly
|
|
107
|
+
output = tb.search_wikipedia(SearchWikipediaArgs(query="Ada Lovelace", lang="en"))
|
|
108
|
+
for title, summary in output.results:
|
|
109
|
+
print(title)
|
|
110
|
+
print(summary)
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
---
|
|
114
|
+
|
|
115
|
+
## License
|
|
116
|
+
|
|
117
|
+
Apache 2.0 — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,390 @@
|
|
|
1
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
2
|
+
# you may not use this file except in compliance with the License.
|
|
3
|
+
# You may obtain a copy of the License at
|
|
4
|
+
#
|
|
5
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
6
|
+
#
|
|
7
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
8
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
9
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
10
|
+
# See the License for the specific language governing permissions and
|
|
11
|
+
# limitations under the License.
|
|
12
|
+
import concurrent.futures
|
|
13
|
+
import dataclasses
|
|
14
|
+
from functools import lru_cache
|
|
15
|
+
from typing import Optional, Tuple, Dict, Any, List, Type
|
|
16
|
+
from urllib.parse import urlencode
|
|
17
|
+
|
|
18
|
+
import requests
|
|
19
|
+
from ovos_plugin_manager.agents import (
|
|
20
|
+
load_extractive_qa_plugin,
|
|
21
|
+
load_reranker_plugin,
|
|
22
|
+
)
|
|
23
|
+
from ovos_plugin_manager.keywords import load_keyword_extract_plugin
|
|
24
|
+
from ovos_plugin_manager.templates.agent_tools import AgentTool, ToolBox, ToolOutput, ToolArguments
|
|
25
|
+
from ovos_plugin_manager.templates.agents import ExtractiveQAEngine, ReRankerEngine
|
|
26
|
+
from ovos_plugin_manager.templates.agents import RetrievalEngine
|
|
27
|
+
from ovos_plugin_manager.templates.keywords import KeywordExtractor
|
|
28
|
+
from ovos_utils.log import LOG
|
|
29
|
+
from ovos_utils.parse import fuzzy_match, MatchStrategy
|
|
30
|
+
from ovos_utils.text_utils import rm_parentheses
|
|
31
|
+
from ovos_wikipedia.version import VERSION_BUILD, VERSION_MAJOR, VERSION_MINOR
|
|
32
|
+
from pydantic import Field
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclasses.dataclass
|
|
36
|
+
class WikipediaResult:
|
|
37
|
+
"""
|
|
38
|
+
Data container for a Wikipedia search result.
|
|
39
|
+
|
|
40
|
+
Attributes:
|
|
41
|
+
page_id: The unique Wikipedia page identifier.
|
|
42
|
+
lang: BCP-47 language code of the page.
|
|
43
|
+
title: Title of the Wikipedia article.
|
|
44
|
+
summary: Plain-text introductory summary.
|
|
45
|
+
image: URL to the primary article image.
|
|
46
|
+
conf: Confidence score of the result (0.0 to 1.0).
|
|
47
|
+
query: The original search query that produced this result.
|
|
48
|
+
best_passage: The specific snippet extracted by a QA engine, if available.
|
|
49
|
+
"""
|
|
50
|
+
page_id: str
|
|
51
|
+
lang: str
|
|
52
|
+
title: str
|
|
53
|
+
summary: str
|
|
54
|
+
image: Optional[str] = None
|
|
55
|
+
conf: float = 0.5
|
|
56
|
+
query: Optional[str] = None
|
|
57
|
+
best_passage: Optional[str] = None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class WikipediaRetrievalEngine(RetrievalEngine):
|
|
61
|
+
"""
|
|
62
|
+
A solver for answering questions using Wikipedia search and summaries.
|
|
63
|
+
Utilizes keyword extraction, extractive QA, and reranking plugins.
|
|
64
|
+
"""
|
|
65
|
+
USER_AGENT: str = (
|
|
66
|
+
f"ovos-wikipedia-solver/{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD} "
|
|
67
|
+
f"(https://github.com/OpenVoiceOS/ovos-wikipedia-solver)"
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
|
71
|
+
"""
|
|
72
|
+
Initializes the Wikipedia engine and loads sub-plugins.
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
config: Configuration mapping for the specific engine.
|
|
76
|
+
"""
|
|
77
|
+
super().__init__(config)
|
|
78
|
+
self._kword_extractor: Optional[KeywordExtractor] = self._load_keyword_extractor()
|
|
79
|
+
self._extractive_qa: Optional[ExtractiveQAEngine] = self._load_extractive_qa()
|
|
80
|
+
self._reranker: Optional[ReRankerEngine] = self._load_reranker()
|
|
81
|
+
LOG.debug(f"KeywordExtractor: {self._kword_extractor}")
|
|
82
|
+
LOG.debug(f"Reranker: {self._reranker}")
|
|
83
|
+
LOG.debug(f"ExtractiveQA: {self._extractive_qa}")
|
|
84
|
+
|
|
85
|
+
def query(self, query: str, lang: Optional[str] = None, k: int = 3) -> List[Tuple[str, float]]:
|
|
86
|
+
"""
|
|
87
|
+
Searches the knowledge base for relevant documents or data.
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
query: The search string.
|
|
91
|
+
lang: BCP-47 language code.
|
|
92
|
+
k: The maximum number of results to return.
|
|
93
|
+
|
|
94
|
+
Returns:
|
|
95
|
+
List of tuples containing (content, score) for the top k matches.
|
|
96
|
+
"""
|
|
97
|
+
return [(page.best_passage or page.summary, page.conf)
|
|
98
|
+
for page in self.search(query, lang, k)]
|
|
99
|
+
|
|
100
|
+
###########################
|
|
101
|
+
# plugin loading
|
|
102
|
+
###########################
|
|
103
|
+
def _load_extractive_qa(self) -> Optional[ExtractiveQAEngine]:
|
|
104
|
+
"""
|
|
105
|
+
Loads the Extractive QA plugin defined in the config.
|
|
106
|
+
|
|
107
|
+
Returns:
|
|
108
|
+
An instance of ExtractiveQAEngine or None if loading fails.
|
|
109
|
+
"""
|
|
110
|
+
plugin_name: Optional[str] = self.config.get("extractive_qa")
|
|
111
|
+
if plugin_name:
|
|
112
|
+
cls: Optional[Type[ExtractiveQAEngine]] = load_extractive_qa_plugin(plugin_name)
|
|
113
|
+
if cls:
|
|
114
|
+
try:
|
|
115
|
+
return cls()
|
|
116
|
+
except Exception as e:
|
|
117
|
+
LOG.warning(f"Extractive QA plugin '{plugin_name}' failed to load ({e}), using BM25")
|
|
118
|
+
return None
|
|
119
|
+
|
|
120
|
+
def _load_reranker(self) -> Optional[ReRankerEngine]:
|
|
121
|
+
"""
|
|
122
|
+
Loads the ReRanker plugin defined in the config.
|
|
123
|
+
|
|
124
|
+
Returns:
|
|
125
|
+
An instance of ReRankerEngine or None if loading fails.
|
|
126
|
+
"""
|
|
127
|
+
plugin_name: Optional[str] = self.config.get("reranker")
|
|
128
|
+
if plugin_name:
|
|
129
|
+
cls: Optional[Type[ReRankerEngine]] = load_reranker_plugin(plugin_name)
|
|
130
|
+
if cls:
|
|
131
|
+
try:
|
|
132
|
+
return cls()
|
|
133
|
+
except Exception as e:
|
|
134
|
+
LOG.warning(f"Reranker plugin '{plugin_name}' failed to load ({e}), using BM25")
|
|
135
|
+
return None
|
|
136
|
+
|
|
137
|
+
def _load_keyword_extractor(self) -> Optional[KeywordExtractor]:
|
|
138
|
+
"""
|
|
139
|
+
Loads the Keyword Extractor plugin defined in the config.
|
|
140
|
+
|
|
141
|
+
Returns:
|
|
142
|
+
An instance of KeywordExtractor or None if loading fails.
|
|
143
|
+
"""
|
|
144
|
+
plugin_name: Optional[str] = self.config.get("keyword_extractor")
|
|
145
|
+
if plugin_name:
|
|
146
|
+
cls: Optional[Type[KeywordExtractor]] = load_keyword_extract_plugin(plugin_name)
|
|
147
|
+
if cls:
|
|
148
|
+
try:
|
|
149
|
+
return cls()
|
|
150
|
+
except Exception as e:
|
|
151
|
+
LOG.warning(f"KeywordExtractor plugin '{plugin_name}' failed to load ({e}), using RAKE")
|
|
152
|
+
return None
|
|
153
|
+
|
|
154
|
+
###########################
|
|
155
|
+
# wikipedia integration
|
|
156
|
+
###########################
|
|
157
|
+
def search(self, query: str, lang: Optional[str] = None, top_k: int = 3) -> List[WikipediaResult]:
|
|
158
|
+
"""
|
|
159
|
+
Performs a Wikipedia search, fetches page content, and optionally reranks results.
|
|
160
|
+
|
|
161
|
+
Args:
|
|
162
|
+
query: The search string.
|
|
163
|
+
lang: BCP-47 language code (e.g., "en-US").
|
|
164
|
+
top_k: Number of search results to process.
|
|
165
|
+
|
|
166
|
+
Returns:
|
|
167
|
+
A list of WikipediaResult objects sorted by confidence.
|
|
168
|
+
"""
|
|
169
|
+
LOG.debug(f"WikiSolver query: {query}")
|
|
170
|
+
lang = (lang or self.lang).split("-")[0]
|
|
171
|
+
search_params = urlencode({
|
|
172
|
+
"action": "query",
|
|
173
|
+
"list": "search",
|
|
174
|
+
"srsearch": query,
|
|
175
|
+
"format": "json",
|
|
176
|
+
})
|
|
177
|
+
search_url = f"https://{lang}.wikipedia.org/w/api.php?{search_params}"
|
|
178
|
+
try:
|
|
179
|
+
search_results: List[Dict[str, Any]] = requests.get(
|
|
180
|
+
search_url,
|
|
181
|
+
timeout=5,
|
|
182
|
+
headers={"User-Agent": self.USER_AGENT}
|
|
183
|
+
).json().get("query", {}).get("search", [])
|
|
184
|
+
except Exception as e:
|
|
185
|
+
LOG.error(f"Error fetching search results: {e}")
|
|
186
|
+
search_results = []
|
|
187
|
+
|
|
188
|
+
if not search_results:
|
|
189
|
+
if self._kword_extractor:
|
|
190
|
+
keywords: List[str] = self._kword_extractor.extract(query, lang=lang)
|
|
191
|
+
if keywords:
|
|
192
|
+
fallback_query = max(keywords, key=len)
|
|
193
|
+
if fallback_query and fallback_query != query:
|
|
194
|
+
LOG.debug(f"WikiSolver Fallback, new query: {fallback_query}")
|
|
195
|
+
return self.search(fallback_query, lang=lang, top_k=top_k)
|
|
196
|
+
return []
|
|
197
|
+
|
|
198
|
+
LOG.debug(f"Matched {len(search_results)} Wikipedia pages, using top {top_k}")
|
|
199
|
+
search_results = search_results[:top_k]
|
|
200
|
+
|
|
201
|
+
# Prepare for parallel fetch and maintain original order
|
|
202
|
+
summaries: List[Optional[WikipediaResult]] = [None] * len(search_results)
|
|
203
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
|
|
204
|
+
future_to_idx = {
|
|
205
|
+
executor.submit(self._get_page_data, str(r["pageid"]), lang, query, idx): idx
|
|
206
|
+
for idx, r in enumerate(search_results)
|
|
207
|
+
if "(disambiguation)" not in r["title"]
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
for future in concurrent.futures.as_completed(future_to_idx):
|
|
211
|
+
idx = future_to_idx[future]
|
|
212
|
+
summaries[idx] = future.result()
|
|
213
|
+
|
|
214
|
+
res: List[WikipediaResult] = [s for s in summaries if s is not None]
|
|
215
|
+
|
|
216
|
+
if self._extractive_qa:
|
|
217
|
+
for page in res:
|
|
218
|
+
page.best_passage = self._extractive_qa.get_best_passage(
|
|
219
|
+
evidence=page.summary,
|
|
220
|
+
question=query,
|
|
221
|
+
lang=lang
|
|
222
|
+
).strip()
|
|
223
|
+
|
|
224
|
+
return self._rerank_pages(query, res, lang)
|
|
225
|
+
|
|
226
|
+
@classmethod
|
|
227
|
+
def _get_page_data(cls, pid: str, lang: str, query: Optional[str] = None, idx: int = 0) -> Optional[
|
|
228
|
+
WikipediaResult]:
|
|
229
|
+
"""
|
|
230
|
+
Fetch the title, plain-text intro summary, and an image URL for a Wikipedia page.
|
|
231
|
+
|
|
232
|
+
Args:
|
|
233
|
+
pid: Wikipedia pageid.
|
|
234
|
+
lang: Language code (e.g., "en", "pt").
|
|
235
|
+
query: The original query for scoring purposes.
|
|
236
|
+
idx: The original search index for scoring purposes.
|
|
237
|
+
|
|
238
|
+
Returns:
|
|
239
|
+
A WikipediaResult object if successful, else None.
|
|
240
|
+
"""
|
|
241
|
+
url = (
|
|
242
|
+
f"https://{lang}.wikipedia.org/w/api.php?format=json&action=query&"
|
|
243
|
+
f"prop=extracts|pageimages&exintro&explaintext&redirects=1&pageids={pid}"
|
|
244
|
+
)
|
|
245
|
+
try:
|
|
246
|
+
disambiguation_indicators = ["may refer to:", "refers to:"]
|
|
247
|
+
response = requests.get(url, timeout=5, headers={"User-Agent": cls.USER_AGENT}).json()
|
|
248
|
+
page = response["query"]["pages"][pid]
|
|
249
|
+
|
|
250
|
+
summary = rm_parentheses(page.get("extract", ""))
|
|
251
|
+
if any(i in summary for i in disambiguation_indicators):
|
|
252
|
+
return None # Disambiguation list page
|
|
253
|
+
|
|
254
|
+
img: Optional[str] = None
|
|
255
|
+
if "thumbnail" in page:
|
|
256
|
+
thumbnail = page["thumbnail"]["source"]
|
|
257
|
+
parts = thumbnail.split("/")[:-1]
|
|
258
|
+
img = "/".join(part for part in parts if part != "thumb")
|
|
259
|
+
|
|
260
|
+
result = WikipediaResult(
|
|
261
|
+
lang=lang,
|
|
262
|
+
title=page["title"],
|
|
263
|
+
summary=summary.strip(),
|
|
264
|
+
image=img,
|
|
265
|
+
page_id=pid,
|
|
266
|
+
conf=cls._score_page(query or "", page["title"], summary, idx),
|
|
267
|
+
query=query
|
|
268
|
+
)
|
|
269
|
+
return result
|
|
270
|
+
except Exception as e:
|
|
271
|
+
LOG.error(f"Error fetching page data for PID {pid}: {e}")
|
|
272
|
+
return None
|
|
273
|
+
|
|
274
|
+
###########################
|
|
275
|
+
# helpers to score results
|
|
276
|
+
###########################
|
|
277
|
+
@staticmethod
|
|
278
|
+
@lru_cache(maxsize=128)
|
|
279
|
+
def _score_page(query: str, title: str, summary: str, idx: int) -> float:
|
|
280
|
+
"""
|
|
281
|
+
Compute a relevance score for a Wikipedia page given a search query.
|
|
282
|
+
|
|
283
|
+
Args:
|
|
284
|
+
query: The user's search query.
|
|
285
|
+
title: The page title.
|
|
286
|
+
summary: The page summary text.
|
|
287
|
+
idx: The page's index in the original search results.
|
|
288
|
+
|
|
289
|
+
Returns:
|
|
290
|
+
Relevance score where higher values indicate greater relevance.
|
|
291
|
+
"""
|
|
292
|
+
page_mod = 1 - (idx * 0.05) # Favor original order returned by Wikipedia
|
|
293
|
+
title_score = fuzzy_match(query, rm_parentheses(title), MatchStrategy.TOKEN_SET_RATIO)
|
|
294
|
+
summary_score = fuzzy_match(summary, query, MatchStrategy.TOKEN_SET_RATIO)
|
|
295
|
+
return title_score * summary_score * page_mod
|
|
296
|
+
|
|
297
|
+
def _rerank_pages(self, query: str, results: List[WikipediaResult], lang: str) -> List[WikipediaResult]:
|
|
298
|
+
"""
|
|
299
|
+
Uses the loaded ReRanker plugin to re-score the list of results.
|
|
300
|
+
|
|
301
|
+
Args:
|
|
302
|
+
query: The user's search query.
|
|
303
|
+
results: List of fetched WikipediaResult objects.
|
|
304
|
+
lang: Language code.
|
|
305
|
+
|
|
306
|
+
Returns:
|
|
307
|
+
List of WikipediaResult objects sorted by updated confidence scores.
|
|
308
|
+
"""
|
|
309
|
+
if self._reranker is not None:
|
|
310
|
+
ranked = self._reranker.rerank(query, [r.best_passage or r.summary for r in results], lang)
|
|
311
|
+
for idx, (score, summary) in enumerate(ranked):
|
|
312
|
+
LOG.debug(f"{results[idx].title}: {results[idx].conf} -> {score}")
|
|
313
|
+
results[idx].conf = score
|
|
314
|
+
return sorted(results, key=lambda k: k.conf, reverse=True)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
class SearchWikipediaArgs(ToolArguments):
|
|
318
|
+
"""Input arguments for the search_wikipedia tool."""
|
|
319
|
+
query: str = Field(..., description="The search query to look up on Wikipedia.")
|
|
320
|
+
lang: str = Field(..., description="BCP-47 language code, e.g. 'en', 'pt', 'de'.")
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
class SearchWikipediaOutput(ToolOutput):
|
|
324
|
+
"""Output from the search_wikipedia tool."""
|
|
325
|
+
results: List[Tuple[str, str]] = Field(
|
|
326
|
+
...,
|
|
327
|
+
description="List of (title, summary) pairs for the top matching Wikipedia articles."
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
class WikipediaToolbox(ToolBox):
|
|
332
|
+
toolbox_id = "ovos-wikipedia-tools"
|
|
333
|
+
|
|
334
|
+
def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
|
|
335
|
+
"""
|
|
336
|
+
Initialise the toolbox.
|
|
337
|
+
|
|
338
|
+
Args:
|
|
339
|
+
config: Plugin configuration dict (currently unused).
|
|
340
|
+
"""
|
|
341
|
+
self.wiki = WikipediaRetrievalEngine(config)
|
|
342
|
+
super().__init__(toolbox_id=self.toolbox_id)
|
|
343
|
+
|
|
344
|
+
def search_wikipedia(self, args: SearchWikipediaArgs) -> SearchWikipediaOutput:
|
|
345
|
+
"""
|
|
346
|
+
Search Wikipedia for articles matching the given query.
|
|
347
|
+
|
|
348
|
+
Args:
|
|
349
|
+
args: Validated ``SearchWikipediaArgs`` with ``query`` and ``lang`` fields.
|
|
350
|
+
|
|
351
|
+
Returns:
|
|
352
|
+
``SearchWikipediaOutput`` with a list of (title, summary) pairs.
|
|
353
|
+
"""
|
|
354
|
+
return SearchWikipediaOutput(
|
|
355
|
+
results=[(res.title, res.summary)
|
|
356
|
+
for res in self.wiki.search(query=args.query, lang=args.lang)],
|
|
357
|
+
)
|
|
358
|
+
|
|
359
|
+
def discover_tools(self) -> List[AgentTool]:
|
|
360
|
+
"""
|
|
361
|
+
Abstract method to be implemented by concrete ToolBox plugins.
|
|
362
|
+
|
|
363
|
+
This method must define and return the list of AgentTools provided by this plugin.
|
|
364
|
+
The implementation should be idempotent (safe to call multiple times).
|
|
365
|
+
|
|
366
|
+
Returns:
|
|
367
|
+
A list of instantiated AgentTool objects.
|
|
368
|
+
"""
|
|
369
|
+
return [
|
|
370
|
+
AgentTool(
|
|
371
|
+
name="search_wikipedia",
|
|
372
|
+
description=(
|
|
373
|
+
"Search Wikipedia for information about a topic. "
|
|
374
|
+
"Returns a list of matching article titles and their introductory summaries."
|
|
375
|
+
),
|
|
376
|
+
argument_schema=SearchWikipediaArgs,
|
|
377
|
+
output_schema=SearchWikipediaOutput,
|
|
378
|
+
tool_call=self.search_wikipedia,
|
|
379
|
+
)
|
|
380
|
+
]
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
if __name__ == "__main__":
|
|
384
|
+
LOG.set_level("ERROR")
|
|
385
|
+
|
|
386
|
+
s = WikipediaRetrievalEngine()
|
|
387
|
+
|
|
388
|
+
data = s.search("who is venus")
|
|
389
|
+
for r in data:
|
|
390
|
+
print(r)
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ovos-wikipedia-plugin
|
|
3
|
+
Version: 1.0.0a1
|
|
4
|
+
Summary: OVOS Wikipedia plugin
|
|
5
|
+
License: Apache-2.0
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: ovos-utils<1.0.0,>=0.7.0
|
|
9
|
+
Requires-Dist: ovos-plugin-manager<3.0.0,>=2.3.0a1
|
|
10
|
+
Requires-Dist: ovos-bus-client>=1.0.1
|
|
11
|
+
Requires-Dist: rapidfuzz
|
|
12
|
+
Requires-Dist: pydantic
|
|
13
|
+
Requires-Dist: requests
|
|
14
|
+
|
|
15
|
+
# ovos-wikipedia-plugin
|
|
16
|
+
|
|
17
|
+
[](https://pypi.org/project/ovos-wikipedia-plugin/)
|
|
18
|
+
[](LICENSE)
|
|
19
|
+
[](https://www.python.org/)
|
|
20
|
+
|
|
21
|
+
Wikipedia integration for [OpenVoiceOS](https://openvoiceos.org). Provides a **retrieval engine** for RAG pipelines and
|
|
22
|
+
an **agent toolbox** for tool-using agents, both as standard OPM plugins.
|
|
23
|
+
|
|
24
|
+
---
|
|
25
|
+
|
|
26
|
+
## Installation
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install ovos-wikipedia-plugin
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
---
|
|
33
|
+
|
|
34
|
+
## OPM Entry Points
|
|
35
|
+
|
|
36
|
+
| Entry point | Class | Use case |
|
|
37
|
+
|--------------------------------------------------|----------------------------|----------------------------------------------------------|
|
|
38
|
+
| `opm.agents.retrieval` — `ovos-wikipedia-plugin` | `WikipediaRetrievalEngine` | Retrieval — returns ranked `(passage, score)` tuples |
|
|
39
|
+
| `opm.agents.toolbox` — `ovos-wikipedia-tool` | `WikipediaToolbox` | Agent tool use — exposes `search_wikipedia` |
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## Retrieval Engine
|
|
44
|
+
|
|
45
|
+
`WikipediaRetrievalEngine` implements the `RetrievalEngine` OPM interface. It searches Wikipedia, fetches article
|
|
46
|
+
summaries in parallel, scores them for relevance, and optionally applies extractive QA and reranking sub-plugins.
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
from ovos_wikipedia import WikipediaRetrievalEngine
|
|
50
|
+
|
|
51
|
+
engine = WikipediaRetrievalEngine()
|
|
52
|
+
|
|
53
|
+
# Returns List[WikipediaResult] sorted by relevance
|
|
54
|
+
for result in engine.search("Ada Lovelace", lang="en"):
|
|
55
|
+
print(result.title, result.conf)
|
|
56
|
+
print(result.summary)
|
|
57
|
+
if result.image:
|
|
58
|
+
print(result.image)
|
|
59
|
+
|
|
60
|
+
# RAG interface: List[Tuple[str, float]] (passage, score)
|
|
61
|
+
passages = engine.query("who was Ada Lovelace", lang="en", k=3)
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
### Optional sub-plugins
|
|
65
|
+
|
|
66
|
+
| Config key | OPM type | Effect |
|
|
67
|
+
|---------------------|----------------------------|-----------------------------------------------------------|
|
|
68
|
+
| `extractive_qa` | `opm.agents.extractive_qa` | Extract the best passage from each article summary |
|
|
69
|
+
| `reranker` | `opm.agents.reranker` | Re-score results with a cross-encoder |
|
|
70
|
+
| `keyword_extractor` | `opm.plugin.keywords` | Rewrite the query when the initial search returns nothing |
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
engine = WikipediaRetrievalEngine(config={
|
|
74
|
+
"extractive_qa": "ovos-bm25-solver",
|
|
75
|
+
"reranker": "ovos-bm25-reranker",
|
|
76
|
+
"keyword_extractor": "ovos-rake-keywords",
|
|
77
|
+
})
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## Agent Toolbox
|
|
83
|
+
|
|
84
|
+
`WikipediaToolbox` exposes a single `search_wikipedia` tool that any OPM-compatible agent loop (
|
|
85
|
+
e.g. [ovos-agentic-loop](https://github.com/TigreGotico/ovos-agentic-loop)) can discover and call.
|
|
86
|
+
|
|
87
|
+
### Loading via entry point (recommended)
|
|
88
|
+
|
|
89
|
+
Any agent loop that supports `opm.agents.toolbox` entry points will auto-discover this toolbox by name:
|
|
90
|
+
|
|
91
|
+
```json
|
|
92
|
+
{
|
|
93
|
+
"name": "ResearchAgent",
|
|
94
|
+
"solvers": [
|
|
95
|
+
"ovos-react-loop"
|
|
96
|
+
],
|
|
97
|
+
"ovos-react-loop": {
|
|
98
|
+
"brain": "ovos-chat-openai-plugin",
|
|
99
|
+
"ovos-chat-openai-plugin": {
|
|
100
|
+
"api_url": "http://localhost:11434/v1/chat/completions"
|
|
101
|
+
},
|
|
102
|
+
"toolboxes": [
|
|
103
|
+
"ovos-wikipedia-tool"
|
|
104
|
+
]
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
### Direct usage
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
from ovos_wikipedia import WikipediaToolbox, SearchWikipediaArgs
|
|
113
|
+
|
|
114
|
+
tb = WikipediaToolbox()
|
|
115
|
+
|
|
116
|
+
# Discover the tool (used internally by agent loops)
|
|
117
|
+
tools = tb.discover_tools()
|
|
118
|
+
# [AgentTool(name="search_wikipedia", description="Search Wikipedia for information about a topic...")]
|
|
119
|
+
|
|
120
|
+
# Call directly
|
|
121
|
+
output = tb.search_wikipedia(SearchWikipediaArgs(query="Ada Lovelace", lang="en"))
|
|
122
|
+
for title, summary in output.results:
|
|
123
|
+
print(title)
|
|
124
|
+
print(summary)
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
---
|
|
128
|
+
|
|
129
|
+
## License
|
|
130
|
+
|
|
131
|
+
Apache 2.0 — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
ovos_wikipedia/__init__.py
|
|
4
|
+
ovos_wikipedia/version.py
|
|
5
|
+
ovos_wikipedia_plugin.egg-info/PKG-INFO
|
|
6
|
+
ovos_wikipedia_plugin.egg-info/SOURCES.txt
|
|
7
|
+
ovos_wikipedia_plugin.egg-info/dependency_links.txt
|
|
8
|
+
ovos_wikipedia_plugin.egg-info/entry_points.txt
|
|
9
|
+
ovos_wikipedia_plugin.egg-info/requires.txt
|
|
10
|
+
ovos_wikipedia_plugin.egg-info/top_level.txt
|
|
11
|
+
tests/test_wikipedia_solver.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
ovos_wikipedia
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=42", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ovos-wikipedia-plugin"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "OVOS Wikipedia plugin"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "Apache-2.0" }
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
dependencies = [
|
|
13
|
+
"ovos-utils>=0.7.0,<1.0.0",
|
|
14
|
+
"ovos-plugin-manager>=2.3.0a1,<3.0.0",
|
|
15
|
+
"ovos-bus-client>=1.0.1",
|
|
16
|
+
"rapidfuzz",
|
|
17
|
+
"pydantic",
|
|
18
|
+
"requests"
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.entry-points."opm.agents.retrieval"]
|
|
22
|
+
ovos-wikipedia-plugin = "ovos_wikipedia:WikipediaRetrievalEngine"
|
|
23
|
+
|
|
24
|
+
[project.entry-points."opm.agents.toolbox"]
|
|
25
|
+
ovos-wikipedia-tool = "ovos_wikipedia:WikipediaToolbox"
|
|
26
|
+
|
|
27
|
+
[tool.setuptools.dynamic]
|
|
28
|
+
version = {attr = "ovos_wikipedia.version.__version__"}
|
|
@@ -0,0 +1,435 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Unit tests for WikipediaRetrievalEngine and WikipediaToolbox.
|
|
3
|
+
|
|
4
|
+
Uses mocked HTTP calls (no network required) unless marked as integration tests.
|
|
5
|
+
"""
|
|
6
|
+
import unittest
|
|
7
|
+
from unittest.mock import patch, MagicMock
|
|
8
|
+
|
|
9
|
+
from ovos_wikipedia import WikipediaRetrievalEngine, WikipediaResult, WikipediaToolbox
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _make_engine(**cfg) -> WikipediaRetrievalEngine:
|
|
13
|
+
return WikipediaRetrievalEngine(config=cfg or {})
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _fake_result(title="Test Page", summary="Test summary about the topic.", conf=0.8) -> WikipediaResult:
|
|
17
|
+
return WikipediaResult(
|
|
18
|
+
page_id="123",
|
|
19
|
+
lang="en",
|
|
20
|
+
title=title,
|
|
21
|
+
summary=summary,
|
|
22
|
+
conf=conf,
|
|
23
|
+
query="test query",
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# ---------------------------------------------------------------------------
|
|
28
|
+
# WikipediaResult dataclass
|
|
29
|
+
# ---------------------------------------------------------------------------
|
|
30
|
+
|
|
31
|
+
class TestWikipediaResult(unittest.TestCase):
|
|
32
|
+
|
|
33
|
+
def test_required_fields(self):
|
|
34
|
+
r = WikipediaResult(page_id="1", lang="en", title="T", summary="S")
|
|
35
|
+
self.assertEqual(r.page_id, "1")
|
|
36
|
+
self.assertEqual(r.lang, "en")
|
|
37
|
+
self.assertIsNone(r.image)
|
|
38
|
+
self.assertIsNone(r.best_passage)
|
|
39
|
+
self.assertEqual(r.conf, 0.5)
|
|
40
|
+
|
|
41
|
+
def test_optional_fields(self):
|
|
42
|
+
r = WikipediaResult(page_id="1", lang="en", title="T", summary="S",
|
|
43
|
+
image="http://img", conf=0.9, query="q", best_passage="p")
|
|
44
|
+
self.assertEqual(r.image, "http://img")
|
|
45
|
+
self.assertEqual(r.conf, 0.9)
|
|
46
|
+
self.assertEqual(r.best_passage, "p")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# ---------------------------------------------------------------------------
|
|
50
|
+
# Plugin loading
|
|
51
|
+
# ---------------------------------------------------------------------------
|
|
52
|
+
|
|
53
|
+
class TestPluginLoading(unittest.TestCase):
|
|
54
|
+
|
|
55
|
+
def test_no_plugins_when_config_empty(self):
|
|
56
|
+
s = _make_engine()
|
|
57
|
+
self.assertIsNone(s._extractive_qa)
|
|
58
|
+
self.assertIsNone(s._reranker)
|
|
59
|
+
self.assertIsNone(s._kword_extractor)
|
|
60
|
+
|
|
61
|
+
def test_extractive_qa_loads_when_plugin_available(self):
|
|
62
|
+
mock_cls = MagicMock(return_value=MagicMock())
|
|
63
|
+
with patch("ovos_wikipedia.load_extractive_qa_plugin", return_value=mock_cls):
|
|
64
|
+
s = WikipediaRetrievalEngine(config={"extractive_qa": "some-plugin"})
|
|
65
|
+
self.assertIsNotNone(s._extractive_qa)
|
|
66
|
+
|
|
67
|
+
def test_reranker_loads_when_plugin_available(self):
|
|
68
|
+
mock_cls = MagicMock(return_value=MagicMock())
|
|
69
|
+
with patch("ovos_wikipedia.load_reranker_plugin", return_value=mock_cls):
|
|
70
|
+
s = WikipediaRetrievalEngine(config={"reranker": "some-plugin"})
|
|
71
|
+
self.assertIsNotNone(s._reranker)
|
|
72
|
+
|
|
73
|
+
def test_keyword_extractor_loads_when_plugin_available(self):
|
|
74
|
+
mock_cls = MagicMock(return_value=MagicMock())
|
|
75
|
+
with patch("ovos_wikipedia.load_keyword_extract_plugin", return_value=mock_cls):
|
|
76
|
+
s = WikipediaRetrievalEngine(config={"keyword_extractor": "some-plugin"})
|
|
77
|
+
self.assertIsNotNone(s._kword_extractor)
|
|
78
|
+
|
|
79
|
+
def test_unknown_extractive_qa_plugin_returns_none(self):
|
|
80
|
+
with patch("ovos_wikipedia.load_extractive_qa_plugin", return_value=None):
|
|
81
|
+
s = WikipediaRetrievalEngine(config={"extractive_qa": "nonexistent-plugin"})
|
|
82
|
+
self.assertIsNone(s._extractive_qa)
|
|
83
|
+
|
|
84
|
+
def test_unknown_reranker_plugin_returns_none(self):
|
|
85
|
+
with patch("ovos_wikipedia.load_reranker_plugin", return_value=None):
|
|
86
|
+
s = WikipediaRetrievalEngine(config={"reranker": "nonexistent-plugin"})
|
|
87
|
+
self.assertIsNone(s._reranker)
|
|
88
|
+
|
|
89
|
+
def test_plugin_init_exception_returns_none(self):
|
|
90
|
+
mock_cls = MagicMock(side_effect=RuntimeError("init failed"))
|
|
91
|
+
with patch("ovos_wikipedia.load_extractive_qa_plugin", return_value=mock_cls):
|
|
92
|
+
s = WikipediaRetrievalEngine(config={"extractive_qa": "bad-plugin"})
|
|
93
|
+
self.assertIsNone(s._extractive_qa)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# ---------------------------------------------------------------------------
|
|
97
|
+
# _score_page (pure function)
|
|
98
|
+
# ---------------------------------------------------------------------------
|
|
99
|
+
|
|
100
|
+
class TestScorePage(unittest.TestCase):
|
|
101
|
+
|
|
102
|
+
def test_perfect_title_match_scores_high(self):
|
|
103
|
+
WikipediaRetrievalEngine._score_page.cache_clear()
|
|
104
|
+
score = WikipediaRetrievalEngine._score_page("python", "Python", "Python is a programming language.", 0)
|
|
105
|
+
self.assertGreater(score, 0.1)
|
|
106
|
+
|
|
107
|
+
def test_unrelated_title_scores_low(self):
|
|
108
|
+
WikipediaRetrievalEngine._score_page.cache_clear()
|
|
109
|
+
score = WikipediaRetrievalEngine._score_page("python language", "Banana", "Banana is a tropical fruit.", 0)
|
|
110
|
+
self.assertLess(score, 0.5)
|
|
111
|
+
|
|
112
|
+
def test_later_index_scores_lower(self):
|
|
113
|
+
WikipediaRetrievalEngine._score_page.cache_clear()
|
|
114
|
+
score_first = WikipediaRetrievalEngine._score_page("python", "Python", "Python language.", 0)
|
|
115
|
+
score_later = WikipediaRetrievalEngine._score_page("python", "Python", "Python language.", 3)
|
|
116
|
+
self.assertGreater(score_first, score_later)
|
|
117
|
+
|
|
118
|
+
def test_empty_query_does_not_crash(self):
|
|
119
|
+
WikipediaRetrievalEngine._score_page.cache_clear()
|
|
120
|
+
score = WikipediaRetrievalEngine._score_page("", "Python", "Python is a programming language.", 0)
|
|
121
|
+
self.assertIsInstance(score, float)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
# ---------------------------------------------------------------------------
|
|
125
|
+
# _rerank_pages
|
|
126
|
+
# ---------------------------------------------------------------------------
|
|
127
|
+
|
|
128
|
+
class TestRerankPages(unittest.TestCase):
|
|
129
|
+
|
|
130
|
+
def setUp(self):
|
|
131
|
+
self.s = _make_engine()
|
|
132
|
+
|
|
133
|
+
def test_sorted_by_conf_descending_without_reranker(self):
|
|
134
|
+
pages = [
|
|
135
|
+
_fake_result(title="B", conf=0.3),
|
|
136
|
+
_fake_result(title="A", conf=0.9),
|
|
137
|
+
_fake_result(title="C", conf=0.6),
|
|
138
|
+
]
|
|
139
|
+
result = self.s._rerank_pages("query", pages, "en")
|
|
140
|
+
confs = [r.conf for r in result]
|
|
141
|
+
self.assertEqual(confs, sorted(confs, reverse=True))
|
|
142
|
+
|
|
143
|
+
def test_reranker_updates_conf_scores(self):
|
|
144
|
+
mock_reranker = MagicMock()
|
|
145
|
+
mock_reranker.rerank.return_value = [(0.95, "summary A"), (0.1, "summary B")]
|
|
146
|
+
self.s._reranker = mock_reranker
|
|
147
|
+
|
|
148
|
+
pages = [
|
|
149
|
+
_fake_result(title="A", conf=0.3, summary="summary A"),
|
|
150
|
+
_fake_result(title="B", conf=0.8, summary="summary B"),
|
|
151
|
+
]
|
|
152
|
+
result = self.s._rerank_pages("query", pages, "en")
|
|
153
|
+
self.assertEqual(result[0].conf, 0.95)
|
|
154
|
+
self.assertEqual(result[1].conf, 0.1)
|
|
155
|
+
|
|
156
|
+
def test_reranker_called_with_summaries(self):
|
|
157
|
+
mock_reranker = MagicMock()
|
|
158
|
+
mock_reranker.rerank.return_value = [(0.9, "s1"), (0.8, "s2")]
|
|
159
|
+
self.s._reranker = mock_reranker
|
|
160
|
+
|
|
161
|
+
p1 = _fake_result(summary="s1")
|
|
162
|
+
p2 = _fake_result(summary="s2")
|
|
163
|
+
self.s._rerank_pages("q", [p1, p2], "en")
|
|
164
|
+
mock_reranker.rerank.assert_called_once_with("q", ["s1", "s2"], "en")
|
|
165
|
+
|
|
166
|
+
def test_best_passage_used_over_summary_for_reranker(self):
|
|
167
|
+
mock_reranker = MagicMock()
|
|
168
|
+
mock_reranker.rerank.return_value = [(0.9, "passage1")]
|
|
169
|
+
self.s._reranker = mock_reranker
|
|
170
|
+
|
|
171
|
+
p = _fake_result(summary="full summary")
|
|
172
|
+
p.best_passage = "passage1"
|
|
173
|
+
self.s._rerank_pages("q", [p], "en")
|
|
174
|
+
mock_reranker.rerank.assert_called_once_with("q", ["passage1"], "en")
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
# ---------------------------------------------------------------------------
|
|
178
|
+
# _get_page_data (mocked HTTP)
|
|
179
|
+
# ---------------------------------------------------------------------------
|
|
180
|
+
|
|
181
|
+
class TestGetPageData(unittest.TestCase):
|
|
182
|
+
|
|
183
|
+
def _make_response(self, pid, title, extract, thumbnail=None):
|
|
184
|
+
page = {"title": title, "extract": extract}
|
|
185
|
+
if thumbnail:
|
|
186
|
+
page["thumbnail"] = {"source": thumbnail}
|
|
187
|
+
return {"query": {"pages": {pid: page}}}
|
|
188
|
+
|
|
189
|
+
@patch("ovos_wikipedia.requests.get")
|
|
190
|
+
def test_returns_wikipedia_result(self, mock_get):
|
|
191
|
+
mock_get.return_value.json.return_value = self._make_response(
|
|
192
|
+
"42", "Python (programming language)", "Python is a high-level language."
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
result = WikipediaRetrievalEngine._get_page_data("42", "en", "python", 0)
|
|
196
|
+
self.assertIsInstance(result, WikipediaResult)
|
|
197
|
+
self.assertEqual(result.title, "Python (programming language)")
|
|
198
|
+
self.assertEqual(result.lang, "en")
|
|
199
|
+
self.assertEqual(result.page_id, "42")
|
|
200
|
+
|
|
201
|
+
@patch("ovos_wikipedia.requests.get")
|
|
202
|
+
def test_disambiguation_page_returns_none(self, mock_get):
|
|
203
|
+
mock_get.return_value.json.return_value = self._make_response(
|
|
204
|
+
"99", "Mercury", "Mercury may refer to: the planet, the element, the god."
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
result = WikipediaRetrievalEngine._get_page_data("99", "en", "mercury", 0)
|
|
208
|
+
self.assertIsNone(result)
|
|
209
|
+
|
|
210
|
+
@patch("ovos_wikipedia.requests.get")
|
|
211
|
+
def test_image_url_strips_thumb_path(self, mock_get):
|
|
212
|
+
thumbnail = "https://upload.wikimedia.org/thumb/img/320px-img.jpg"
|
|
213
|
+
mock_get.return_value.json.return_value = self._make_response(
|
|
214
|
+
"7", "Eiffel Tower", "The Eiffel Tower is in Paris.", thumbnail=thumbnail
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
result = WikipediaRetrievalEngine._get_page_data("7", "en", "eiffel tower", 0)
|
|
218
|
+
self.assertIsNotNone(result.image)
|
|
219
|
+
self.assertNotIn("thumb", result.image)
|
|
220
|
+
self.assertNotIn("320px-img.jpg", result.image)
|
|
221
|
+
|
|
222
|
+
@patch("ovos_wikipedia.requests.get", side_effect=Exception("network error"))
|
|
223
|
+
def test_network_error_returns_none(self, mock_get):
|
|
224
|
+
|
|
225
|
+
result = WikipediaRetrievalEngine._get_page_data("1", "en", "q", 0)
|
|
226
|
+
self.assertIsNone(result)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
# ---------------------------------------------------------------------------
|
|
230
|
+
# search (mocked HTTP)
|
|
231
|
+
# ---------------------------------------------------------------------------
|
|
232
|
+
|
|
233
|
+
_SEARCH_RESULTS = {
|
|
234
|
+
"query": {
|
|
235
|
+
"search": [
|
|
236
|
+
{"pageid": 1001, "title": "Python (programming language)"},
|
|
237
|
+
{"pageid": 1002, "title": "Python (snake)"},
|
|
238
|
+
]
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
_PAGE_DATA_1001 = {
|
|
243
|
+
"query": {"pages": {"1001": {"title": "Python (programming language)", "extract": "Python is a language."}}}
|
|
244
|
+
}
|
|
245
|
+
_PAGE_DATA_1002 = {
|
|
246
|
+
"query": {"pages": {"1002": {"title": "Python (snake)", "extract": "Python is a large snake."}}}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
class TestSearch(unittest.TestCase):
|
|
251
|
+
|
|
252
|
+
def setUp(self):
|
|
253
|
+
self.s = _make_engine()
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _patch_requests(self, search_data, page_data_map):
|
|
257
|
+
def side_effect(url, **kwargs):
|
|
258
|
+
mock_resp = MagicMock()
|
|
259
|
+
if "list=search" in url:
|
|
260
|
+
mock_resp.json.return_value = search_data
|
|
261
|
+
else:
|
|
262
|
+
for pid, data in page_data_map.items():
|
|
263
|
+
if f"pageids={pid}" in url:
|
|
264
|
+
mock_resp.json.return_value = data
|
|
265
|
+
break
|
|
266
|
+
else:
|
|
267
|
+
mock_resp.json.return_value = {"query": {"pages": {}}}
|
|
268
|
+
return mock_resp
|
|
269
|
+
return side_effect
|
|
270
|
+
|
|
271
|
+
@patch("ovos_wikipedia.requests.get")
|
|
272
|
+
def test_returns_list_of_results(self, mock_get):
|
|
273
|
+
mock_get.side_effect = self._patch_requests(
|
|
274
|
+
_SEARCH_RESULTS, {"1001": _PAGE_DATA_1001, "1002": _PAGE_DATA_1002}
|
|
275
|
+
)
|
|
276
|
+
results = self.s.search("python", "en", top_k=2)
|
|
277
|
+
self.assertIsInstance(results, list)
|
|
278
|
+
self.assertGreater(len(results), 0)
|
|
279
|
+
self.assertIsInstance(results[0], WikipediaResult)
|
|
280
|
+
|
|
281
|
+
@patch("ovos_wikipedia.requests.get")
|
|
282
|
+
def test_results_sorted_by_conf_descending(self, mock_get):
|
|
283
|
+
mock_get.side_effect = self._patch_requests(
|
|
284
|
+
_SEARCH_RESULTS, {"1001": _PAGE_DATA_1001, "1002": _PAGE_DATA_1002}
|
|
285
|
+
)
|
|
286
|
+
results = self.s.search("python", "en", top_k=2)
|
|
287
|
+
confs = [r.conf for r in results]
|
|
288
|
+
self.assertEqual(confs, sorted(confs, reverse=True))
|
|
289
|
+
|
|
290
|
+
@patch("ovos_wikipedia.requests.get")
|
|
291
|
+
def test_empty_search_returns_empty_list(self, mock_get):
|
|
292
|
+
mock_get.return_value.json.return_value = {"query": {"search": []}}
|
|
293
|
+
results = self.s.search("xyzzy_nonexistent_12345", "en")
|
|
294
|
+
self.assertEqual(results, [])
|
|
295
|
+
|
|
296
|
+
@patch("ovos_wikipedia.requests.get")
|
|
297
|
+
def test_network_error_returns_empty_list(self, mock_get):
|
|
298
|
+
mock_get.side_effect = Exception("timeout")
|
|
299
|
+
results = self.s.search("python", "en")
|
|
300
|
+
self.assertEqual(results, [])
|
|
301
|
+
|
|
302
|
+
@patch("ovos_wikipedia.requests.get")
|
|
303
|
+
def test_disambiguation_pages_are_skipped(self, mock_get):
|
|
304
|
+
search_data = {"query": {"search": [{"pageid": 555, "title": "Mercury (disambiguation)"}]}}
|
|
305
|
+
mock_get.side_effect = self._patch_requests(search_data, {})
|
|
306
|
+
results = self.s.search("mercury", "en")
|
|
307
|
+
self.assertEqual(results, [])
|
|
308
|
+
|
|
309
|
+
@patch("ovos_wikipedia.requests.get")
|
|
310
|
+
def test_extractive_qa_called_when_loaded(self, mock_get):
|
|
311
|
+
mock_get.side_effect = self._patch_requests(
|
|
312
|
+
_SEARCH_RESULTS, {"1001": _PAGE_DATA_1001, "1002": _PAGE_DATA_1002}
|
|
313
|
+
)
|
|
314
|
+
mock_qa = MagicMock()
|
|
315
|
+
mock_qa.get_best_passage.return_value = "Python is a language."
|
|
316
|
+
self.s._extractive_qa = mock_qa
|
|
317
|
+
|
|
318
|
+
results = self.s.search("python programming", "en", top_k=2)
|
|
319
|
+
self.assertTrue(mock_qa.get_best_passage.called)
|
|
320
|
+
for r in results:
|
|
321
|
+
self.assertEqual(r.best_passage, "Python is a language.")
|
|
322
|
+
|
|
323
|
+
@patch("ovos_wikipedia.requests.get")
|
|
324
|
+
def test_keyword_fallback_when_empty_results(self, mock_get):
|
|
325
|
+
call_count = [0]
|
|
326
|
+
|
|
327
|
+
def side_effect(url, **kwargs):
|
|
328
|
+
resp = MagicMock()
|
|
329
|
+
if "list=search" in url:
|
|
330
|
+
if call_count[0] == 0:
|
|
331
|
+
resp.json.return_value = {"query": {"search": []}}
|
|
332
|
+
else:
|
|
333
|
+
resp.json.return_value = _SEARCH_RESULTS
|
|
334
|
+
call_count[0] += 1
|
|
335
|
+
else:
|
|
336
|
+
resp.json.return_value = _PAGE_DATA_1001
|
|
337
|
+
return resp
|
|
338
|
+
|
|
339
|
+
mock_get.side_effect = side_effect
|
|
340
|
+
|
|
341
|
+
mock_kw = MagicMock()
|
|
342
|
+
mock_kw.extract.return_value = ["python programming language"]
|
|
343
|
+
self.s._kword_extractor = mock_kw
|
|
344
|
+
|
|
345
|
+
results = self.s.search("what is python used for programming", "en")
|
|
346
|
+
self.assertGreater(len(results), 0)
|
|
347
|
+
mock_kw.extract.assert_called_once()
|
|
348
|
+
|
|
349
|
+
@patch("ovos_wikipedia.requests.get")
|
|
350
|
+
def test_lang_code_stripped_to_two_chars(self, mock_get):
|
|
351
|
+
mock_get.return_value.json.return_value = {"query": {"search": []}}
|
|
352
|
+
self.s.search("python", "en-US")
|
|
353
|
+
search_url = mock_get.call_args_list[0][0][0]
|
|
354
|
+
self.assertIn("en.wikipedia.org", search_url)
|
|
355
|
+
self.assertNotIn("en-US", search_url)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
# ---------------------------------------------------------------------------
|
|
359
|
+
# query() method
|
|
360
|
+
# ---------------------------------------------------------------------------
|
|
361
|
+
|
|
362
|
+
class TestQuery(unittest.TestCase):
|
|
363
|
+
|
|
364
|
+
def setUp(self):
|
|
365
|
+
self.s = _make_engine()
|
|
366
|
+
|
|
367
|
+
def test_returns_list_of_tuples(self):
|
|
368
|
+
fake_pages = [
|
|
369
|
+
_fake_result(summary="Summary one.", conf=0.9),
|
|
370
|
+
_fake_result(summary="Summary two.", conf=0.7),
|
|
371
|
+
]
|
|
372
|
+
with patch.object(self.s, "search", return_value=fake_pages):
|
|
373
|
+
results = self.s.query("test query", "en", k=2)
|
|
374
|
+
self.assertEqual(len(results), 2)
|
|
375
|
+
for content, score in results:
|
|
376
|
+
self.assertIsInstance(content, str)
|
|
377
|
+
self.assertIsInstance(score, float)
|
|
378
|
+
|
|
379
|
+
def test_best_passage_preferred_over_summary(self):
|
|
380
|
+
page = _fake_result(summary="Full summary text.")
|
|
381
|
+
page.best_passage = "Extracted passage."
|
|
382
|
+
with patch.object(self.s, "search", return_value=[page]):
|
|
383
|
+
results = self.s.query("q", "en")
|
|
384
|
+
self.assertEqual(results[0][0], "Extracted passage.")
|
|
385
|
+
|
|
386
|
+
def test_summary_used_when_no_best_passage(self):
|
|
387
|
+
page = _fake_result(summary="Full summary text.")
|
|
388
|
+
page.best_passage = None
|
|
389
|
+
with patch.object(self.s, "search", return_value=[page]):
|
|
390
|
+
results = self.s.query("q", "en")
|
|
391
|
+
self.assertEqual(results[0][0], "Full summary text.")
|
|
392
|
+
|
|
393
|
+
def test_empty_search_returns_empty_list(self):
|
|
394
|
+
with patch.object(self.s, "search", return_value=[]):
|
|
395
|
+
results = self.s.query("q", "en")
|
|
396
|
+
self.assertEqual(results, [])
|
|
397
|
+
|
|
398
|
+
def test_k_forwarded_to_search(self):
|
|
399
|
+
with patch.object(self.s, "search", return_value=[]) as mock_search:
|
|
400
|
+
self.s.query("q", "en", k=5)
|
|
401
|
+
mock_search.assert_called_once_with("q", "en", 5)
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
# ---------------------------------------------------------------------------
|
|
405
|
+
# WikipediaToolbox
|
|
406
|
+
# ---------------------------------------------------------------------------
|
|
407
|
+
|
|
408
|
+
class TestWikipediaToolbox(unittest.TestCase):
|
|
409
|
+
|
|
410
|
+
def test_discover_tools_returns_search_tool(self):
|
|
411
|
+
tb = WikipediaToolbox()
|
|
412
|
+
tools = tb.discover_tools()
|
|
413
|
+
self.assertEqual(len(tools), 1)
|
|
414
|
+
self.assertEqual(tools[0].name, "search_wikipedia")
|
|
415
|
+
|
|
416
|
+
def test_search_wikipedia_returns_title_summary_pairs(self):
|
|
417
|
+
tb = WikipediaToolbox()
|
|
418
|
+
fake_pages = [
|
|
419
|
+
_fake_result(title="Python", summary="A programming language."),
|
|
420
|
+
]
|
|
421
|
+
with patch.object(tb.wiki, "search", return_value=fake_pages):
|
|
422
|
+
from ovos_wikipedia import SearchWikipediaArgs
|
|
423
|
+
output = tb.search_wikipedia(SearchWikipediaArgs(query="python", lang="en"))
|
|
424
|
+
self.assertEqual(output.results, [("Python", "A programming language.")])
|
|
425
|
+
|
|
426
|
+
def test_search_wikipedia_empty_results(self):
|
|
427
|
+
tb = WikipediaToolbox()
|
|
428
|
+
with patch.object(tb.wiki, "search", return_value=[]):
|
|
429
|
+
from ovos_wikipedia import SearchWikipediaArgs
|
|
430
|
+
output = tb.search_wikipedia(SearchWikipediaArgs(query="xyzzy", lang="en"))
|
|
431
|
+
self.assertEqual(output.results, [])
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
if __name__ == "__main__":
|
|
435
|
+
unittest.main()
|