KorAPClient 1.2.1__tar.gz → 1.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- korapclient-1.5.1/KorAPClient/__init__.py +841 -0
- {korapclient-1.2.1 → korapclient-1.5.1}/KorAPClient/tests/test_korapclient.py +141 -2
- {korapclient-1.2.1 → korapclient-1.5.1}/KorAPClient.egg-info/PKG-INFO +108 -11
- {korapclient-1.2.1 → korapclient-1.5.1}/KorAPClient.egg-info/SOURCES.txt +1 -12
- {korapclient-1.2.1 → korapclient-1.5.1}/KorAPClient.egg-info/top_level.txt +0 -4
- {korapclient-1.2.1 → korapclient-1.5.1}/PKG-INFO +108 -11
- {korapclient-1.2.1 → korapclient-1.5.1}/Readme.md +104 -6
- {korapclient-1.2.1 → korapclient-1.5.1}/pyproject.toml +4 -5
- korapclient-1.2.1/KorAPClient/__init__.py +0 -442
- korapclient-1.2.1/README.md +0 -1
- korapclient-1.2.1/examples/fetchAll.py +0 -18
- korapclient-1.2.1/examples/github-example.py +0 -20
- korapclient-1.2.1/examples/ids_theme.py +0 -106
- korapclient-1.2.1/examples/jason.py +0 -14
- korapclient-1.2.1/examples/make_sense.py +0 -18
- korapclient-1.2.1/examples/pandas-bokeh.py +0 -42
- korapclient-1.2.1/examples/stardust.py +0 -40
- korapclient-1.2.1/examples/test.py +0 -17
- korapclient-1.2.1/src/pythonkorapclient/__init__.py +0 -0
- korapclient-1.2.1/tests/__init__.py +0 -0
- {korapclient-1.2.1 → korapclient-1.5.1}/KorAPClient/__main__.py +0 -0
- {korapclient-1.2.1 → korapclient-1.5.1}/KorAPClient/tests/__init__.py +0 -0
- {korapclient-1.2.1 → korapclient-1.5.1}/KorAPClient.egg-info/dependency_links.txt +0 -0
- {korapclient-1.2.1 → korapclient-1.5.1}/KorAPClient.egg-info/requires.txt +0 -0
- {korapclient-1.2.1 → korapclient-1.5.1}/LICENSE.txt +0 -0
- {korapclient-1.2.1 → korapclient-1.5.1}/MANIFEST.in +0 -0
- {korapclient-1.2.1 → korapclient-1.5.1}/examples/collocates.py +0 -0
- {korapclient-1.2.1 → korapclient-1.5.1}/examples/hello_world.py +0 -0
- {korapclient-1.2.1 → korapclient-1.5.1}/examples/hello_world_interactive.py +0 -0
- {korapclient-1.2.1 → korapclient-1.5.1}/examples/variant_by_country_interactive_barchart.py +0 -0
- {korapclient-1.2.1 → korapclient-1.5.1}/examples/writtenVSspoken.py +0 -0
- {korapclient-1.2.1 → korapclient-1.5.1}/setup.cfg +0 -0
|
@@ -0,0 +1,841 @@
|
|
|
1
|
+
__pdoc__ = {'tests': False}
|
|
2
|
+
|
|
3
|
+
import html
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
import tempfile
|
|
7
|
+
import warnings
|
|
8
|
+
import webbrowser
|
|
9
|
+
from itertools import product
|
|
10
|
+
|
|
11
|
+
import pandas as pd
|
|
12
|
+
from rpy2.rinterface_lib.sexp import StrSexpVector, NULLType
|
|
13
|
+
from rpy2.robjects import numpy2ri
|
|
14
|
+
from rpy2.robjects.conversion import localconverter, get_conversion
|
|
15
|
+
from rpy2.rinterface import NULL
|
|
16
|
+
|
|
17
|
+
import rpy2.robjects as robjects
|
|
18
|
+
import rpy2.robjects.packages as packages
|
|
19
|
+
import rpy2.robjects.pandas2ri as pandas2ri
|
|
20
|
+
from rpy2 import rinterface as ri
|
|
21
|
+
from packaging import version
|
|
22
|
+
from rpy2.robjects.methods import RS4
|
|
23
|
+
|
|
24
|
+
CURRENT_R_PACKAGE_VERSION = "1.5.1"
|
|
25
|
+
|
|
26
|
+
KorAPClient = packages.importr('RKorAPClient')
|
|
27
|
+
if version.parse(KorAPClient.__version__) < version.parse(CURRENT_R_PACKAGE_VERSION):
|
|
28
|
+
warnings.warn("R-package RKorAPClient version " + KorAPClient.__version__ + " is outdated, please update.",
|
|
29
|
+
DeprecationWarning)
|
|
30
|
+
|
|
31
|
+
korapclient_converter = robjects.conversion.Converter('base empty converter')
|
|
32
|
+
|
|
33
|
+
# Export NULL
|
|
34
|
+
NULL = NULL
|
|
35
|
+
|
|
36
|
+
@korapclient_converter.py2rpy.register(list)
|
|
37
|
+
def _rpy2py_robject(listObject):
|
|
38
|
+
return robjects.StrVector(listObject)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@korapclient_converter.py2rpy.register(dict)
|
|
42
|
+
def _dict_to_named_vector(dictObject):
|
|
43
|
+
"""Convert dicts to named R vectors, e.g. {"Germany": "pubPlaceKey=DE"} for labelled virtual corpora."""
|
|
44
|
+
if all(isinstance(value, str) for value in dictObject.values()):
|
|
45
|
+
vector = robjects.StrVector(list(dictObject.values()))
|
|
46
|
+
vector.names = robjects.StrVector([str(key) for key in dictObject.keys()])
|
|
47
|
+
return vector
|
|
48
|
+
return robjects.ListVector(dictObject)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
robjects.conversion.set_conversion(robjects.default_converter + pandas2ri.converter + korapclient_converter)
|
|
52
|
+
|
|
53
|
+
fix_null_types = robjects.default_converter
|
|
54
|
+
|
|
55
|
+
@fix_null_types.rpy2py.register(NULLType)
|
|
56
|
+
def to_str(obj):
|
|
57
|
+
return ""
|
|
58
|
+
|
|
59
|
+
fix_lists_in_dataframes = robjects.default_converter
|
|
60
|
+
|
|
61
|
+
@fix_lists_in_dataframes.rpy2py.register(StrSexpVector)
|
|
62
|
+
def to_str(obj):
|
|
63
|
+
for i in range(len(obj)):
|
|
64
|
+
obj[i] = str(obj[i])
|
|
65
|
+
return "\t".join(obj)
|
|
66
|
+
|
|
67
|
+
def my_cv(obj, cv):
|
|
68
|
+
if isinstance(obj, ri.StrSexpVector):
|
|
69
|
+
for i in range(len(obj)):
|
|
70
|
+
obj[i] = str(obj[i])
|
|
71
|
+
return StrSexpVector((obj))
|
|
72
|
+
else:
|
|
73
|
+
return cv.rpy2py(obj)
|
|
74
|
+
|
|
75
|
+
def toDataFrame(obj):
|
|
76
|
+
cv = get_conversion() # get the converter from current context
|
|
77
|
+
names = []
|
|
78
|
+
objects = []
|
|
79
|
+
for i in range(len(obj)):
|
|
80
|
+
if isinstance(obj[i], ri.ListSexpVector):
|
|
81
|
+
list_name = obj.names[i] + "." if not isinstance(obj.names, NULLType) else "l" + str(i) + "."
|
|
82
|
+
for j in range(len(obj[i])):
|
|
83
|
+
local_name = str(obj[i].names[j]) if not isinstance(obj[i].names, NULLType) else str(j)
|
|
84
|
+
names.append(list_name + local_name)
|
|
85
|
+
objects.append(obj[i][j])
|
|
86
|
+
else:
|
|
87
|
+
names.append(obj.names[i])
|
|
88
|
+
objects.append(obj[i])
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
return pd.DataFrame(
|
|
92
|
+
{str(k): my_cv(objects[i], cv) for i, k in enumerate(names)}
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
# associate the converter with R data.frame class
|
|
96
|
+
fix_lists_in_dataframes.rpy2py_nc_map[ri.ListSexpVector].update({"data.frame": toDataFrame})
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _call_keeping_r_result(function, *args, **kwargs):
|
|
101
|
+
"""Call an R function with the usual argument conversion, but return its result as unconverted R object."""
|
|
102
|
+
cv = get_conversion()
|
|
103
|
+
return ri.SexpClosure.__call__(function, *[cv.py2rpy(arg) for arg in args],
|
|
104
|
+
**{key: cv.py2rpy(value) for key, value in kwargs.items()})
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
_flat_columns = robjects.r("""function(x)
|
|
108
|
+
as.data.frame(unclass(x)[!vapply(x, is.list, logical(1))], stringsAsFactors = FALSE, check.names = FALSE)""")
|
|
109
|
+
|
|
110
|
+
_join_list_columns = robjects.r("""function(kqo) {
|
|
111
|
+
join <- function(x) if (length(x) == 0 || all(is.na(x))) NA_character_ else paste(x, collapse = " ")
|
|
112
|
+
df <- kqo@collectedMatches
|
|
113
|
+
df[] <- lapply(df, function(column) if (is.list(column)) vapply(column, join, character(1)) else column)
|
|
114
|
+
kqo@collectedMatches <- df
|
|
115
|
+
kqo
|
|
116
|
+
}""")
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _without_na_strings(df):
|
|
120
|
+
"""Replace R's NA_character_ by None in the string columns of a DataFrame."""
|
|
121
|
+
for name in df.columns[df.dtypes == object]:
|
|
122
|
+
df[name] = df[name].map(lambda value: None if value is ri.NA_Character else value)
|
|
123
|
+
return df
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _r_list_to_dict(obj):
|
|
127
|
+
result = {}
|
|
128
|
+
for name, value in zip(obj.names, obj):
|
|
129
|
+
if isinstance(value, NULLType):
|
|
130
|
+
result[name] = None
|
|
131
|
+
elif isinstance(value, ri.ListSexpVector):
|
|
132
|
+
result[name] = _r_list_to_dict(value)
|
|
133
|
+
elif len(value) == 1:
|
|
134
|
+
result[name] = None if value[0] is ri.NA_Character else value[0]
|
|
135
|
+
else:
|
|
136
|
+
result[name] = list(value)
|
|
137
|
+
return result
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _r_value_to_py(value):
|
|
141
|
+
"""Convert an R scalar or vector to a Python value, with NA as None."""
|
|
142
|
+
if isinstance(value, NULLType):
|
|
143
|
+
return None
|
|
144
|
+
values = [None if item in (ri.NA_Character, ri.NA_Integer, ri.NA_Logical) or
|
|
145
|
+
(isinstance(item, float) and item != item) else item for item in value]
|
|
146
|
+
return values[0] if len(values) == 1 else values
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _dispersion_to_pandas(res):
|
|
150
|
+
"""Convert a dispersion result with nested tables to a DataFrame with DataFrames and dicts in its list columns."""
|
|
151
|
+
cv = get_conversion()
|
|
152
|
+
df = _without_na_strings(_flat_columns(res))
|
|
153
|
+
for position, (name, column) in enumerate(zip(res.names, res)):
|
|
154
|
+
if isinstance(column, ri.ListSexpVector):
|
|
155
|
+
values = [_without_na_strings(cv.rpy2py(value)) if "data.frame" in value.rclass
|
|
156
|
+
else _r_list_to_dict(value) for value in column]
|
|
157
|
+
df.insert(position, name, pd.Series(values, index=df.index, dtype=object))
|
|
158
|
+
return df
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
_hc_dispersion = robjects.r("""function(rows, distributions, apiUrls, indexRevisions, probabilities, partition, top) {
|
|
162
|
+
rows <- tibble::as_tibble(rows)
|
|
163
|
+
rows$distribution <- lapply(distributions, function(part) {
|
|
164
|
+
part <- tibble::as_tibble(part)
|
|
165
|
+
for (field in intersect(c("part", "vc", "webUIRequestUrl"), names(part)))
|
|
166
|
+
part[[field]] <- as.character(part[[field]])
|
|
167
|
+
part
|
|
168
|
+
})
|
|
169
|
+
rows$sampling <- lapply(seq_along(probabilities), function(i)
|
|
170
|
+
list(apiUrl = apiUrls[[i]], indexRevision = indexRevisions[[i]], inclusionProbability = probabilities[[i]]))
|
|
171
|
+
RKorAPClient::hc_dispersion(rows, partition = partition, top = top)
|
|
172
|
+
}""")
|
|
173
|
+
|
|
174
|
+
_save_widget = robjects.r("""function(widget, file, title)
|
|
175
|
+
htmlwidgets::saveWidget(widget, file, selfcontained = FALSE, libdir = "lib", title = title)""")
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _inline_resources(document, directory):
|
|
179
|
+
"""Embed the scripts and stylesheets that an HTML document loads from directory into the document."""
|
|
180
|
+
def read(name):
|
|
181
|
+
with open(os.path.join(directory, name), encoding="utf-8") as file:
|
|
182
|
+
return file.read()
|
|
183
|
+
|
|
184
|
+
document = re.sub(r'<script src="(lib/[^"]+)"></script>',
|
|
185
|
+
lambda match: "<script>" + read(match.group(1)).replace("</script", "<\\/script") + "</script>",
|
|
186
|
+
document)
|
|
187
|
+
return re.sub(r'<link href="(lib/[^"]+)" rel="stylesheet" />',
|
|
188
|
+
lambda match: "<style>" + read(match.group(1)) + "</style>", document)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
class Highchart:
|
|
192
|
+
"""Interactive [Highcharts](https://www.highcharts.com/) chart created by an R plotting function.
|
|
193
|
+
|
|
194
|
+
In Jupyter notebooks, the chart is displayed when it is the last expression of a cell.
|
|
195
|
+
Elsewhere use `Highchart.show` or `Highchart.save`.
|
|
196
|
+
"""
|
|
197
|
+
|
|
198
|
+
def __init__(self, widget, title="KorAP chart", height=600):
|
|
199
|
+
self.widget = widget
|
|
200
|
+
self.title = title
|
|
201
|
+
self.height = height
|
|
202
|
+
|
|
203
|
+
def to_html(self):
|
|
204
|
+
"""Return the chart as self-contained HTML document."""
|
|
205
|
+
with tempfile.TemporaryDirectory() as directory:
|
|
206
|
+
path = os.path.join(directory, "chart.html")
|
|
207
|
+
_save_widget(self.widget, path, self.title)
|
|
208
|
+
with open(path, encoding="utf-8") as file:
|
|
209
|
+
return _inline_resources(file.read(), directory)
|
|
210
|
+
|
|
211
|
+
def save(self, path):
|
|
212
|
+
"""Save the chart as self-contained HTML file.
|
|
213
|
+
|
|
214
|
+
- **path** - name of the HTML file
|
|
215
|
+
|
|
216
|
+
Returns:
|
|
217
|
+
absolute path of the HTML file
|
|
218
|
+
"""
|
|
219
|
+
path = os.path.abspath(os.path.expanduser(path))
|
|
220
|
+
with open(path, "w", encoding="utf-8") as file:
|
|
221
|
+
file.write(self.to_html())
|
|
222
|
+
return path
|
|
223
|
+
|
|
224
|
+
def show(self):
|
|
225
|
+
"""Open the chart in the default web browser."""
|
|
226
|
+
file = tempfile.NamedTemporaryFile(suffix=".html", delete=False)
|
|
227
|
+
file.close()
|
|
228
|
+
webbrowser.open("file://" + self.save(file.name))
|
|
229
|
+
|
|
230
|
+
def _repr_html_(self):
|
|
231
|
+
return '<iframe srcdoc="{}" style="width: 100%; height: {}px; border: none"></iframe>'.format(
|
|
232
|
+
html.escape(self.to_html(), quote=True), self.height)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def hc_dispersion(df, partition=None, top=None):
|
|
236
|
+
""" **EXPERIMENTAL**: Plot the distribution of dispersion analysis results across corpus parts.
|
|
237
|
+
|
|
238
|
+
Creates an interactive bar chart that compares, for each corpus part, its share of the corpus tokens
|
|
239
|
+
with its share of the hits of each query.
|
|
240
|
+
|
|
241
|
+
- **df** - DataFrame returned by `KorAPConnection.dispersionQuery` or `KorAPQuery.dispersionAnalysis`. Results of several calls can be combined with `pandas.concat`, as long as they refer to the same corpus parts.
|
|
242
|
+
- **partition** - name of the partition to plot, e.g. `"source"`. Required if `df` contains more than one partition.
|
|
243
|
+
- **top** - number of corpus parts to show individually; the remaining parts are combined (default = 12, or all parts for calendar partitions like `"year"`). Use `float("inf")` to show all parts.
|
|
244
|
+
|
|
245
|
+
Returns:
|
|
246
|
+
`Highchart`
|
|
247
|
+
|
|
248
|
+
Details:
|
|
249
|
+
The series are labelled with the `variant` column of `df` if there is one, otherwise with its `query` column.
|
|
250
|
+
|
|
251
|
+
Example:
|
|
252
|
+
```
|
|
253
|
+
$ kcon = KorAPConnection()
|
|
254
|
+
$ df = kcon.dispersionQuery(["Ameisenplage", "Mückenplage"], by="region", vc="pubDate in 2022")
|
|
255
|
+
$ hc_dispersion(df).save("dispersion.html")
|
|
256
|
+
```
|
|
257
|
+
"""
|
|
258
|
+
required = ["partition", "query", "distribution", "estimated", "sampling"]
|
|
259
|
+
if not isinstance(df, pd.DataFrame) or len(df) == 0 or not all(name in df.columns for name in required):
|
|
260
|
+
raise ValueError("Use a result of dispersionQuery() or dispersionAnalysis() with distribution data.")
|
|
261
|
+
cv = get_conversion()
|
|
262
|
+
|
|
263
|
+
def r_list(values):
|
|
264
|
+
return ri.ListSexpVector([cv.py2rpy(value) for value in values])
|
|
265
|
+
|
|
266
|
+
def sampling_field(name, na):
|
|
267
|
+
values = [sampling.get(name) if isinstance(sampling, dict) else None for sampling in df["sampling"]]
|
|
268
|
+
return r_list([na if value is None else value for value in values])
|
|
269
|
+
|
|
270
|
+
rows = df.drop(columns=["distribution", "sampling"]).reset_index(drop=True)
|
|
271
|
+
widget = ri.SexpClosure.__call__(
|
|
272
|
+
_hc_dispersion, cv.py2rpy(rows), r_list(df["distribution"]),
|
|
273
|
+
sampling_field("apiUrl", ri.NA_Character), sampling_field("indexRevision", ri.NA_Character),
|
|
274
|
+
sampling_field("inclusionProbability", ri.NA_Real),
|
|
275
|
+
NULL if partition is None else cv.py2rpy(partition), NULL if top is None else cv.py2rpy(top))
|
|
276
|
+
return Highchart(widget, title="Dispersion" if partition is None else "Dispersion by " + partition)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
class KorAPCorpusPartition(robjects.vectors.DataFrame):
|
|
280
|
+
"""Corpus parts and their token counts, as returned by `KorAPConnection.corpusPartition`.
|
|
281
|
+
|
|
282
|
+
Pass it as `partitions` argument to `KorAPConnection.dispersionQuery`. Use `toDataFrame` to inspect it.
|
|
283
|
+
"""
|
|
284
|
+
|
|
285
|
+
def toDataFrame(self):
|
|
286
|
+
"""Returns the parts as DataFrame with columns `'part', 'value', 'vc', 'tokens'`."""
|
|
287
|
+
return _without_na_strings(_flat_columns(self))
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def expand_grid(dictionary):
|
|
291
|
+
"""Create a pandas DataFrame from all combinations of inputs
|
|
292
|
+
|
|
293
|
+
- **dictionary** - dict with variable names as keys and their values as vectors
|
|
294
|
+
|
|
295
|
+
Returns:
|
|
296
|
+
DataFrame with column names as specified by the dictionary key and all combinations of the specified values
|
|
297
|
+
in the rows.
|
|
298
|
+
|
|
299
|
+
Example:
|
|
300
|
+
```
|
|
301
|
+
$ df = expand_grid({"Year": range(2010, 2019), "Country": ["DE", "CH"] })
|
|
302
|
+
|
|
303
|
+
$ df["vc"] = "textType=/Zeit.*/ & pubPlaceKey = " + df.Country + " & pubDate in " + list(map(str, df.Year))
|
|
304
|
+
```
|
|
305
|
+
"""
|
|
306
|
+
|
|
307
|
+
return pd.DataFrame([row for row in product(*dictionary.values())],
|
|
308
|
+
columns=dictionary.keys())
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
# noinspection PyPep8Naming
|
|
312
|
+
class KorAPConnection(RS4):
|
|
313
|
+
"""Connection to a KorAP server."""
|
|
314
|
+
|
|
315
|
+
def __init__(self, *args, **kwargs):
|
|
316
|
+
"""Constructor keyword arguments:
|
|
317
|
+
|
|
318
|
+
- **KorAPUrl** (default = `"https://korap.ids-mannheim.de/"`)
|
|
319
|
+
- **apiVersion** (default = 'v1.0')
|
|
320
|
+
- **apiUrl**
|
|
321
|
+
- **accessToken** (default = `getAccessToken(KorAPUrl)`
|
|
322
|
+
- **userAgent** (default = `"Python-KorAP-Client"`)
|
|
323
|
+
- **timeout** - maximum time in seconds a request may take (default = 240). Long running searches are no longer
|
|
324
|
+
aborted after 10 minutes of low transfer rates; only this timeout applies.
|
|
325
|
+
- **verbose** (default = False)
|
|
326
|
+
- **cache** (default = True)
|
|
327
|
+
"""
|
|
328
|
+
if 'userAgent' not in kwargs:
|
|
329
|
+
kwargs["userAgent"] = "Python-KorAP-Client"
|
|
330
|
+
kco = KorAPClient.KorAPConnection(*args, **kwargs)
|
|
331
|
+
super().__init__(kco)
|
|
332
|
+
|
|
333
|
+
def auth(self, *args, **kwargs):
|
|
334
|
+
""" Authorize PythonKorAPClient to make KorAP queries and download results on behalf of the user.
|
|
335
|
+
|
|
336
|
+
- **kco** - `KorAPConnection` object
|
|
337
|
+
- **app_id** - OAuth2 application id. Defaults to the generic KorAP client application id.
|
|
338
|
+
- **app_secret** - OAuth2 application secret. Used with confidential client applications. Defaults to `NULL`.
|
|
339
|
+
- **scope** - OAuth2 scope. Defaults to "search match_info".
|
|
340
|
+
|
|
341
|
+
Returns:
|
|
342
|
+
|
|
343
|
+
Potentially authorized `KorAPConnection`|`RS4` with access token in `.slots['accessToken']`.
|
|
344
|
+
|
|
345
|
+
Example:
|
|
346
|
+
```
|
|
347
|
+
from KorAPClient import KorAPConnection, NULL
|
|
348
|
+
|
|
349
|
+
### Create a KorAPConnection object without an existing access token
|
|
350
|
+
|
|
351
|
+
kcon = KorAPConnection(accessToken=NULL, verbose=True).auth()
|
|
352
|
+
|
|
353
|
+
### Perform a query using the authenticated connection
|
|
354
|
+
|
|
355
|
+
q = kcon.corpusQuery("Ameisenplage", metadataOnly=False)
|
|
356
|
+
|
|
357
|
+
### Fetch all results
|
|
358
|
+
|
|
359
|
+
q = q.fetchAll()
|
|
360
|
+
|
|
361
|
+
### Access the collected matches
|
|
362
|
+
|
|
363
|
+
print(q.slots['collectedMatches'].snippet)
|
|
364
|
+
```
|
|
365
|
+
|
|
366
|
+
"""
|
|
367
|
+
|
|
368
|
+
kco = KorAPClient.auth(self, *args, **kwargs)
|
|
369
|
+
super().__init__(kco)
|
|
370
|
+
return self
|
|
371
|
+
|
|
372
|
+
def corpusStats(self, *args, **kwargs):
|
|
373
|
+
"""Query the size of the whole corpus or a virtual corpus specified by the vc argument.
|
|
374
|
+
|
|
375
|
+
- **vc** - virtual corpus definition, list thereof, or dict mapping labels to definitions (default = "").
|
|
376
|
+
Dict keys are used as labels in a `label` column (see `KorAPConnection.frequencyQuery`).
|
|
377
|
+
- **verbose** (default = kco@verbose)
|
|
378
|
+
- **as.df** (default = True)
|
|
379
|
+
- **cacheAs** - path to an RDS file to keep the result in (see `KorAPConnection.collocationAnalysis`)
|
|
380
|
+
|
|
381
|
+
Returns:
|
|
382
|
+
`DataFrame`|`RS4`
|
|
383
|
+
|
|
384
|
+
Example:
|
|
385
|
+
```
|
|
386
|
+
$ df = kcon.corpusStats("pubDate in 2018 & textType=/Zeit.*/ & pubPlaceKey=IT", **{"as.df": True})
|
|
387
|
+
$ df["tokens"]
|
|
388
|
+
12150897
|
|
389
|
+
```
|
|
390
|
+
"""
|
|
391
|
+
default_kwargs = {"as.df": True}
|
|
392
|
+
default_kwargs.update(kwargs)
|
|
393
|
+
return KorAPClient.corpusStats(self, *args, **default_kwargs)
|
|
394
|
+
|
|
395
|
+
def frequencyQuery(self, *args, **kwargs):
|
|
396
|
+
"""Query relative frequency of search term(s).
|
|
397
|
+
|
|
398
|
+
- **query** - query string or list of query strings
|
|
399
|
+
- **vc** - virtual corpus definition, list thereof, or dict mapping labels to definitions (default: "")
|
|
400
|
+
- **conf.level** - confidence level of the returned confidence interval (default = 0.95)
|
|
401
|
+
- **as.alternatives** - decides whether queries should be treated as mutually exclusive and exhaustive wrt. to some meaningful class (e.g. spelling variants of a certain word form) (default = False)
|
|
402
|
+
- **KorAPUrl** - instead of specifying the `query` and `vc` string parameters, you can copy your KorAP query URL here from the browser
|
|
403
|
+
- **metadataOnly** - determines whether queries should return only metadata without any snippets. This can also be useful to prevent access rewrites. (default = True)
|
|
404
|
+
- **ql** - query language: `"poliqarp" | "cosmas2" | "annis" | "cql" | "fcsql"` (default = `"poliqarp"`)
|
|
405
|
+
- **accessRewriteFatal** - abort if query or given vc had to be rewritten due to insufficient rights (not yet implemented) (default = `True`)
|
|
406
|
+
- **verbose** - (default = `self.verbose`)
|
|
407
|
+
- **expand** - bool that decides if `query` and `vc` parameters are expanded to all of their combinations (default = `len(vc) != len(query)`)
|
|
408
|
+
- **cacheAs** - path to an RDS file to keep the result in (see `KorAPConnection.collocationAnalysis`)
|
|
409
|
+
|
|
410
|
+
Returns:
|
|
411
|
+
DataFrame with columns `'query', 'totalResults', 'vc', 'webUIRequestUrl', 'queryDuration', 'total', 'f',
|
|
412
|
+
'conf.low', 'conf.high'`. `queryDuration` is the time in seconds a request took on the client side.
|
|
413
|
+
If `vc` is a dict, its keys are used as labels in an additional `label` column.
|
|
414
|
+
A warning is issued if a request failed or the server cut a search short (`timeExceeded`).
|
|
415
|
+
|
|
416
|
+
Example:
|
|
417
|
+
```
|
|
418
|
+
$ kcon = KorAPConnection(verbose=True)
|
|
419
|
+
$ kcon.frequencyQuery("Ameisenplage", vc=["pubDate in "+str(y) for y in range(2010,2015)])
|
|
420
|
+
query totalResults ... conf.low conf.high
|
|
421
|
+
1 Ameisenplage 3 ... 9.727696e-10 1.200289e-08
|
|
422
|
+
2 Ameisenplage 12 ... 3.838218e-09 1.275717e-08
|
|
423
|
+
3 Ameisenplage 5 ... 2.013352e-09 1.356500e-08
|
|
424
|
+
4 Ameisenplage 6 ... 2.691331e-09 1.519888e-08
|
|
425
|
+
5 Ameisenplage 3 ... 8.629463e-10 1.064780e-08
|
|
426
|
+
```
|
|
427
|
+
|
|
428
|
+
To label virtual corpora, pass a dict:
|
|
429
|
+
```
|
|
430
|
+
$ vc = {"Germany": "pubPlaceKey=DE", "Switzerland": "pubPlaceKey=CH"}
|
|
431
|
+
$ kcon.frequencyQuery("Ameisenplage", vc=vc)[["query", "label", "totalResults"]]
|
|
432
|
+
query label totalResults
|
|
433
|
+
1 Ameisenplage Germany 178
|
|
434
|
+
2 Ameisenplage Switzerland 27
|
|
435
|
+
```
|
|
436
|
+
"""
|
|
437
|
+
return KorAPClient.frequencyQuery(self, *args, **kwargs)
|
|
438
|
+
|
|
439
|
+
def collocationScoreQuery(self, node, collocate, vc="", **kwargs):
|
|
440
|
+
"""Get collocation scores for given node(s) and collocate(s).
|
|
441
|
+
|
|
442
|
+
- **node** - target word
|
|
443
|
+
- **collocate** - collocate of target word or list thereof (all combinations of collocates and vcs are queried)
|
|
444
|
+
- **vc** - virtual corpus definition, list thereof, or dict mapping labels to definitions (default: ""). Dict keys are used as labels.
|
|
445
|
+
- **lemmatizeNodeQuery** - logical, set to TRUE if node query should be lemmatized, i.e. x -> [tt/l=x]
|
|
446
|
+
- **lemmatizeCollocateQuery** - logical, set to TRUE if collocate query should be lemmatized, i.e. x -> [tt/l=x]
|
|
447
|
+
- **leftContextSize** - size of the left context window
|
|
448
|
+
- **rightContextSize** - size of the right context window
|
|
449
|
+
- **scoreFunctions** - dict of R (!) score functions of the form function(O1, O2, O, N, E, window_size), see e.g. KorAPClient.pmi
|
|
450
|
+
- **smoothingConstant** - smoothing constant will be added to all observed values
|
|
451
|
+
- **cacheAs** - path to an RDS file to keep the result in (see `KorAPConnection.collocationAnalysis`)
|
|
452
|
+
|
|
453
|
+
Returns:
|
|
454
|
+
DataFrame with columns `'node', 'collocate', 'label', 'vc','webUIRequestUrl', 'w', 'leftContextSize',
|
|
455
|
+
'rightContextSize', 'N', 'O', 'O1', 'O2', 'E', 'pmi', 'mi2', 'mi3', 'logDice', 'll'`
|
|
456
|
+
|
|
457
|
+
Example:
|
|
458
|
+
```
|
|
459
|
+
$ kcon = KorAPConnection(verbose=True)
|
|
460
|
+
$ df = kcon.collocationScoreQuery("Grund", "triftiger")
|
|
461
|
+
```
|
|
462
|
+
"""
|
|
463
|
+
return KorAPClient.collocationScoreQuery(self, node, collocate, vc, **kwargs)
|
|
464
|
+
|
|
465
|
+
def collocationAnalysis(self, node, vc="", **kwargs):
|
|
466
|
+
""" **EXPERIMENTAL**: Performs a collocation analysis for the given node (or query) in the given virtual corpus.
|
|
467
|
+
|
|
468
|
+
- **node** - target word or list of target words
|
|
469
|
+
- **vc** - string, list of strings, or dict mapping labels to strings describing the virtual corpus in which the query should be performed. An empty string (default) means the whole corpus, as far as it is license-wise accessible.
|
|
470
|
+
- **lemmatizeNodeQuery** - if True, node query will be lemmatized, i.e. x -> [tt/l=x]
|
|
471
|
+
- **minOccur** - minimum absolute number of observed co-occurrences to consider a collocate candidate
|
|
472
|
+
- **leftContextSize** - size of the left context window
|
|
473
|
+
- **rightContextSize** - size of the right context window
|
|
474
|
+
- **topCollocatesLimit** - limit analysis to the n most frequent collocates in the search hits sample
|
|
475
|
+
- **searchHitsSampleLimit** - limit the size of the search hits sample
|
|
476
|
+
- **ignoreCollocateCase** - bool, set to True if collocate case should be ignored
|
|
477
|
+
- **withinSpan** - KorAP span specification (see <https://korap.ids-mannheim.de/doc/ql/poliqarp-plus?embedded=true#spans>) for collocations to be searched within. Defaults to `base/s=s`
|
|
478
|
+
- **exactFrequencies** - if False, extrapolate observed co-occurrence frequencies from frequencies in search hits sample, otherwise retrieve exact co-occurrence frequencies
|
|
479
|
+
- **stopwords** - vector of stopwords not to be considered as collocates
|
|
480
|
+
- **seed** - seed for random page collecting order
|
|
481
|
+
- **expand** - if True, node and vc parameters are expanded to all of their combinations
|
|
482
|
+
- **minObservedExpectedRatio** - minimum ratio of observed to expected co-occurrence frequency for a collocate
|
|
483
|
+
to be kept (default = 1, i.e. collocates occurring less often than expected by chance are discarded). Set to 0
|
|
484
|
+
for the unfiltered results of earlier versions.
|
|
485
|
+
- **cacheAs** - path to an RDS file to keep the result in. If the file exists and records the same call, it is
|
|
486
|
+
read back instead of contacting the server; otherwise the analysis is run and its result stored there
|
|
487
|
+
(`.rds` is appended if missing). By default, no file is used.
|
|
488
|
+
|
|
489
|
+
Returns:
|
|
490
|
+
DataFrame with columns `'node', 'collocate', 'label', 'vc','webUIRequestUrl', 'w', 'leftContextSize',
|
|
491
|
+
'rightContextSize', 'N', 'O', 'O1', 'O2', 'E', 'pmi', 'mi2', 'mi3', 'logDice', 'll'`
|
|
492
|
+
|
|
493
|
+
Details:
|
|
494
|
+
The collocation analysis is currently implemented on the client side, as some of the functionality is not yet provided by the KorAP backend. Mainly for this reason it is very slow (several minutes, up to hours), but on the other hand very flexible. You can, for example, perform the analysis in arbitrary virtual corpora, use complex node queries, and look for expression-internal collocates using the focus function (see examples and demo).
|
|
495
|
+
`logDice` is computed as defined by Rychlý (2008), `14 + log2(2 * O / (O1 + O2))`, so that values are comparable to those of other tools. `ll` follows Evert (2004).
|
|
496
|
+
When `vc` is a list or dict of virtual corpora, the results are compared across them (**experimental**, see `?collocationAnalysis` in R).
|
|
497
|
+
To increase speed at the cost of accuracy and possible false negatives, you can decrease searchHitsSampleLimit and/or topCollocatesLimit and/or set exactFrequencies to FALSE.
|
|
498
|
+
Note that currently not the tokenization provided by the backend, i.e. the corpus itself, is used, but a tinkered one. This can also lead to false negatives and to frequencies that differ from corresponding ones acquired via the web user interface.
|
|
499
|
+
|
|
500
|
+
Example:
|
|
501
|
+
```
|
|
502
|
+
$ kcon = KorAPConnection(verbose=True)
|
|
503
|
+
$ df = kcon.collocationAnalysis("Grund")
|
|
504
|
+
```
|
|
505
|
+
"""
|
|
506
|
+
return KorAPClient.collocationAnalysis(self, node, vc, **kwargs)
|
|
507
|
+
|
|
508
|
+
def mergeDuplicateCollocates(self, *args, **kwargs):
|
|
509
|
+
"""Merge collocation analysis results for different context positions."""
|
|
510
|
+
return KorAPClient.mergeDuplicateCollocates(*args, **kwargs)
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def corpusQuery(self, *args, **kwargs):
|
|
514
|
+
"""Query search term(s).
|
|
515
|
+
|
|
516
|
+
- **query** - query string or list of query strings
|
|
517
|
+
- **vc** - virtual corpus definition or list thereof (default: "")
|
|
518
|
+
- **KorAPUrl** - instead of specifying the `query` and `vc` string parameters, you can copy your KorAP query URL here from the browser
|
|
519
|
+
- **metadataOnly** - determines whether queries should return only metadata without any snippets. This can also be useful to prevent access rewrites. (default = True)
|
|
520
|
+
- **ql** - query language: `"poliqarp" | "cosmas2" | "annis" | "cql" | "fcsql"` (default = `"poliqarp"`)
|
|
521
|
+
- **fields** - (meta)data fields that will be fetched for every match (default = `["corpusSigle", "textSigle", "pubDate", "pubPlace", "availability", "textClass", "dmozDomain", "wikiDomain", "snippet", "tokens"]`).
|
|
522
|
+
Since DeReKo-KorAP-2026-II, the topic domain of DeReKo texts is given by `dmozDomain` and `wikiDomain` instead of `textClass`.
|
|
523
|
+
- **verbose** - (default = `self.verbose`)
|
|
524
|
+
|
|
525
|
+
Returns:
|
|
526
|
+
`KorAPQuery`
|
|
527
|
+
|
|
528
|
+
Example:
|
|
529
|
+
```
|
|
530
|
+
$ kcon = KorAPConnection(verbose=True)
|
|
531
|
+
$ q = kcon.corpusQuery("Ameisenplage")
|
|
532
|
+
$ q = q.fetchAll()
|
|
533
|
+
$ q.slots['collectedMatches']
|
|
534
|
+
corpusSigle ... textClass
|
|
535
|
+
1 WPD17 ... NaN
|
|
536
|
+
2 WPD17 ... NaN
|
|
537
|
+
3 WPD17 ... NaN
|
|
538
|
+
4 WPD17 ... NaN
|
|
539
|
+
5 WPD17 ... NaN
|
|
540
|
+
.. ... ... ...
|
|
541
|
+
126 Z83 ... freizeit-unterhaltung reisen
|
|
542
|
+
127 MZE03 ... freizeit-unterhaltung reisen natur-umwelt wett...
|
|
543
|
+
128 MZE03 ... freizeit-unterhaltung reisen staat-gesellschaf...
|
|
544
|
+
129 MZE14 ... wissenschaft populaerwissenschaft freizeit-unt...
|
|
545
|
+
130 MZE00 ... wissenschaft populaerwissenschaft
|
|
546
|
+
[130 rows x 6 columns]
|
|
547
|
+
```
|
|
548
|
+
"""
|
|
549
|
+
return KorAPQuery(self, *args, **kwargs)
|
|
550
|
+
|
|
551
|
+
def textMetadata(self, textSigle, **kwargs):
|
|
552
|
+
""" Retrieves metadata for a text, identified by its sigle (id) using the corresponding KorAP API
|
|
553
|
+
(see `Kustvakt Wiki https://github.com/KorAP/Kustvakt/wiki/Service:-Metadata-Retrieval`).
|
|
554
|
+
|
|
555
|
+
- **textSigle** - unique text id (concatenation of corpus, document and text ids, separated by `/`, e.g. ) or list thereof
|
|
556
|
+
- **cacheAs** - path to an RDS file to keep the result in (see `KorAPConnection.collocationAnalysis`)
|
|
557
|
+
|
|
558
|
+
Returns:
|
|
559
|
+
DataFrame with columns for each metadata property. In case of errors, such as non-existing texts/sigles, the tibble will also contain a column called `errors`.
|
|
560
|
+
If there are metadata columns you cannot make sense of, please ignore them. The function simply returns all the metadata it gets from the server.
|
|
561
|
+
|
|
562
|
+
Example:
|
|
563
|
+
```
|
|
564
|
+
$ kcon = KorAPConnection(verbose=True)
|
|
565
|
+
$ kcon.textMetadata(["WUD17/A97/08542", "WUD17/B96/57558", "WUD17/A97/08541"])
|
|
566
|
+
```
|
|
567
|
+
"""
|
|
568
|
+
return KorAPClient.textMetadata(self, textSigle, **kwargs)
|
|
569
|
+
|
|
570
|
+
def maintenanceInfo(self, refresh=False):
|
|
571
|
+
"""Inspect planned maintenance of the KorAP instance
|
|
572
|
+
|
|
573
|
+
Retrieves the public maintenance schedule from the KorAP frontend of the connected instance. No authorization is needed.
|
|
574
|
+
Before server requests, the client checks for maintenance notices at most once every five minutes and announces each
|
|
575
|
+
maintenance window. To disable these checks, run `rpy2.robjects.r("options(rkorap.maintenance.notices = FALSE)")`.
|
|
576
|
+
|
|
577
|
+
- **refresh** - if True, bypass the five-minute schedule cache (default = `False`)
|
|
578
|
+
|
|
579
|
+
Returns:
|
|
580
|
+
dict with `'state'` (`"ok"`, `"stale"`, `"unknown"` or `"unsupported"`), `'time_zone'`, the Unix timestamps
|
|
581
|
+
`'last_success'` and `'valid_until'` and an `'events'` DataFrame with the columns `'start', 'end'` (ISO 8601),
|
|
582
|
+
`'start_epoch', 'end_epoch'` (Unix timestamps), `'all_day'` and `'ongoing'`. All-day end dates are exclusive.
|
|
583
|
+
An empty schedule with state `"unknown"` does not mean that no maintenance is planned.
|
|
584
|
+
|
|
585
|
+
Example:
|
|
586
|
+
```
|
|
587
|
+
$ kcon = KorAPConnection()
|
|
588
|
+
$ kcon.maintenanceInfo()["events"]
|
|
589
|
+
```
|
|
590
|
+
"""
|
|
591
|
+
res = _call_keeping_r_result(KorAPClient.maintenanceInfo, self, refresh=refresh)
|
|
592
|
+
cv = get_conversion()
|
|
593
|
+
return {name: cv.rpy2py(value) if "data.frame" in value.rclass else _r_value_to_py(value)
|
|
594
|
+
for name, value in zip(res.names, res)}
|
|
595
|
+
|
|
596
|
+
def corpusPartition(self, by="source", values=None, vc="", **kwargs):
|
|
597
|
+
""" **EXPERIMENTAL**: Define corpus parts for a dispersion analysis.
|
|
598
|
+
|
|
599
|
+
Discovers metadata groups and their token counts, or defines a custom partition. Discovery checks the base
|
|
600
|
+
corpus independently of query hits, so that parts without hits are retained.
|
|
601
|
+
|
|
602
|
+
- **by** - metadata field. The aliases `"source"`, `"year"` and `"region"` stand for `corpusTitle`, `pubDate` (publication year) and `pubPlaceKey` respectively.
|
|
603
|
+
- **values** - list of metadata values, or dict mapping part labels to values. If neither `values` nor `parts` is given, the groups are discovered automatically.
|
|
604
|
+
- **vc** - base virtual corpus. All parts are intersected with this corpus. (default = "")
|
|
605
|
+
- **parts** - alternatively, a dict mapping part labels to virtual corpus definitions
|
|
606
|
+
- **regex** - optional trailing-suffix grouping rule for automatic discovery (see `KorAPQuery.dispersionAnalysis`)
|
|
607
|
+
- **verbose** - (default = `self.verbose`)
|
|
608
|
+
- **cacheAs** - path to an RDS file to keep the result in (see `KorAPConnection.collocationAnalysis`)
|
|
609
|
+
|
|
610
|
+
Returns:
|
|
611
|
+
`KorAPCorpusPartition` that can be passed to `KorAPConnection.dispersionQuery`.
|
|
612
|
+
Use its `toDataFrame()` method to get the parts with their virtual corpus definitions and token counts.
|
|
613
|
+
A warning is issued if the listed parts do not cover the whole base corpus.
|
|
614
|
+
|
|
615
|
+
Example:
|
|
616
|
+
```
|
|
617
|
+
$ kcon = KorAPConnection()
|
|
618
|
+
$ countries = kcon.corpusPartition("region", ["DE", "AT", "CH"], vc="pubDate in 2022")
|
|
619
|
+
$ countries.toDataFrame()[["part", "tokens"]]
|
|
620
|
+
part tokens
|
|
621
|
+
1 DE 681374138.0
|
|
622
|
+
2 AT 121466921.0
|
|
623
|
+
3 CH 96168291.0
|
|
624
|
+
$ kcon.dispersionQuery("Ameisenplage", partitions=countries)
|
|
625
|
+
```
|
|
626
|
+
"""
|
|
627
|
+
if isinstance(values, dict):
|
|
628
|
+
kwargs["values"] = {label: str(value) for label, value in values.items()}
|
|
629
|
+
elif values is not None:
|
|
630
|
+
kwargs["values"] = [str(value) for value in values]
|
|
631
|
+
return KorAPCorpusPartition(_call_keeping_r_result(KorAPClient.corpusPartition, self, by, vc=vc, **kwargs))
|
|
632
|
+
|
|
633
|
+
def dispersionQuery(self, query, partitions=None, **kwargs):
|
|
634
|
+
""" **EXPERIMENTAL**: Query the dispersion of search term(s) across corpus metadata.
|
|
635
|
+
|
|
636
|
+
Computes dispersion measures for one or more queries, i.e. how evenly their hits are distributed across
|
|
637
|
+
corpus parts such as sources, countries or years, taking the sizes of the parts into account.
|
|
638
|
+
|
|
639
|
+
- **query** - query string or list of query strings
|
|
640
|
+
- **partitions** - `KorAPCorpusPartition` (see `KorAPConnection.corpusPartition`) or dict of them. If not given, the groups are discovered automatically using `by`.
|
|
641
|
+
- **by** - metadata field or list of fields for automatic grouping: `"source"` (default), `"year"`, `"region"` or a metadata field name
|
|
642
|
+
- **regex** - optional trailing-suffix grouping rule for automatic grouping (see `KorAPQuery.dispersionAnalysis`)
|
|
643
|
+
- **vc** - base virtual corpus for automatic grouping (default = ""). With explicit partitions, their base virtual corpus is used instead.
|
|
644
|
+
- **scoreFunctions** - dict of R (!) measure functions of the form function(frequencies, sizes), see e.g. `KorAPClient.dispersionDP`. (default = `KorAPClient.defaultDispersionFunctions()`)
|
|
645
|
+
- **exactFrequencies** - if True (default), request exact counts for each part. If False, estimate them from a sample of at most `searchHitsSampleLimit` hits.
|
|
646
|
+
- **searchHitsSampleLimit** - maximum metadata sample size (default = 20000)
|
|
647
|
+
- **seed** - seed for the random selection of result pages (default = 7)
|
|
648
|
+
- **ql** - query language (default = `"poliqarp"`)
|
|
649
|
+
- **verbose** - (default = `self.verbose`)
|
|
650
|
+
- **cacheAs** - path to an RDS file to keep the result in (see `KorAPConnection.collocationAnalysis`)
|
|
651
|
+
|
|
652
|
+
Returns:
|
|
653
|
+
DataFrame with one row per query and partition and the columns `'query', 'partition', 'totalResults',
|
|
654
|
+
'sampledHits', 'usedHits', 'excludedHits', 'nParts', 'estimated', 'rangeIsLowerBound', 'distribution',
|
|
655
|
+
'sampling'`, followed by one column per measure, by default `'DP', 'DPnorm', 'JuillandD', 'CarrollD2',
|
|
656
|
+
'RosengrenS', 'range'`. Each `distribution` cell holds a DataFrame with the sizes, hit counts, relative
|
|
657
|
+
frequencies and concordance URLs of the corpus parts; each `sampling` cell holds a dict that records how
|
|
658
|
+
the counts were obtained.
|
|
659
|
+
|
|
660
|
+
Details:
|
|
661
|
+
Smaller `DP` and `DPnorm` values mean a more even distribution, whereas larger `JuillandD`, `CarrollD2` and `RosengrenS` values do. `range` is the number of parts with at least one hit.
|
|
662
|
+
Discovering the groups of a metadata field and their sizes can take time on the first call, especially for `by="source"` in large virtual corpora.
|
|
663
|
+
See `?dispersionQuery` and `` ?`dispersion-score-functions` `` in R for the interpretation of the measures.
|
|
664
|
+
|
|
665
|
+
Example:
|
|
666
|
+
```
|
|
667
|
+
$ kcon = KorAPConnection()
|
|
668
|
+
$ df = kcon.dispersionQuery(["Ameisenplage", "Mückenplage"], by="region", vc="pubDate in 2022")
|
|
669
|
+
$ df[["query", "totalResults", "nParts", "DP", "JuillandD", "range"]]
|
|
670
|
+
query totalResults nParts DP JuillandD range
|
|
671
|
+
1 Ameisenplage 2.0 7 0.265146 0.000000 1.0
|
|
672
|
+
2 Mückenplage 18.0 7 0.062939 0.495076 3.0
|
|
673
|
+
$ df["distribution"].iloc[1][["part", "tokens", "frequency"]]
|
|
674
|
+
```
|
|
675
|
+
"""
|
|
676
|
+
if partitions is not None:
|
|
677
|
+
kwargs["partitions"] = partitions
|
|
678
|
+
return _dispersion_to_pandas(_call_keeping_r_result(KorAPClient.dispersionQuery, self, query, **kwargs))
|
|
679
|
+
|
|
680
|
+
class KorAPQuery(RS4):
|
|
681
|
+
"""Query to a KorAP server."""
|
|
682
|
+
|
|
683
|
+
def __init__(self, *args, **kwargs):
|
|
684
|
+
kco = KorAPClient.corpusQuery(*args, **kwargs)
|
|
685
|
+
super().__init__(kco)
|
|
686
|
+
|
|
687
|
+
def fetchNext(self, *args, **kwargs):
|
|
688
|
+
"""Fetch next couple of query results
|
|
689
|
+
|
|
690
|
+
- **offset** - start offset for query results to fetch
|
|
691
|
+
- **maxFetch** - maximum number of query results to fetch
|
|
692
|
+
- **verbose**
|
|
693
|
+
- **randomizePageOrder** - fetch result pages in pseudo random order if true. (default = `False`)
|
|
694
|
+
|
|
695
|
+
Returns:
|
|
696
|
+
`KorAPQuery`
|
|
697
|
+
"""
|
|
698
|
+
|
|
699
|
+
res = KorAPClient.fetchNext(self, *args, **kwargs)
|
|
700
|
+
with localconverter(fix_lists_in_dataframes):
|
|
701
|
+
df = res.slots['collectedMatches']
|
|
702
|
+
res.slots['collectedMatches'] = df
|
|
703
|
+
super().__init__(res)
|
|
704
|
+
return self
|
|
705
|
+
|
|
706
|
+
def fetchRest(self, *args, **kwargs):
|
|
707
|
+
"""Fetch remaining query results
|
|
708
|
+
|
|
709
|
+
- **verbose**
|
|
710
|
+
|
|
711
|
+
Returns:
|
|
712
|
+
`KorAPQuery`
|
|
713
|
+
"""
|
|
714
|
+
res = KorAPClient.fetchRest(self, *args, **kwargs)
|
|
715
|
+
with localconverter(fix_lists_in_dataframes):
|
|
716
|
+
df = res.slots['collectedMatches']
|
|
717
|
+
res.slots['collectedMatches'] = df
|
|
718
|
+
super().__init__(res)
|
|
719
|
+
return self
|
|
720
|
+
|
|
721
|
+
def fetchAll(self, *args, **kwargs):
|
|
722
|
+
"""Fetch all query results
|
|
723
|
+
|
|
724
|
+
- **verbose**
|
|
725
|
+
|
|
726
|
+
Returns:
|
|
727
|
+
`KorAPQuery`
|
|
728
|
+
|
|
729
|
+
Example:
|
|
730
|
+
See `KorAPConnection.corpusQuery`.
|
|
731
|
+
"""
|
|
732
|
+
res = KorAPClient.fetchRest(self, *args, **kwargs)
|
|
733
|
+
with localconverter(fix_lists_in_dataframes):
|
|
734
|
+
df = res.slots['collectedMatches']
|
|
735
|
+
res.slots['collectedMatches'] = df
|
|
736
|
+
super().__init__(res)
|
|
737
|
+
return self
|
|
738
|
+
|
|
739
|
+
def fetchAnnotations(self, *args, **kwargs):
|
|
740
|
+
"""Fetches and parses linguistic annotations for the collected matches.
|
|
741
|
+
|
|
742
|
+
This method enriches the `collectedMatches` DataFrame with additional columns
|
|
743
|
+
containing linguistic annotations like lemma, part-of-speech (POS), and
|
|
744
|
+
morphology for each token in the match and its context.
|
|
745
|
+
|
|
746
|
+
Args:
|
|
747
|
+
foundry (str, optional): The foundry (annotation layer) to fetch.
|
|
748
|
+
Defaults to "tt" (TreeTagger).
|
|
749
|
+
overwrite (bool, optional): If True, existing annotation columns will be
|
|
750
|
+
overwritten. Defaults to False.
|
|
751
|
+
verbose (bool, optional): If True, prints progress information. Defaults
|
|
752
|
+
to the verbosity setting of the
|
|
753
|
+
KorAPConnection object.
|
|
754
|
+
*args: Positional arguments passed to the underlying R function.
|
|
755
|
+
**kwargs: Keyword arguments passed to the underlying R function.
|
|
756
|
+
|
|
757
|
+
Returns:
|
|
758
|
+
KorAPQuery: A new KorAPQuery object with the `collectedMatches` DataFrame
|
|
759
|
+
updated to include the fetched annotations.
|
|
760
|
+
|
|
761
|
+
Example:
|
|
762
|
+
```
|
|
763
|
+
from KorAPClient import KorAPConnection
|
|
764
|
+
|
|
765
|
+
# Authentication might be required for snippets and annotations
|
|
766
|
+
kcon = KorAPConnection(verbose=True).auth()
|
|
767
|
+
|
|
768
|
+
# Perform a query and fetch all matches
|
|
769
|
+
q = kcon.corpusQuery("Ameisenplage", metadataOnly=False).fetchAll()
|
|
770
|
+
|
|
771
|
+
# Fetch annotations for the matches
|
|
772
|
+
q_annotated = q.fetchAnnotations()
|
|
773
|
+
|
|
774
|
+
# Display the collected matches with new annotation columns
|
|
775
|
+
print(q_annotated.slots['collectedMatches'][['snippet', 'lemma.left', 'lemma.match', 'lemma.right']].head())
|
|
776
|
+
```
|
|
777
|
+
"""
|
|
778
|
+
res = KorAPClient.fetchAnnotations(self, *args, **kwargs)
|
|
779
|
+
with localconverter(fix_lists_in_dataframes):
|
|
780
|
+
df = res.slots['collectedMatches']
|
|
781
|
+
res.slots['collectedMatches'] = df
|
|
782
|
+
super().__init__(res)
|
|
783
|
+
return self
|
|
784
|
+
|
|
785
|
+
def fetchSamplePages(self, *args, **kwargs):
|
|
786
|
+
"""Fetch a reproducible random sample of result pages
|
|
787
|
+
|
|
788
|
+
Unlike `fetchNext`, which takes the first hits, complete result pages are selected at random from the
|
|
789
|
+
entire result set. If all results fit within the limit, all of them are fetched.
|
|
790
|
+
|
|
791
|
+
- **maxHits** - maximum number of hits to retrieve. Complete pages are used, so the limit is rounded down to a multiple of 50. (default = 2000)
|
|
792
|
+
- **seed** - seed for choosing pages (default = 7)
|
|
793
|
+
- **verbose**
|
|
794
|
+
- **warn** - warn that the result is a sample of pages (default = True)
|
|
795
|
+
|
|
796
|
+
Returns:
|
|
797
|
+
`KorAPQuery`, whose sample is reused by `KorAPQuery.dispersionAnalysis`
|
|
798
|
+
|
|
799
|
+
Example:
|
|
800
|
+
```
|
|
801
|
+
$ q = KorAPConnection().corpusQuery("Gendern").fetchSamplePages(maxHits=2000, seed=7)
|
|
802
|
+
$ q.dispersionAnalysis(by="region")
|
|
803
|
+
```
|
|
804
|
+
"""
|
|
805
|
+
# multivalued metadata fields, such as dmozDomain, are joined by spaces, as in the other fetch methods
|
|
806
|
+
res = _join_list_columns(KorAPClient.fetchSamplePages(self, *args, **kwargs))
|
|
807
|
+
super().__init__(res)
|
|
808
|
+
return self
|
|
809
|
+
|
|
810
|
+
def dispersionAnalysis(self, by="source", **kwargs):
|
|
811
|
+
""" **EXPERIMENTAL**: Calculates how evenly the hits of the query are distributed across corpus metadata.
|
|
812
|
+
|
|
813
|
+
The groups of the chosen metadata dimension and their sizes are discovered automatically, including
|
|
814
|
+
groups without hits.
|
|
815
|
+
|
|
816
|
+
- **by** - metadata field or list of fields. `"source"` (default) groups by `corpusTitle`, ignoring trailing years; `"year"` groups `pubDate` by publication year; `"region"` uses `pubPlaceKey`.
|
|
817
|
+
- **regex** - optional regular expression removing a trailing suffix from metadata values, ending in `$`, e.g. `" [0-9]{4}$"`, which is the default for `by="source"`
|
|
818
|
+
- **scoreFunctions** - dict of R (!) measure functions (see `KorAPConnection.dispersionQuery`)
|
|
819
|
+
- **exactFrequencies** - by default, a sample fetched with `KorAPQuery.fetchSamplePages` or completely fetched metadata are reused; otherwise exact counts per group are requested. True requests exact counts even after a fetch; False permits sampling metadata.
|
|
820
|
+
- **searchHitsSampleLimit** - maximum metadata sample size (default = 20000)
|
|
821
|
+
- **seed** - seed for the random selection of result pages (default = 7)
|
|
822
|
+
- **verbose**
|
|
823
|
+
- **cacheAs** - path to an RDS file to keep the result in (see `KorAPConnection.collocationAnalysis`)
|
|
824
|
+
|
|
825
|
+
Returns:
|
|
826
|
+
DataFrame as described in `KorAPConnection.dispersionQuery`
|
|
827
|
+
|
|
828
|
+
Details:
|
|
829
|
+
Use a metadata-only query, which is the default of `KorAPConnection.corpusQuery`.
|
|
830
|
+
To compare the dispersion of several queries, use `KorAPConnection.dispersionQuery`.
|
|
831
|
+
|
|
832
|
+
Example:
|
|
833
|
+
```
|
|
834
|
+
$ q = KorAPConnection().corpusQuery("Mückenplage", vc="pubDate in 2022")
|
|
835
|
+
$ df = q.dispersionAnalysis(by="region")
|
|
836
|
+
$ df[["query", "partition", "totalResults", "nParts", "DP", "DPnorm", "range"]]
|
|
837
|
+
query partition totalResults nParts DP DPnorm range
|
|
838
|
+
1 Mückenplage region 18.0 7 0.062939 0.062939 3.0
|
|
839
|
+
```
|
|
840
|
+
"""
|
|
841
|
+
return _dispersion_to_pandas(_call_keeping_r_result(KorAPClient.dispersionAnalysis, self, by, **kwargs))
|