KorAPClient 1.2.1__tar.gz → 1.5.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. korapclient-1.5.1/KorAPClient/__init__.py +841 -0
  2. {korapclient-1.2.1 → korapclient-1.5.1}/KorAPClient/tests/test_korapclient.py +141 -2
  3. {korapclient-1.2.1 → korapclient-1.5.1}/KorAPClient.egg-info/PKG-INFO +108 -11
  4. {korapclient-1.2.1 → korapclient-1.5.1}/KorAPClient.egg-info/SOURCES.txt +1 -12
  5. {korapclient-1.2.1 → korapclient-1.5.1}/KorAPClient.egg-info/top_level.txt +0 -4
  6. {korapclient-1.2.1 → korapclient-1.5.1}/PKG-INFO +108 -11
  7. {korapclient-1.2.1 → korapclient-1.5.1}/Readme.md +104 -6
  8. {korapclient-1.2.1 → korapclient-1.5.1}/pyproject.toml +4 -5
  9. korapclient-1.2.1/KorAPClient/__init__.py +0 -442
  10. korapclient-1.2.1/README.md +0 -1
  11. korapclient-1.2.1/examples/fetchAll.py +0 -18
  12. korapclient-1.2.1/examples/github-example.py +0 -20
  13. korapclient-1.2.1/examples/ids_theme.py +0 -106
  14. korapclient-1.2.1/examples/jason.py +0 -14
  15. korapclient-1.2.1/examples/make_sense.py +0 -18
  16. korapclient-1.2.1/examples/pandas-bokeh.py +0 -42
  17. korapclient-1.2.1/examples/stardust.py +0 -40
  18. korapclient-1.2.1/examples/test.py +0 -17
  19. korapclient-1.2.1/src/pythonkorapclient/__init__.py +0 -0
  20. korapclient-1.2.1/tests/__init__.py +0 -0
  21. {korapclient-1.2.1 → korapclient-1.5.1}/KorAPClient/__main__.py +0 -0
  22. {korapclient-1.2.1 → korapclient-1.5.1}/KorAPClient/tests/__init__.py +0 -0
  23. {korapclient-1.2.1 → korapclient-1.5.1}/KorAPClient.egg-info/dependency_links.txt +0 -0
  24. {korapclient-1.2.1 → korapclient-1.5.1}/KorAPClient.egg-info/requires.txt +0 -0
  25. {korapclient-1.2.1 → korapclient-1.5.1}/LICENSE.txt +0 -0
  26. {korapclient-1.2.1 → korapclient-1.5.1}/MANIFEST.in +0 -0
  27. {korapclient-1.2.1 → korapclient-1.5.1}/examples/collocates.py +0 -0
  28. {korapclient-1.2.1 → korapclient-1.5.1}/examples/hello_world.py +0 -0
  29. {korapclient-1.2.1 → korapclient-1.5.1}/examples/hello_world_interactive.py +0 -0
  30. {korapclient-1.2.1 → korapclient-1.5.1}/examples/variant_by_country_interactive_barchart.py +0 -0
  31. {korapclient-1.2.1 → korapclient-1.5.1}/examples/writtenVSspoken.py +0 -0
  32. {korapclient-1.2.1 → korapclient-1.5.1}/setup.cfg +0 -0
@@ -0,0 +1,841 @@
1
+ __pdoc__ = {'tests': False}
2
+
3
+ import html
4
+ import os
5
+ import re
6
+ import tempfile
7
+ import warnings
8
+ import webbrowser
9
+ from itertools import product
10
+
11
+ import pandas as pd
12
+ from rpy2.rinterface_lib.sexp import StrSexpVector, NULLType
13
+ from rpy2.robjects import numpy2ri
14
+ from rpy2.robjects.conversion import localconverter, get_conversion
15
+ from rpy2.rinterface import NULL
16
+
17
+ import rpy2.robjects as robjects
18
+ import rpy2.robjects.packages as packages
19
+ import rpy2.robjects.pandas2ri as pandas2ri
20
+ from rpy2 import rinterface as ri
21
+ from packaging import version
22
+ from rpy2.robjects.methods import RS4
23
+
24
+ CURRENT_R_PACKAGE_VERSION = "1.5.1"
25
+
26
+ KorAPClient = packages.importr('RKorAPClient')
27
+ if version.parse(KorAPClient.__version__) < version.parse(CURRENT_R_PACKAGE_VERSION):
28
+ warnings.warn("R-package RKorAPClient version " + KorAPClient.__version__ + " is outdated, please update.",
29
+ DeprecationWarning)
30
+
31
+ korapclient_converter = robjects.conversion.Converter('base empty converter')
32
+
33
+ # Export NULL
34
+ NULL = NULL
35
+
36
+ @korapclient_converter.py2rpy.register(list)
37
+ def _rpy2py_robject(listObject):
38
+ return robjects.StrVector(listObject)
39
+
40
+
41
+ @korapclient_converter.py2rpy.register(dict)
42
+ def _dict_to_named_vector(dictObject):
43
+ """Convert dicts to named R vectors, e.g. {"Germany": "pubPlaceKey=DE"} for labelled virtual corpora."""
44
+ if all(isinstance(value, str) for value in dictObject.values()):
45
+ vector = robjects.StrVector(list(dictObject.values()))
46
+ vector.names = robjects.StrVector([str(key) for key in dictObject.keys()])
47
+ return vector
48
+ return robjects.ListVector(dictObject)
49
+
50
+
51
+ robjects.conversion.set_conversion(robjects.default_converter + pandas2ri.converter + korapclient_converter)
52
+
53
+ fix_null_types = robjects.default_converter
54
+
55
+ @fix_null_types.rpy2py.register(NULLType)
56
+ def to_str(obj):
57
+ return ""
58
+
59
+ fix_lists_in_dataframes = robjects.default_converter
60
+
61
+ @fix_lists_in_dataframes.rpy2py.register(StrSexpVector)
62
+ def to_str(obj):
63
+ for i in range(len(obj)):
64
+ obj[i] = str(obj[i])
65
+ return "\t".join(obj)
66
+
67
+ def my_cv(obj, cv):
68
+ if isinstance(obj, ri.StrSexpVector):
69
+ for i in range(len(obj)):
70
+ obj[i] = str(obj[i])
71
+ return StrSexpVector((obj))
72
+ else:
73
+ return cv.rpy2py(obj)
74
+
75
+ def toDataFrame(obj):
76
+ cv = get_conversion() # get the converter from current context
77
+ names = []
78
+ objects = []
79
+ for i in range(len(obj)):
80
+ if isinstance(obj[i], ri.ListSexpVector):
81
+ list_name = obj.names[i] + "." if not isinstance(obj.names, NULLType) else "l" + str(i) + "."
82
+ for j in range(len(obj[i])):
83
+ local_name = str(obj[i].names[j]) if not isinstance(obj[i].names, NULLType) else str(j)
84
+ names.append(list_name + local_name)
85
+ objects.append(obj[i][j])
86
+ else:
87
+ names.append(obj.names[i])
88
+ objects.append(obj[i])
89
+
90
+
91
+ return pd.DataFrame(
92
+ {str(k): my_cv(objects[i], cv) for i, k in enumerate(names)}
93
+ )
94
+
95
+ # associate the converter with R data.frame class
96
+ fix_lists_in_dataframes.rpy2py_nc_map[ri.ListSexpVector].update({"data.frame": toDataFrame})
97
+
98
+
99
+
100
+ def _call_keeping_r_result(function, *args, **kwargs):
101
+ """Call an R function with the usual argument conversion, but return its result as unconverted R object."""
102
+ cv = get_conversion()
103
+ return ri.SexpClosure.__call__(function, *[cv.py2rpy(arg) for arg in args],
104
+ **{key: cv.py2rpy(value) for key, value in kwargs.items()})
105
+
106
+
107
+ _flat_columns = robjects.r("""function(x)
108
+ as.data.frame(unclass(x)[!vapply(x, is.list, logical(1))], stringsAsFactors = FALSE, check.names = FALSE)""")
109
+
110
+ _join_list_columns = robjects.r("""function(kqo) {
111
+ join <- function(x) if (length(x) == 0 || all(is.na(x))) NA_character_ else paste(x, collapse = " ")
112
+ df <- kqo@collectedMatches
113
+ df[] <- lapply(df, function(column) if (is.list(column)) vapply(column, join, character(1)) else column)
114
+ kqo@collectedMatches <- df
115
+ kqo
116
+ }""")
117
+
118
+
119
+ def _without_na_strings(df):
120
+ """Replace R's NA_character_ by None in the string columns of a DataFrame."""
121
+ for name in df.columns[df.dtypes == object]:
122
+ df[name] = df[name].map(lambda value: None if value is ri.NA_Character else value)
123
+ return df
124
+
125
+
126
+ def _r_list_to_dict(obj):
127
+ result = {}
128
+ for name, value in zip(obj.names, obj):
129
+ if isinstance(value, NULLType):
130
+ result[name] = None
131
+ elif isinstance(value, ri.ListSexpVector):
132
+ result[name] = _r_list_to_dict(value)
133
+ elif len(value) == 1:
134
+ result[name] = None if value[0] is ri.NA_Character else value[0]
135
+ else:
136
+ result[name] = list(value)
137
+ return result
138
+
139
+
140
+ def _r_value_to_py(value):
141
+ """Convert an R scalar or vector to a Python value, with NA as None."""
142
+ if isinstance(value, NULLType):
143
+ return None
144
+ values = [None if item in (ri.NA_Character, ri.NA_Integer, ri.NA_Logical) or
145
+ (isinstance(item, float) and item != item) else item for item in value]
146
+ return values[0] if len(values) == 1 else values
147
+
148
+
149
+ def _dispersion_to_pandas(res):
150
+ """Convert a dispersion result with nested tables to a DataFrame with DataFrames and dicts in its list columns."""
151
+ cv = get_conversion()
152
+ df = _without_na_strings(_flat_columns(res))
153
+ for position, (name, column) in enumerate(zip(res.names, res)):
154
+ if isinstance(column, ri.ListSexpVector):
155
+ values = [_without_na_strings(cv.rpy2py(value)) if "data.frame" in value.rclass
156
+ else _r_list_to_dict(value) for value in column]
157
+ df.insert(position, name, pd.Series(values, index=df.index, dtype=object))
158
+ return df
159
+
160
+
161
+ _hc_dispersion = robjects.r("""function(rows, distributions, apiUrls, indexRevisions, probabilities, partition, top) {
162
+ rows <- tibble::as_tibble(rows)
163
+ rows$distribution <- lapply(distributions, function(part) {
164
+ part <- tibble::as_tibble(part)
165
+ for (field in intersect(c("part", "vc", "webUIRequestUrl"), names(part)))
166
+ part[[field]] <- as.character(part[[field]])
167
+ part
168
+ })
169
+ rows$sampling <- lapply(seq_along(probabilities), function(i)
170
+ list(apiUrl = apiUrls[[i]], indexRevision = indexRevisions[[i]], inclusionProbability = probabilities[[i]]))
171
+ RKorAPClient::hc_dispersion(rows, partition = partition, top = top)
172
+ }""")
173
+
174
+ _save_widget = robjects.r("""function(widget, file, title)
175
+ htmlwidgets::saveWidget(widget, file, selfcontained = FALSE, libdir = "lib", title = title)""")
176
+
177
+
178
+ def _inline_resources(document, directory):
179
+ """Embed the scripts and stylesheets that an HTML document loads from directory into the document."""
180
+ def read(name):
181
+ with open(os.path.join(directory, name), encoding="utf-8") as file:
182
+ return file.read()
183
+
184
+ document = re.sub(r'<script src="(lib/[^"]+)"></script>',
185
+ lambda match: "<script>" + read(match.group(1)).replace("</script", "<\\/script") + "</script>",
186
+ document)
187
+ return re.sub(r'<link href="(lib/[^"]+)" rel="stylesheet" />',
188
+ lambda match: "<style>" + read(match.group(1)) + "</style>", document)
189
+
190
+
191
+ class Highchart:
192
+ """Interactive [Highcharts](https://www.highcharts.com/) chart created by an R plotting function.
193
+
194
+ In Jupyter notebooks, the chart is displayed when it is the last expression of a cell.
195
+ Elsewhere use `Highchart.show` or `Highchart.save`.
196
+ """
197
+
198
+ def __init__(self, widget, title="KorAP chart", height=600):
199
+ self.widget = widget
200
+ self.title = title
201
+ self.height = height
202
+
203
+ def to_html(self):
204
+ """Return the chart as self-contained HTML document."""
205
+ with tempfile.TemporaryDirectory() as directory:
206
+ path = os.path.join(directory, "chart.html")
207
+ _save_widget(self.widget, path, self.title)
208
+ with open(path, encoding="utf-8") as file:
209
+ return _inline_resources(file.read(), directory)
210
+
211
+ def save(self, path):
212
+ """Save the chart as self-contained HTML file.
213
+
214
+ - **path** - name of the HTML file
215
+
216
+ Returns:
217
+ absolute path of the HTML file
218
+ """
219
+ path = os.path.abspath(os.path.expanduser(path))
220
+ with open(path, "w", encoding="utf-8") as file:
221
+ file.write(self.to_html())
222
+ return path
223
+
224
+ def show(self):
225
+ """Open the chart in the default web browser."""
226
+ file = tempfile.NamedTemporaryFile(suffix=".html", delete=False)
227
+ file.close()
228
+ webbrowser.open("file://" + self.save(file.name))
229
+
230
+ def _repr_html_(self):
231
+ return '<iframe srcdoc="{}" style="width: 100%; height: {}px; border: none"></iframe>'.format(
232
+ html.escape(self.to_html(), quote=True), self.height)
233
+
234
+
235
+ def hc_dispersion(df, partition=None, top=None):
236
+ """ **EXPERIMENTAL**: Plot the distribution of dispersion analysis results across corpus parts.
237
+
238
+ Creates an interactive bar chart that compares, for each corpus part, its share of the corpus tokens
239
+ with its share of the hits of each query.
240
+
241
+ - **df** - DataFrame returned by `KorAPConnection.dispersionQuery` or `KorAPQuery.dispersionAnalysis`. Results of several calls can be combined with `pandas.concat`, as long as they refer to the same corpus parts.
242
+ - **partition** - name of the partition to plot, e.g. `"source"`. Required if `df` contains more than one partition.
243
+ - **top** - number of corpus parts to show individually; the remaining parts are combined (default = 12, or all parts for calendar partitions like `"year"`). Use `float("inf")` to show all parts.
244
+
245
+ Returns:
246
+ `Highchart`
247
+
248
+ Details:
249
+ The series are labelled with the `variant` column of `df` if there is one, otherwise with its `query` column.
250
+
251
+ Example:
252
+ ```
253
+ $ kcon = KorAPConnection()
254
+ $ df = kcon.dispersionQuery(["Ameisenplage", "Mückenplage"], by="region", vc="pubDate in 2022")
255
+ $ hc_dispersion(df).save("dispersion.html")
256
+ ```
257
+ """
258
+ required = ["partition", "query", "distribution", "estimated", "sampling"]
259
+ if not isinstance(df, pd.DataFrame) or len(df) == 0 or not all(name in df.columns for name in required):
260
+ raise ValueError("Use a result of dispersionQuery() or dispersionAnalysis() with distribution data.")
261
+ cv = get_conversion()
262
+
263
+ def r_list(values):
264
+ return ri.ListSexpVector([cv.py2rpy(value) for value in values])
265
+
266
+ def sampling_field(name, na):
267
+ values = [sampling.get(name) if isinstance(sampling, dict) else None for sampling in df["sampling"]]
268
+ return r_list([na if value is None else value for value in values])
269
+
270
+ rows = df.drop(columns=["distribution", "sampling"]).reset_index(drop=True)
271
+ widget = ri.SexpClosure.__call__(
272
+ _hc_dispersion, cv.py2rpy(rows), r_list(df["distribution"]),
273
+ sampling_field("apiUrl", ri.NA_Character), sampling_field("indexRevision", ri.NA_Character),
274
+ sampling_field("inclusionProbability", ri.NA_Real),
275
+ NULL if partition is None else cv.py2rpy(partition), NULL if top is None else cv.py2rpy(top))
276
+ return Highchart(widget, title="Dispersion" if partition is None else "Dispersion by " + partition)
277
+
278
+
279
+ class KorAPCorpusPartition(robjects.vectors.DataFrame):
280
+ """Corpus parts and their token counts, as returned by `KorAPConnection.corpusPartition`.
281
+
282
+ Pass it as `partitions` argument to `KorAPConnection.dispersionQuery`. Use `toDataFrame` to inspect it.
283
+ """
284
+
285
+ def toDataFrame(self):
286
+ """Returns the parts as DataFrame with columns `'part', 'value', 'vc', 'tokens'`."""
287
+ return _without_na_strings(_flat_columns(self))
288
+
289
+
290
+ def expand_grid(dictionary):
291
+ """Create a pandas DataFrame from all combinations of inputs
292
+
293
+ - **dictionary** - dict with variable names as keys and their values as vectors
294
+
295
+ Returns:
296
+ DataFrame with column names as specified by the dictionary key and all combinations of the specified values
297
+ in the rows.
298
+
299
+ Example:
300
+ ```
301
+ $ df = expand_grid({"Year": range(2010, 2019), "Country": ["DE", "CH"] })
302
+
303
+ $ df["vc"] = "textType=/Zeit.*/ & pubPlaceKey = " + df.Country + " & pubDate in " + list(map(str, df.Year))
304
+ ```
305
+ """
306
+
307
+ return pd.DataFrame([row for row in product(*dictionary.values())],
308
+ columns=dictionary.keys())
309
+
310
+
311
+ # noinspection PyPep8Naming
312
+ class KorAPConnection(RS4):
313
+ """Connection to a KorAP server."""
314
+
315
+ def __init__(self, *args, **kwargs):
316
+ """Constructor keyword arguments:
317
+
318
+ - **KorAPUrl** (default = `"https://korap.ids-mannheim.de/"`)
319
+ - **apiVersion** (default = 'v1.0')
320
+ - **apiUrl**
321
+ - **accessToken** (default = `getAccessToken(KorAPUrl)`
322
+ - **userAgent** (default = `"Python-KorAP-Client"`)
323
+ - **timeout** - maximum time in seconds a request may take (default = 240). Long running searches are no longer
324
+ aborted after 10 minutes of low transfer rates; only this timeout applies.
325
+ - **verbose** (default = False)
326
+ - **cache** (default = True)
327
+ """
328
+ if 'userAgent' not in kwargs:
329
+ kwargs["userAgent"] = "Python-KorAP-Client"
330
+ kco = KorAPClient.KorAPConnection(*args, **kwargs)
331
+ super().__init__(kco)
332
+
333
+ def auth(self, *args, **kwargs):
334
+ """ Authorize PythonKorAPClient to make KorAP queries and download results on behalf of the user.
335
+
336
+ - **kco** - `KorAPConnection` object
337
+ - **app_id** - OAuth2 application id. Defaults to the generic KorAP client application id.
338
+ - **app_secret** - OAuth2 application secret. Used with confidential client applications. Defaults to `NULL`.
339
+ - **scope** - OAuth2 scope. Defaults to "search match_info".
340
+
341
+ Returns:
342
+
343
+ Potentially authorized `KorAPConnection`|`RS4` with access token in `.slots['accessToken']`.
344
+
345
+ Example:
346
+ ```
347
+ from KorAPClient import KorAPConnection, NULL
348
+
349
+ ### Create a KorAPConnection object without an existing access token
350
+
351
+ kcon = KorAPConnection(accessToken=NULL, verbose=True).auth()
352
+
353
+ ### Perform a query using the authenticated connection
354
+
355
+ q = kcon.corpusQuery("Ameisenplage", metadataOnly=False)
356
+
357
+ ### Fetch all results
358
+
359
+ q = q.fetchAll()
360
+
361
+ ### Access the collected matches
362
+
363
+ print(q.slots['collectedMatches'].snippet)
364
+ ```
365
+
366
+ """
367
+
368
+ kco = KorAPClient.auth(self, *args, **kwargs)
369
+ super().__init__(kco)
370
+ return self
371
+
372
+ def corpusStats(self, *args, **kwargs):
373
+ """Query the size of the whole corpus or a virtual corpus specified by the vc argument.
374
+
375
+ - **vc** - virtual corpus definition, list thereof, or dict mapping labels to definitions (default = "").
376
+ Dict keys are used as labels in a `label` column (see `KorAPConnection.frequencyQuery`).
377
+ - **verbose** (default = kco@verbose)
378
+ - **as.df** (default = True)
379
+ - **cacheAs** - path to an RDS file to keep the result in (see `KorAPConnection.collocationAnalysis`)
380
+
381
+ Returns:
382
+ `DataFrame`|`RS4`
383
+
384
+ Example:
385
+ ```
386
+ $ df = kcon.corpusStats("pubDate in 2018 & textType=/Zeit.*/ & pubPlaceKey=IT", **{"as.df": True})
387
+ $ df["tokens"]
388
+ 12150897
389
+ ```
390
+ """
391
+ default_kwargs = {"as.df": True}
392
+ default_kwargs.update(kwargs)
393
+ return KorAPClient.corpusStats(self, *args, **default_kwargs)
394
+
395
+ def frequencyQuery(self, *args, **kwargs):
396
+ """Query relative frequency of search term(s).
397
+
398
+ - **query** - query string or list of query strings
399
+ - **vc** - virtual corpus definition, list thereof, or dict mapping labels to definitions (default: "")
400
+ - **conf.level** - confidence level of the returned confidence interval (default = 0.95)
401
+ - **as.alternatives** - decides whether queries should be treated as mutually exclusive and exhaustive wrt. to some meaningful class (e.g. spelling variants of a certain word form) (default = False)
402
+ - **KorAPUrl** - instead of specifying the `query` and `vc` string parameters, you can copy your KorAP query URL here from the browser
403
+ - **metadataOnly** - determines whether queries should return only metadata without any snippets. This can also be useful to prevent access rewrites. (default = True)
404
+ - **ql** - query language: `"poliqarp" | "cosmas2" | "annis" | "cql" | "fcsql"` (default = `"poliqarp"`)
405
+ - **accessRewriteFatal** - abort if query or given vc had to be rewritten due to insufficient rights (not yet implemented) (default = `True`)
406
+ - **verbose** - (default = `self.verbose`)
407
+ - **expand** - bool that decides if `query` and `vc` parameters are expanded to all of their combinations (default = `len(vc) != len(query)`)
408
+ - **cacheAs** - path to an RDS file to keep the result in (see `KorAPConnection.collocationAnalysis`)
409
+
410
+ Returns:
411
+ DataFrame with columns `'query', 'totalResults', 'vc', 'webUIRequestUrl', 'queryDuration', 'total', 'f',
412
+ 'conf.low', 'conf.high'`. `queryDuration` is the time in seconds a request took on the client side.
413
+ If `vc` is a dict, its keys are used as labels in an additional `label` column.
414
+ A warning is issued if a request failed or the server cut a search short (`timeExceeded`).
415
+
416
+ Example:
417
+ ```
418
+ $ kcon = KorAPConnection(verbose=True)
419
+ $ kcon.frequencyQuery("Ameisenplage", vc=["pubDate in "+str(y) for y in range(2010,2015)])
420
+ query totalResults ... conf.low conf.high
421
+ 1 Ameisenplage 3 ... 9.727696e-10 1.200289e-08
422
+ 2 Ameisenplage 12 ... 3.838218e-09 1.275717e-08
423
+ 3 Ameisenplage 5 ... 2.013352e-09 1.356500e-08
424
+ 4 Ameisenplage 6 ... 2.691331e-09 1.519888e-08
425
+ 5 Ameisenplage 3 ... 8.629463e-10 1.064780e-08
426
+ ```
427
+
428
+ To label virtual corpora, pass a dict:
429
+ ```
430
+ $ vc = {"Germany": "pubPlaceKey=DE", "Switzerland": "pubPlaceKey=CH"}
431
+ $ kcon.frequencyQuery("Ameisenplage", vc=vc)[["query", "label", "totalResults"]]
432
+ query label totalResults
433
+ 1 Ameisenplage Germany 178
434
+ 2 Ameisenplage Switzerland 27
435
+ ```
436
+ """
437
+ return KorAPClient.frequencyQuery(self, *args, **kwargs)
438
+
439
+ def collocationScoreQuery(self, node, collocate, vc="", **kwargs):
440
+ """Get collocation scores for given node(s) and collocate(s).
441
+
442
+ - **node** - target word
443
+ - **collocate** - collocate of target word or list thereof (all combinations of collocates and vcs are queried)
444
+ - **vc** - virtual corpus definition, list thereof, or dict mapping labels to definitions (default: ""). Dict keys are used as labels.
445
+ - **lemmatizeNodeQuery** - logical, set to TRUE if node query should be lemmatized, i.e. x -> [tt/l=x]
446
+ - **lemmatizeCollocateQuery** - logical, set to TRUE if collocate query should be lemmatized, i.e. x -> [tt/l=x]
447
+ - **leftContextSize** - size of the left context window
448
+ - **rightContextSize** - size of the right context window
449
+ - **scoreFunctions** - dict of R (!) score functions of the form function(O1, O2, O, N, E, window_size), see e.g. KorAPClient.pmi
450
+ - **smoothingConstant** - smoothing constant will be added to all observed values
451
+ - **cacheAs** - path to an RDS file to keep the result in (see `KorAPConnection.collocationAnalysis`)
452
+
453
+ Returns:
454
+ DataFrame with columns `'node', 'collocate', 'label', 'vc','webUIRequestUrl', 'w', 'leftContextSize',
455
+ 'rightContextSize', 'N', 'O', 'O1', 'O2', 'E', 'pmi', 'mi2', 'mi3', 'logDice', 'll'`
456
+
457
+ Example:
458
+ ```
459
+ $ kcon = KorAPConnection(verbose=True)
460
+ $ df = kcon.collocationScoreQuery("Grund", "triftiger")
461
+ ```
462
+ """
463
+ return KorAPClient.collocationScoreQuery(self, node, collocate, vc, **kwargs)
464
+
465
+ def collocationAnalysis(self, node, vc="", **kwargs):
466
+ """ **EXPERIMENTAL**: Performs a collocation analysis for the given node (or query) in the given virtual corpus.
467
+
468
+ - **node** - target word or list of target words
469
+ - **vc** - string, list of strings, or dict mapping labels to strings describing the virtual corpus in which the query should be performed. An empty string (default) means the whole corpus, as far as it is license-wise accessible.
470
+ - **lemmatizeNodeQuery** - if True, node query will be lemmatized, i.e. x -> [tt/l=x]
471
+ - **minOccur** - minimum absolute number of observed co-occurrences to consider a collocate candidate
472
+ - **leftContextSize** - size of the left context window
473
+ - **rightContextSize** - size of the right context window
474
+ - **topCollocatesLimit** - limit analysis to the n most frequent collocates in the search hits sample
475
+ - **searchHitsSampleLimit** - limit the size of the search hits sample
476
+ - **ignoreCollocateCase** - bool, set to True if collocate case should be ignored
477
+ - **withinSpan** - KorAP span specification (see <https://korap.ids-mannheim.de/doc/ql/poliqarp-plus?embedded=true#spans>) for collocations to be searched within. Defaults to `base/s=s`
478
+ - **exactFrequencies** - if False, extrapolate observed co-occurrence frequencies from frequencies in search hits sample, otherwise retrieve exact co-occurrence frequencies
479
+ - **stopwords** - vector of stopwords not to be considered as collocates
480
+ - **seed** - seed for random page collecting order
481
+ - **expand** - if True, node and vc parameters are expanded to all of their combinations
482
+ - **minObservedExpectedRatio** - minimum ratio of observed to expected co-occurrence frequency for a collocate
483
+ to be kept (default = 1, i.e. collocates occurring less often than expected by chance are discarded). Set to 0
484
+ for the unfiltered results of earlier versions.
485
+ - **cacheAs** - path to an RDS file to keep the result in. If the file exists and records the same call, it is
486
+ read back instead of contacting the server; otherwise the analysis is run and its result stored there
487
+ (`.rds` is appended if missing). By default, no file is used.
488
+
489
+ Returns:
490
+ DataFrame with columns `'node', 'collocate', 'label', 'vc','webUIRequestUrl', 'w', 'leftContextSize',
491
+ 'rightContextSize', 'N', 'O', 'O1', 'O2', 'E', 'pmi', 'mi2', 'mi3', 'logDice', 'll'`
492
+
493
+ Details:
494
+ The collocation analysis is currently implemented on the client side, as some of the functionality is not yet provided by the KorAP backend. Mainly for this reason it is very slow (several minutes, up to hours), but on the other hand very flexible. You can, for example, perform the analysis in arbitrary virtual corpora, use complex node queries, and look for expression-internal collocates using the focus function (see examples and demo).
495
+ `logDice` is computed as defined by Rychlý (2008), `14 + log2(2 * O / (O1 + O2))`, so that values are comparable to those of other tools. `ll` follows Evert (2004).
496
+ When `vc` is a list or dict of virtual corpora, the results are compared across them (**experimental**, see `?collocationAnalysis` in R).
497
+ To increase speed at the cost of accuracy and possible false negatives, you can decrease searchHitsSampleLimit and/or topCollocatesLimit and/or set exactFrequencies to FALSE.
498
+ Note that currently not the tokenization provided by the backend, i.e. the corpus itself, is used, but a tinkered one. This can also lead to false negatives and to frequencies that differ from corresponding ones acquired via the web user interface.
499
+
500
+ Example:
501
+ ```
502
+ $ kcon = KorAPConnection(verbose=True)
503
+ $ df = kcon.collocationAnalysis("Grund")
504
+ ```
505
+ """
506
+ return KorAPClient.collocationAnalysis(self, node, vc, **kwargs)
507
+
508
+ def mergeDuplicateCollocates(self, *args, **kwargs):
509
+ """Merge collocation analysis results for different context positions."""
510
+ return KorAPClient.mergeDuplicateCollocates(*args, **kwargs)
511
+
512
+
513
+ def corpusQuery(self, *args, **kwargs):
514
+ """Query search term(s).
515
+
516
+ - **query** - query string or list of query strings
517
+ - **vc** - virtual corpus definition or list thereof (default: "")
518
+ - **KorAPUrl** - instead of specifying the `query` and `vc` string parameters, you can copy your KorAP query URL here from the browser
519
+ - **metadataOnly** - determines whether queries should return only metadata without any snippets. This can also be useful to prevent access rewrites. (default = True)
520
+ - **ql** - query language: `"poliqarp" | "cosmas2" | "annis" | "cql" | "fcsql"` (default = `"poliqarp"`)
521
+ - **fields** - (meta)data fields that will be fetched for every match (default = `["corpusSigle", "textSigle", "pubDate", "pubPlace", "availability", "textClass", "dmozDomain", "wikiDomain", "snippet", "tokens"]`).
522
+ Since DeReKo-KorAP-2026-II, the topic domain of DeReKo texts is given by `dmozDomain` and `wikiDomain` instead of `textClass`.
523
+ - **verbose** - (default = `self.verbose`)
524
+
525
+ Returns:
526
+ `KorAPQuery`
527
+
528
+ Example:
529
+ ```
530
+ $ kcon = KorAPConnection(verbose=True)
531
+ $ q = kcon.corpusQuery("Ameisenplage")
532
+ $ q = q.fetchAll()
533
+ $ q.slots['collectedMatches']
534
+ corpusSigle ... textClass
535
+ 1 WPD17 ... NaN
536
+ 2 WPD17 ... NaN
537
+ 3 WPD17 ... NaN
538
+ 4 WPD17 ... NaN
539
+ 5 WPD17 ... NaN
540
+ .. ... ... ...
541
+ 126 Z83 ... freizeit-unterhaltung reisen
542
+ 127 MZE03 ... freizeit-unterhaltung reisen natur-umwelt wett...
543
+ 128 MZE03 ... freizeit-unterhaltung reisen staat-gesellschaf...
544
+ 129 MZE14 ... wissenschaft populaerwissenschaft freizeit-unt...
545
+ 130 MZE00 ... wissenschaft populaerwissenschaft
546
+ [130 rows x 6 columns]
547
+ ```
548
+ """
549
+ return KorAPQuery(self, *args, **kwargs)
550
+
551
+ def textMetadata(self, textSigle, **kwargs):
552
+ """ Retrieves metadata for a text, identified by its sigle (id) using the corresponding KorAP API
553
+ (see `Kustvakt Wiki https://github.com/KorAP/Kustvakt/wiki/Service:-Metadata-Retrieval`).
554
+
555
+ - **textSigle** - unique text id (concatenation of corpus, document and text ids, separated by `/`, e.g. ) or list thereof
556
+ - **cacheAs** - path to an RDS file to keep the result in (see `KorAPConnection.collocationAnalysis`)
557
+
558
+ Returns:
559
+ DataFrame with columns for each metadata property. In case of errors, such as non-existing texts/sigles, the tibble will also contain a column called `errors`.
560
+ If there are metadata columns you cannot make sense of, please ignore them. The function simply returns all the metadata it gets from the server.
561
+
562
+ Example:
563
+ ```
564
+ $ kcon = KorAPConnection(verbose=True)
565
+ $ kcon.textMetadata(["WUD17/A97/08542", "WUD17/B96/57558", "WUD17/A97/08541"])
566
+ ```
567
+ """
568
+ return KorAPClient.textMetadata(self, textSigle, **kwargs)
569
+
570
+ def maintenanceInfo(self, refresh=False):
571
+ """Inspect planned maintenance of the KorAP instance
572
+
573
+ Retrieves the public maintenance schedule from the KorAP frontend of the connected instance. No authorization is needed.
574
+ Before server requests, the client checks for maintenance notices at most once every five minutes and announces each
575
+ maintenance window. To disable these checks, run `rpy2.robjects.r("options(rkorap.maintenance.notices = FALSE)")`.
576
+
577
+ - **refresh** - if True, bypass the five-minute schedule cache (default = `False`)
578
+
579
+ Returns:
580
+ dict with `'state'` (`"ok"`, `"stale"`, `"unknown"` or `"unsupported"`), `'time_zone'`, the Unix timestamps
581
+ `'last_success'` and `'valid_until'` and an `'events'` DataFrame with the columns `'start', 'end'` (ISO 8601),
582
+ `'start_epoch', 'end_epoch'` (Unix timestamps), `'all_day'` and `'ongoing'`. All-day end dates are exclusive.
583
+ An empty schedule with state `"unknown"` does not mean that no maintenance is planned.
584
+
585
+ Example:
586
+ ```
587
+ $ kcon = KorAPConnection()
588
+ $ kcon.maintenanceInfo()["events"]
589
+ ```
590
+ """
591
+ res = _call_keeping_r_result(KorAPClient.maintenanceInfo, self, refresh=refresh)
592
+ cv = get_conversion()
593
+ return {name: cv.rpy2py(value) if "data.frame" in value.rclass else _r_value_to_py(value)
594
+ for name, value in zip(res.names, res)}
595
+
596
+ def corpusPartition(self, by="source", values=None, vc="", **kwargs):
597
+ """ **EXPERIMENTAL**: Define corpus parts for a dispersion analysis.
598
+
599
+ Discovers metadata groups and their token counts, or defines a custom partition. Discovery checks the base
600
+ corpus independently of query hits, so that parts without hits are retained.
601
+
602
+ - **by** - metadata field. The aliases `"source"`, `"year"` and `"region"` stand for `corpusTitle`, `pubDate` (publication year) and `pubPlaceKey` respectively.
603
+ - **values** - list of metadata values, or dict mapping part labels to values. If neither `values` nor `parts` is given, the groups are discovered automatically.
604
+ - **vc** - base virtual corpus. All parts are intersected with this corpus. (default = "")
605
+ - **parts** - alternatively, a dict mapping part labels to virtual corpus definitions
606
+ - **regex** - optional trailing-suffix grouping rule for automatic discovery (see `KorAPQuery.dispersionAnalysis`)
607
+ - **verbose** - (default = `self.verbose`)
608
+ - **cacheAs** - path to an RDS file to keep the result in (see `KorAPConnection.collocationAnalysis`)
609
+
610
+ Returns:
611
+ `KorAPCorpusPartition` that can be passed to `KorAPConnection.dispersionQuery`.
612
+ Use its `toDataFrame()` method to get the parts with their virtual corpus definitions and token counts.
613
+ A warning is issued if the listed parts do not cover the whole base corpus.
614
+
615
+ Example:
616
+ ```
617
+ $ kcon = KorAPConnection()
618
+ $ countries = kcon.corpusPartition("region", ["DE", "AT", "CH"], vc="pubDate in 2022")
619
+ $ countries.toDataFrame()[["part", "tokens"]]
620
+ part tokens
621
+ 1 DE 681374138.0
622
+ 2 AT 121466921.0
623
+ 3 CH 96168291.0
624
+ $ kcon.dispersionQuery("Ameisenplage", partitions=countries)
625
+ ```
626
+ """
627
+ if isinstance(values, dict):
628
+ kwargs["values"] = {label: str(value) for label, value in values.items()}
629
+ elif values is not None:
630
+ kwargs["values"] = [str(value) for value in values]
631
+ return KorAPCorpusPartition(_call_keeping_r_result(KorAPClient.corpusPartition, self, by, vc=vc, **kwargs))
632
+
633
+ def dispersionQuery(self, query, partitions=None, **kwargs):
634
+ """ **EXPERIMENTAL**: Query the dispersion of search term(s) across corpus metadata.
635
+
636
+ Computes dispersion measures for one or more queries, i.e. how evenly their hits are distributed across
637
+ corpus parts such as sources, countries or years, taking the sizes of the parts into account.
638
+
639
+ - **query** - query string or list of query strings
640
+ - **partitions** - `KorAPCorpusPartition` (see `KorAPConnection.corpusPartition`) or dict of them. If not given, the groups are discovered automatically using `by`.
641
+ - **by** - metadata field or list of fields for automatic grouping: `"source"` (default), `"year"`, `"region"` or a metadata field name
642
+ - **regex** - optional trailing-suffix grouping rule for automatic grouping (see `KorAPQuery.dispersionAnalysis`)
643
+ - **vc** - base virtual corpus for automatic grouping (default = ""). With explicit partitions, their base virtual corpus is used instead.
644
+ - **scoreFunctions** - dict of R (!) measure functions of the form function(frequencies, sizes), see e.g. `KorAPClient.dispersionDP`. (default = `KorAPClient.defaultDispersionFunctions()`)
645
+ - **exactFrequencies** - if True (default), request exact counts for each part. If False, estimate them from a sample of at most `searchHitsSampleLimit` hits.
646
+ - **searchHitsSampleLimit** - maximum metadata sample size (default = 20000)
647
+ - **seed** - seed for the random selection of result pages (default = 7)
648
+ - **ql** - query language (default = `"poliqarp"`)
649
+ - **verbose** - (default = `self.verbose`)
650
+ - **cacheAs** - path to an RDS file to keep the result in (see `KorAPConnection.collocationAnalysis`)
651
+
652
+ Returns:
653
+ DataFrame with one row per query and partition and the columns `'query', 'partition', 'totalResults',
654
+ 'sampledHits', 'usedHits', 'excludedHits', 'nParts', 'estimated', 'rangeIsLowerBound', 'distribution',
655
+ 'sampling'`, followed by one column per measure, by default `'DP', 'DPnorm', 'JuillandD', 'CarrollD2',
656
+ 'RosengrenS', 'range'`. Each `distribution` cell holds a DataFrame with the sizes, hit counts, relative
657
+ frequencies and concordance URLs of the corpus parts; each `sampling` cell holds a dict that records how
658
+ the counts were obtained.
659
+
660
+ Details:
661
+ Smaller `DP` and `DPnorm` values mean a more even distribution, whereas larger `JuillandD`, `CarrollD2` and `RosengrenS` values do. `range` is the number of parts with at least one hit.
662
+ Discovering the groups of a metadata field and their sizes can take time on the first call, especially for `by="source"` in large virtual corpora.
663
+ See `?dispersionQuery` and `` ?`dispersion-score-functions` `` in R for the interpretation of the measures.
664
+
665
+ Example:
666
+ ```
667
+ $ kcon = KorAPConnection()
668
+ $ df = kcon.dispersionQuery(["Ameisenplage", "Mückenplage"], by="region", vc="pubDate in 2022")
669
+ $ df[["query", "totalResults", "nParts", "DP", "JuillandD", "range"]]
670
+ query totalResults nParts DP JuillandD range
671
+ 1 Ameisenplage 2.0 7 0.265146 0.000000 1.0
672
+ 2 Mückenplage 18.0 7 0.062939 0.495076 3.0
673
+ $ df["distribution"].iloc[1][["part", "tokens", "frequency"]]
674
+ ```
675
+ """
676
+ if partitions is not None:
677
+ kwargs["partitions"] = partitions
678
+ return _dispersion_to_pandas(_call_keeping_r_result(KorAPClient.dispersionQuery, self, query, **kwargs))
679
+
680
+ class KorAPQuery(RS4):
681
+ """Query to a KorAP server."""
682
+
683
+ def __init__(self, *args, **kwargs):
684
+ kco = KorAPClient.corpusQuery(*args, **kwargs)
685
+ super().__init__(kco)
686
+
687
+ def fetchNext(self, *args, **kwargs):
688
+ """Fetch next couple of query results
689
+
690
+ - **offset** - start offset for query results to fetch
691
+ - **maxFetch** - maximum number of query results to fetch
692
+ - **verbose**
693
+ - **randomizePageOrder** - fetch result pages in pseudo random order if true. (default = `False`)
694
+
695
+ Returns:
696
+ `KorAPQuery`
697
+ """
698
+
699
+ res = KorAPClient.fetchNext(self, *args, **kwargs)
700
+ with localconverter(fix_lists_in_dataframes):
701
+ df = res.slots['collectedMatches']
702
+ res.slots['collectedMatches'] = df
703
+ super().__init__(res)
704
+ return self
705
+
706
+ def fetchRest(self, *args, **kwargs):
707
+ """Fetch remaining query results
708
+
709
+ - **verbose**
710
+
711
+ Returns:
712
+ `KorAPQuery`
713
+ """
714
+ res = KorAPClient.fetchRest(self, *args, **kwargs)
715
+ with localconverter(fix_lists_in_dataframes):
716
+ df = res.slots['collectedMatches']
717
+ res.slots['collectedMatches'] = df
718
+ super().__init__(res)
719
+ return self
720
+
721
+ def fetchAll(self, *args, **kwargs):
722
+ """Fetch all query results
723
+
724
+ - **verbose**
725
+
726
+ Returns:
727
+ `KorAPQuery`
728
+
729
+ Example:
730
+ See `KorAPConnection.corpusQuery`.
731
+ """
732
+ res = KorAPClient.fetchRest(self, *args, **kwargs)
733
+ with localconverter(fix_lists_in_dataframes):
734
+ df = res.slots['collectedMatches']
735
+ res.slots['collectedMatches'] = df
736
+ super().__init__(res)
737
+ return self
738
+
739
+ def fetchAnnotations(self, *args, **kwargs):
740
+ """Fetches and parses linguistic annotations for the collected matches.
741
+
742
+ This method enriches the `collectedMatches` DataFrame with additional columns
743
+ containing linguistic annotations like lemma, part-of-speech (POS), and
744
+ morphology for each token in the match and its context.
745
+
746
+ Args:
747
+ foundry (str, optional): The foundry (annotation layer) to fetch.
748
+ Defaults to "tt" (TreeTagger).
749
+ overwrite (bool, optional): If True, existing annotation columns will be
750
+ overwritten. Defaults to False.
751
+ verbose (bool, optional): If True, prints progress information. Defaults
752
+ to the verbosity setting of the
753
+ KorAPConnection object.
754
+ *args: Positional arguments passed to the underlying R function.
755
+ **kwargs: Keyword arguments passed to the underlying R function.
756
+
757
+ Returns:
758
+ KorAPQuery: A new KorAPQuery object with the `collectedMatches` DataFrame
759
+ updated to include the fetched annotations.
760
+
761
+ Example:
762
+ ```
763
+ from KorAPClient import KorAPConnection
764
+
765
+ # Authentication might be required for snippets and annotations
766
+ kcon = KorAPConnection(verbose=True).auth()
767
+
768
+ # Perform a query and fetch all matches
769
+ q = kcon.corpusQuery("Ameisenplage", metadataOnly=False).fetchAll()
770
+
771
+ # Fetch annotations for the matches
772
+ q_annotated = q.fetchAnnotations()
773
+
774
+ # Display the collected matches with new annotation columns
775
+ print(q_annotated.slots['collectedMatches'][['snippet', 'lemma.left', 'lemma.match', 'lemma.right']].head())
776
+ ```
777
+ """
778
+ res = KorAPClient.fetchAnnotations(self, *args, **kwargs)
779
+ with localconverter(fix_lists_in_dataframes):
780
+ df = res.slots['collectedMatches']
781
+ res.slots['collectedMatches'] = df
782
+ super().__init__(res)
783
+ return self
784
+
785
+ def fetchSamplePages(self, *args, **kwargs):
786
+ """Fetch a reproducible random sample of result pages
787
+
788
+ Unlike `fetchNext`, which takes the first hits, complete result pages are selected at random from the
789
+ entire result set. If all results fit within the limit, all of them are fetched.
790
+
791
+ - **maxHits** - maximum number of hits to retrieve. Complete pages are used, so the limit is rounded down to a multiple of 50. (default = 2000)
792
+ - **seed** - seed for choosing pages (default = 7)
793
+ - **verbose**
794
+ - **warn** - warn that the result is a sample of pages (default = True)
795
+
796
+ Returns:
797
+ `KorAPQuery`, whose sample is reused by `KorAPQuery.dispersionAnalysis`
798
+
799
+ Example:
800
+ ```
801
+ $ q = KorAPConnection().corpusQuery("Gendern").fetchSamplePages(maxHits=2000, seed=7)
802
+ $ q.dispersionAnalysis(by="region")
803
+ ```
804
+ """
805
+ # multivalued metadata fields, such as dmozDomain, are joined by spaces, as in the other fetch methods
806
+ res = _join_list_columns(KorAPClient.fetchSamplePages(self, *args, **kwargs))
807
+ super().__init__(res)
808
+ return self
809
+
810
+ def dispersionAnalysis(self, by="source", **kwargs):
811
+ """ **EXPERIMENTAL**: Calculates how evenly the hits of the query are distributed across corpus metadata.
812
+
813
+ The groups of the chosen metadata dimension and their sizes are discovered automatically, including
814
+ groups without hits.
815
+
816
+ - **by** - metadata field or list of fields. `"source"` (default) groups by `corpusTitle`, ignoring trailing years; `"year"` groups `pubDate` by publication year; `"region"` uses `pubPlaceKey`.
817
+ - **regex** - optional regular expression removing a trailing suffix from metadata values, ending in `$`, e.g. `" [0-9]{4}$"`, which is the default for `by="source"`
818
+ - **scoreFunctions** - dict of R (!) measure functions (see `KorAPConnection.dispersionQuery`)
819
+ - **exactFrequencies** - by default, a sample fetched with `KorAPQuery.fetchSamplePages` or completely fetched metadata are reused; otherwise exact counts per group are requested. True requests exact counts even after a fetch; False permits sampling metadata.
820
+ - **searchHitsSampleLimit** - maximum metadata sample size (default = 20000)
821
+ - **seed** - seed for the random selection of result pages (default = 7)
822
+ - **verbose**
823
+ - **cacheAs** - path to an RDS file to keep the result in (see `KorAPConnection.collocationAnalysis`)
824
+
825
+ Returns:
826
+ DataFrame as described in `KorAPConnection.dispersionQuery`
827
+
828
+ Details:
829
+ Use a metadata-only query, which is the default of `KorAPConnection.corpusQuery`.
830
+ To compare the dispersion of several queries, use `KorAPConnection.dispersionQuery`.
831
+
832
+ Example:
833
+ ```
834
+ $ q = KorAPConnection().corpusQuery("Mückenplage", vc="pubDate in 2022")
835
+ $ df = q.dispersionAnalysis(by="region")
836
+ $ df[["query", "partition", "totalResults", "nParts", "DP", "DPnorm", "range"]]
837
+ query partition totalResults nParts DP DPnorm range
838
+ 1 Mückenplage region 18.0 7 0.062939 0.062939 3.0
839
+ ```
840
+ """
841
+ return _dispersion_to_pandas(_call_keeping_r_result(KorAPClient.dispersionAnalysis, self, by, **kwargs))