denckring-de-data 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- denckring_de_data-0.1.0/.gitignore +39 -0
- denckring_de_data-0.1.0/LICENSE +202 -0
- denckring_de_data-0.1.0/LICENSE-WIKIDATA +12 -0
- denckring_de_data-0.1.0/NOTICE +9 -0
- denckring_de_data-0.1.0/PKG-INFO +42 -0
- denckring_de_data-0.1.0/README.md +25 -0
- denckring_de_data-0.1.0/pyproject.toml +27 -0
- denckring_de_data-0.1.0/scripts/build_lexicon.py +194 -0
- denckring_de_data-0.1.0/src/denckring_de_data/__init__.py +148 -0
- denckring_de_data-0.1.0/src/denckring_de_data/data/metadata.json +9 -0
- denckring_de_data-0.1.0/src/denckring_de_data/data/nouns.txt.gz +0 -0
- denckring_de_data-0.1.0/src/denckring_de_data/data/words.txt.gz +0 -0
- denckring_de_data-0.1.0/src/denckring_de_data/py.typed +0 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
.venv/
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.pyc
|
|
4
|
+
.pytest_cache/
|
|
5
|
+
.mypy_cache/
|
|
6
|
+
.ruff_cache/
|
|
7
|
+
.hypothesis/
|
|
8
|
+
dist/
|
|
9
|
+
*.egg-info/
|
|
10
|
+
dist/
|
|
11
|
+
|
|
12
|
+
# Generated docs and site build
|
|
13
|
+
docs/gallery/
|
|
14
|
+
docs/index.md
|
|
15
|
+
docs/contributing.md
|
|
16
|
+
docs/changelog.md
|
|
17
|
+
site/
|
|
18
|
+
|
|
19
|
+
# The explorer keeps its own environment
|
|
20
|
+
apps/explorer/.venv/
|
|
21
|
+
|
|
22
|
+
# Local secrets and paths for the explorer
|
|
23
|
+
apps/explorer/.env
|
|
24
|
+
|
|
25
|
+
# Claude Code session state, including worktrees
|
|
26
|
+
.claude/
|
|
27
|
+
|
|
28
|
+
# Subagent-driven-development scratch workspace (ledgers, briefs, review packages).
|
|
29
|
+
.superpowers/
|
|
30
|
+
|
|
31
|
+
# Node modules for the explorer's browser reproductions (apps/explorer/tests/browser).
|
|
32
|
+
# The manifest and lockfile beside them ARE tracked, so `npm ci --prefix apps/explorer`
|
|
33
|
+
# reinstalls exactly what those scripts were verified against; the 19MB tree is not.
|
|
34
|
+
apps/explorer/node_modules/
|
|
35
|
+
|
|
36
|
+
# pytest-cov writes this on every --cov run; the report is CI's, not the repo's.
|
|
37
|
+
.coverage
|
|
38
|
+
.coverage.*
|
|
39
|
+
htmlcov/
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
https://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright [yyyy] [name of copyright owner]
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
https://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
The German lexicon in this distribution is derived from Wikidata Lexemes.
|
|
2
|
+
|
|
3
|
+
Wikidata's lexicographical data is released into the public domain under the
|
|
4
|
+
Creative Commons CC0 1.0 Universal Public Domain Dedication:
|
|
5
|
+
|
|
6
|
+
https://creativecommons.org/publicdomain/zero/1.0/
|
|
7
|
+
|
|
8
|
+
CC0 imposes no attribution and no share-alike requirement. This file records the
|
|
9
|
+
provenance because the project records provenance, not because the licence
|
|
10
|
+
compels it.
|
|
11
|
+
|
|
12
|
+
Regenerate the vendored files with scripts/build_lexicon.py.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
denckring-de-data
|
|
2
|
+
Copyright 2026 senzelden
|
|
3
|
+
|
|
4
|
+
The code in this distribution is licensed under the Apache License, Version 2.0.
|
|
5
|
+
See LICENSE.
|
|
6
|
+
|
|
7
|
+
The data in this distribution is derived from Wikidata Lexemes, dedicated to the
|
|
8
|
+
public domain under CC0 1.0 — see LICENSE-WIKIDATA. CC0 requires neither attribution
|
|
9
|
+
nor share-alike; the provenance is recorded because this project records provenance.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: denckring-de-data
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: German lexicon data for denckring: word membership and a noun list
|
|
5
|
+
Author-email: Dennis Senzel <dennis.senzel@posteo.com>
|
|
6
|
+
License-Expression: Apache-2.0 AND CC0-1.0
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
License-File: LICENSE-WIKIDATA
|
|
9
|
+
License-File: NOTICE
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: Natural Language :: German
|
|
12
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
13
|
+
Classifier: Typing :: Typed
|
|
14
|
+
Requires-Python: >=3.11
|
|
15
|
+
Requires-Dist: denckring==0.1.0
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
# denckring-de-data
|
|
19
|
+
|
|
20
|
+
German lexicon data for [denckring](https://github.com/senzelden/denckring):
|
|
21
|
+
word membership and an ordered noun list.
|
|
22
|
+
|
|
23
|
+
```console
|
|
24
|
+
pip install denckring[de]
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Installing this gives the German pack `lexicon.words` and `lexicon.nouns`, which
|
|
28
|
+
is what `charade`, `semordnilap`, `word_square`, `n_plus_7` and `s_plus_7` need
|
|
29
|
+
in order to run in German. Without it those procedures raise
|
|
30
|
+
`MissingCapability` for `de`, exactly as they do for any unmet capability.
|
|
31
|
+
|
|
32
|
+
The data is derived from Wikidata Lexemes and is CC0 — no attribution, no
|
|
33
|
+
share-alike. See `LICENSE-WIKIDATA` for provenance. Regenerate it with
|
|
34
|
+
`scripts/build_lexicon.py`; the vendored files are reproducible from that
|
|
35
|
+
script alone.
|
|
36
|
+
|
|
37
|
+
Umlauts and ß are preserved rather than folded. `Bär` and `Bar` are different
|
|
38
|
+
words, and whether they should be treated alike is a decision ADR 0009 leaves to
|
|
39
|
+
the procedure, not to the lexicon.
|
|
40
|
+
|
|
41
|
+
Membership is casefolded, which deliberately merges ß and ss spellings (e.g.
|
|
42
|
+
Maßen/Massen); the noun list is not casefolded and keeps ß intact.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# denckring-de-data
|
|
2
|
+
|
|
3
|
+
German lexicon data for [denckring](https://github.com/senzelden/denckring):
|
|
4
|
+
word membership and an ordered noun list.
|
|
5
|
+
|
|
6
|
+
```console
|
|
7
|
+
pip install denckring[de]
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
Installing this gives the German pack `lexicon.words` and `lexicon.nouns`, which
|
|
11
|
+
is what `charade`, `semordnilap`, `word_square`, `n_plus_7` and `s_plus_7` need
|
|
12
|
+
in order to run in German. Without it those procedures raise
|
|
13
|
+
`MissingCapability` for `de`, exactly as they do for any unmet capability.
|
|
14
|
+
|
|
15
|
+
The data is derived from Wikidata Lexemes and is CC0 — no attribution, no
|
|
16
|
+
share-alike. See `LICENSE-WIKIDATA` for provenance. Regenerate it with
|
|
17
|
+
`scripts/build_lexicon.py`; the vendored files are reproducible from that
|
|
18
|
+
script alone.
|
|
19
|
+
|
|
20
|
+
Umlauts and ß are preserved rather than folded. `Bär` and `Bar` are different
|
|
21
|
+
words, and whether they should be treated alike is a decision ADR 0009 leaves to
|
|
22
|
+
the procedure, not to the lexicon.
|
|
23
|
+
|
|
24
|
+
Membership is casefolded, which deliberately merges ß and ss spellings (e.g.
|
|
25
|
+
Maßen/Massen); the noun list is not casefolded and keeps ß intact.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "denckring-de-data"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "German lexicon data for denckring: word membership and a noun list"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.11"
|
|
7
|
+
license = "Apache-2.0 AND CC0-1.0"
|
|
8
|
+
license-files = ["LICENSE", "NOTICE", "LICENSE-WIKIDATA"]
|
|
9
|
+
authors = [{ name = "Dennis Senzel", email = "dennis.senzel@posteo.com" }]
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Intended Audience :: Science/Research",
|
|
12
|
+
"Natural Language :: German",
|
|
13
|
+
"Topic :: Text Processing :: Linguistic",
|
|
14
|
+
"Typing :: Typed",
|
|
15
|
+
]
|
|
16
|
+
# Version-locked, for the reason denckring-en-data records.
|
|
17
|
+
dependencies = ["denckring==0.1.0"]
|
|
18
|
+
|
|
19
|
+
[project.entry-points."denckring.lang"]
|
|
20
|
+
de = "denckring_de_data:pack"
|
|
21
|
+
|
|
22
|
+
[build-system]
|
|
23
|
+
requires = ["hatchling"]
|
|
24
|
+
build-backend = "hatchling.build"
|
|
25
|
+
|
|
26
|
+
[tool.hatch.build.targets.wheel]
|
|
27
|
+
packages = ["src/denckring_de_data"]
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"""Regenerate the vendored German lexicon from Wikidata Lexemes.
|
|
2
|
+
|
|
3
|
+
Committed so the data files are reproducible and diffable. CMUdict in the
|
|
4
|
+
English package is not, and a second opaque blob is not worth adding.
|
|
5
|
+
|
|
6
|
+
python scripts/build_lexicon.py
|
|
7
|
+
|
|
8
|
+
Wikidata Lexemes are CC0: no attribution, no share-alike. See LICENSE-WIKIDATA.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import gzip
|
|
14
|
+
import http.client
|
|
15
|
+
import json
|
|
16
|
+
import re
|
|
17
|
+
import sys
|
|
18
|
+
import time
|
|
19
|
+
import urllib.error
|
|
20
|
+
import urllib.parse
|
|
21
|
+
import urllib.request
|
|
22
|
+
from datetime import UTC, datetime
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
#: Network failure modes seen against the shared Wikidata Query Service:
|
|
26
|
+
#: HTTP errors, connection drops, and truncated chunked responses that leave
|
|
27
|
+
#: json.load with a malformed tail.
|
|
28
|
+
NETWORK_ERRORS = (
|
|
29
|
+
urllib.error.HTTPError,
|
|
30
|
+
urllib.error.URLError,
|
|
31
|
+
http.client.HTTPException,
|
|
32
|
+
json.JSONDecodeError,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
ENDPOINT = "https://query.wikidata.org/sparql"
|
|
36
|
+
USER_AGENT = "denckring-de-data/0.1 (https://github.com/senzelden/denckring)"
|
|
37
|
+
DATA = Path(__file__).resolve().parents[1] / "src" / "denckring_de_data" / "data"
|
|
38
|
+
#: Beside the .gz files: the generation date and the two entry counts, so a
|
|
39
|
+
#: silent corpus change fails a test loudly instead of drifting unnoticed
|
|
40
|
+
#: (the gzip bytes alone do not diff cleanly enough to catch that by eye).
|
|
41
|
+
METADATA = DATA / "metadata.json"
|
|
42
|
+
|
|
43
|
+
#: German (Q188), noun (Q1084).
|
|
44
|
+
NOUN_LEMMAS = """
|
|
45
|
+
SELECT DISTINCT ?lemma WHERE {
|
|
46
|
+
?l dct:language wd:Q188 ; wikibase:lexicalCategory wd:Q1084 ; wikibase:lemma ?lemma .
|
|
47
|
+
}
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
#: Every lexical category used by a German lexeme. Queried per-category below
|
|
51
|
+
#: because the single-query ALL_FORMS shape times out on the shared endpoint.
|
|
52
|
+
CATEGORIES = """
|
|
53
|
+
SELECT DISTINCT ?cat WHERE {
|
|
54
|
+
?l dct:language wd:Q188 ; wikibase:lexicalCategory ?cat .
|
|
55
|
+
}
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
#: Every inflected form of German lexemes in one lexical category. Not nouns
|
|
59
|
+
#: alone: `charade` and `word_square` ask "is this a word", and a noun-only
|
|
60
|
+
#: oracle would reject `singen` and `rot`. The endpoint drops large single
|
|
61
|
+
#: queries, so the caller unions this per category (and paginates within a
|
|
62
|
+
#: category if even that is too large).
|
|
63
|
+
FORMS_BY_CATEGORY = """
|
|
64
|
+
SELECT DISTINCT ?rep WHERE {{
|
|
65
|
+
?l dct:language wd:Q188 ; wikibase:lexicalCategory <{category}> ; ontolex:lexicalForm ?f .
|
|
66
|
+
?f ontolex:representation ?rep .
|
|
67
|
+
}}
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
#: Same query, paginated: for categories too large even on their own.
|
|
71
|
+
FORMS_BY_CATEGORY_PAGE = """
|
|
72
|
+
SELECT DISTINCT ?rep WHERE {{
|
|
73
|
+
?l dct:language wd:Q188 ; wikibase:lexicalCategory <{category}> ; ontolex:lexicalForm ?f .
|
|
74
|
+
?f ontolex:representation ?rep .
|
|
75
|
+
}}
|
|
76
|
+
ORDER BY ?rep
|
|
77
|
+
LIMIT {limit} OFFSET {offset}
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
#: A noun the tokeniser returns whole: no spaces, no hyphens, no digits. ADR
|
|
81
|
+
#: 0015's rule for English, applied unchanged. Umlauts and ß are kept — folding
|
|
82
|
+
#: them here would collide `Bär` with `Bar`, and ADR 0009 makes folding a
|
|
83
|
+
#: procedure parameter rather than a property of the data.
|
|
84
|
+
SINGLE_TOKEN = re.compile(r"[A-Za-zÄÖÜäöüß]{2,}")
|
|
85
|
+
|
|
86
|
+
PAGE_SIZE = 100_000
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def query(sparql: str, attempts: int = 4, wait_base: float = 8.0) -> list[str]:
|
|
90
|
+
"""Run one SPARQL query, retrying with backoff. Be a polite client:
|
|
91
|
+
this hits the shared public Wikidata Query Service."""
|
|
92
|
+
url = f"{ENDPOINT}?{urllib.parse.urlencode({'query': sparql})}"
|
|
93
|
+
request = urllib.request.Request(
|
|
94
|
+
url, headers={"User-Agent": USER_AGENT, "Accept": "application/sparql-results+json"}
|
|
95
|
+
)
|
|
96
|
+
last_exc: Exception | None = None
|
|
97
|
+
for attempt in range(attempts):
|
|
98
|
+
try:
|
|
99
|
+
with urllib.request.urlopen(request, timeout=600) as response:
|
|
100
|
+
payload = json.load(response)
|
|
101
|
+
rows = payload["results"]["bindings"]
|
|
102
|
+
return [next(iter(row.values()))["value"] for row in rows]
|
|
103
|
+
except NETWORK_ERRORS as exc:
|
|
104
|
+
last_exc = exc
|
|
105
|
+
if attempt == attempts - 1:
|
|
106
|
+
break
|
|
107
|
+
wait = wait_base * (attempt + 1)
|
|
108
|
+
print(f" retry after {exc!r}, waiting {wait:.0f}s", file=sys.stderr)
|
|
109
|
+
time.sleep(wait)
|
|
110
|
+
assert last_exc is not None
|
|
111
|
+
raise last_exc
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def forms_for_category(category: str) -> list[str]:
|
|
115
|
+
"""All representations for one lexical category, falling back to
|
|
116
|
+
LIMIT/OFFSET pagination if the category is too large for one response."""
|
|
117
|
+
try:
|
|
118
|
+
return query(FORMS_BY_CATEGORY.format(category=category), attempts=2)
|
|
119
|
+
except NETWORK_ERRORS:
|
|
120
|
+
print(f" {category}: paginating", file=sys.stderr)
|
|
121
|
+
|
|
122
|
+
results: list[str] = []
|
|
123
|
+
offset = 0
|
|
124
|
+
while True:
|
|
125
|
+
page = query(
|
|
126
|
+
FORMS_BY_CATEGORY_PAGE.format(category=category, limit=PAGE_SIZE, offset=offset)
|
|
127
|
+
)
|
|
128
|
+
results.extend(page)
|
|
129
|
+
if len(page) < PAGE_SIZE:
|
|
130
|
+
return results
|
|
131
|
+
offset += PAGE_SIZE
|
|
132
|
+
time.sleep(2)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def all_forms() -> set[str]:
|
|
136
|
+
#: casefold(), not lower(): German has no reliable native uppercase ß, so
|
|
137
|
+
#: casefold() maps ß -> "ss" (str.lower() would keep ß but then miss an
|
|
138
|
+
#: all-caps "STRASSE" and split Straße/Strasse into unrelated words,
|
|
139
|
+
#: which is worse). This deliberately merges Swiss and German spellings
|
|
140
|
+
#: of the same word (Anstoßkreis/Anstosskreis) in words.txt.gz. Measured
|
|
141
|
+
#: impact: 110 collision keys covering 222 of 184,040 noun lemmas
|
|
142
|
+
#: (0.12%), all either ß/ss spelling variants or acronym case variants
|
|
143
|
+
#: (AIDS/Aids) - correct behaviour for a membership oracle, not lossy.
|
|
144
|
+
#: nouns.txt.gz keeps ß intact (see main()); only the membership set is
|
|
145
|
+
#: folded. Any consumer of words.txt.gz must casefold its query input
|
|
146
|
+
#: too, or lookups for ß-containing words will silently miss.
|
|
147
|
+
forms: set[str] = set()
|
|
148
|
+
for category in query(CATEGORIES):
|
|
149
|
+
rows = forms_for_category(category)
|
|
150
|
+
print(f" {category}: {len(rows):,} forms", file=sys.stderr)
|
|
151
|
+
forms.update(w.casefold() for w in rows if SINGLE_TOKEN.fullmatch(w))
|
|
152
|
+
time.sleep(2)
|
|
153
|
+
return forms
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def write(path: Path, entries: list[str]) -> int:
|
|
157
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
158
|
+
payload = "\n".join(entries).encode("utf-8")
|
|
159
|
+
# mtime=0: gzip embeds a timestamp by default, so two runs over identical
|
|
160
|
+
# content would otherwise produce different bytes and `git diff` would
|
|
161
|
+
# show "Binary files differ" even when nothing changed.
|
|
162
|
+
path.write_bytes(gzip.compress(payload, mtime=0))
|
|
163
|
+
print(f" {path.name}: {len(entries):,} entries", file=sys.stderr)
|
|
164
|
+
return len(entries)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def write_metadata(noun_count: int, word_count: int) -> None:
|
|
168
|
+
"""Record beside the .gz files what generated them and how many entries
|
|
169
|
+
they hold, so a test can assert the shipped files still match and a
|
|
170
|
+
silent corpus change fails loudly instead of drifting unnoticed."""
|
|
171
|
+
metadata = {
|
|
172
|
+
"generated": datetime.now(UTC).strftime("%Y-%m-%d"),
|
|
173
|
+
"source": "Wikidata Lexemes (Q188, German), CC0 - see LICENSE-WIKIDATA and this script",
|
|
174
|
+
"counts": {"nouns.txt.gz": noun_count, "words.txt.gz": word_count},
|
|
175
|
+
}
|
|
176
|
+
METADATA.write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
177
|
+
print(f" {METADATA.name}: {metadata}", file=sys.stderr)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def main() -> int:
|
|
181
|
+
lemmas = [w for w in query(NOUN_LEMMAS) if SINGLE_TOKEN.fullmatch(w) and w[:1].isupper()]
|
|
182
|
+
nouns = sorted(set(lemmas))
|
|
183
|
+
noun_count = write(DATA / "nouns.txt.gz", nouns)
|
|
184
|
+
|
|
185
|
+
forms = all_forms()
|
|
186
|
+
forms.update(w.casefold() for w in nouns) # same ß -> ss folding, see all_forms()
|
|
187
|
+
word_count = write(DATA / "words.txt.gz", sorted(forms))
|
|
188
|
+
|
|
189
|
+
write_metadata(noun_count, word_count)
|
|
190
|
+
return 0
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
if __name__ == "__main__":
|
|
194
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""German lexicon data for denckring.
|
|
2
|
+
|
|
3
|
+
Installing this package gives the German pack the two lexical capabilities the
|
|
4
|
+
English pack has had since ADR 0015: `lexicon.words` and `lexicon.nouns`.
|
|
5
|
+
Nothing branches on whether it is present — the same procedures answer the same
|
|
6
|
+
calls, in a second language.
|
|
7
|
+
|
|
8
|
+
It also carries the *only* `de` language-pack registration, and since ADR 0030
|
|
9
|
+
that is a load-bearing fact rather than an incidental one. `denckring-de-wiktionary`
|
|
10
|
+
holds German pronunciations and glosses under CC BY-SA, which ADR 0013 forbids
|
|
11
|
+
merging into this CC0 distribution — and `denckring/lang/__init__.py` refuses two
|
|
12
|
+
entry points claiming one language, so it cannot register `de` for itself either.
|
|
13
|
+
The entry point therefore resolves to `pack()` below rather than to a class, and
|
|
14
|
+
that one function is the whole of the seam.
|
|
15
|
+
|
|
16
|
+
The data here is Wikidata Lexemes, CC0. See LICENSE-WIKIDATA.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import gzip
|
|
22
|
+
from functools import lru_cache
|
|
23
|
+
from importlib.resources import files
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import ClassVar
|
|
26
|
+
|
|
27
|
+
from denckring.lang.base import (
|
|
28
|
+
ALPHABET,
|
|
29
|
+
FOLD_DIACRITICS,
|
|
30
|
+
LETTER_SHAPES,
|
|
31
|
+
NOUNS,
|
|
32
|
+
SYLLABLES_HEURISTIC,
|
|
33
|
+
TOKENS,
|
|
34
|
+
WORDS,
|
|
35
|
+
)
|
|
36
|
+
from denckring.lang.de import GermanPack
|
|
37
|
+
|
|
38
|
+
NOUNS_PATH = Path(str(files("denckring_de_data") / "data" / "nouns.txt.gz"))
|
|
39
|
+
WORDS_PATH = Path(str(files("denckring_de_data") / "data" / "words.txt.gz"))
|
|
40
|
+
|
|
41
|
+
__version__ = "0.1.0"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _read(path: Path) -> tuple[str, ...]:
|
|
45
|
+
"""Every line of a gzipped list.
|
|
46
|
+
|
|
47
|
+
A truncated file raises rather than returning a short list: a short noun
|
|
48
|
+
list makes N+7 quietly wrong, and a wrong answer is worse than an exception.
|
|
49
|
+
"""
|
|
50
|
+
with gzip.open(path, "rt", encoding="utf-8") as handle:
|
|
51
|
+
return tuple(line for line in handle.read().split("\n") if line)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@lru_cache(maxsize=1)
|
|
55
|
+
def noun_list() -> tuple[str, ...]:
|
|
56
|
+
"""Every noun lemma, in dictionary order, capitalised as German writes them."""
|
|
57
|
+
return _read(NOUNS_PATH)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@lru_cache(maxsize=1)
|
|
61
|
+
def noun_positions() -> dict[str, int]:
|
|
62
|
+
return {word.casefold(): index for index, word in enumerate(noun_list())}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@lru_cache(maxsize=1)
|
|
66
|
+
def known_words() -> frozenset[str]:
|
|
67
|
+
"""Membership, from every inflected form of every German lexeme.
|
|
68
|
+
|
|
69
|
+
Deliberately broad, in a way callers inherit: it answers "could this be a
|
|
70
|
+
German word" rather than "is this in a dictionary of standard German", and
|
|
71
|
+
procedures resting on it inherit that. The same caveat ADR 0015 recorded for
|
|
72
|
+
English.
|
|
73
|
+
"""
|
|
74
|
+
return frozenset(_read(WORDS_PATH))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class GermanDataPack(GermanPack):
|
|
78
|
+
"""German with a lexicon behind it."""
|
|
79
|
+
|
|
80
|
+
capabilities: ClassVar[frozenset[str]] = frozenset(
|
|
81
|
+
{TOKENS, ALPHABET, FOLD_DIACRITICS, LETTER_SHAPES, SYLLABLES_HEURISTIC, NOUNS, WORDS}
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
def is_word(self, word: str) -> bool:
|
|
85
|
+
return self._lemma(word) in known_words()
|
|
86
|
+
|
|
87
|
+
def nouns(self) -> tuple[str, ...]:
|
|
88
|
+
return noun_list()
|
|
89
|
+
|
|
90
|
+
def noun_index(self, word: str) -> int | None:
|
|
91
|
+
return noun_positions().get(self._lemma(word))
|
|
92
|
+
|
|
93
|
+
@staticmethod
|
|
94
|
+
def _lemma(word: str) -> str:
|
|
95
|
+
"""Casefold, and keep umlauts and ß.
|
|
96
|
+
|
|
97
|
+
English strips to ASCII here. German must not: `fold_diacritics` would
|
|
98
|
+
collide `Bär` with `Bar`, and ADR 0009 makes folding a parameter of the
|
|
99
|
+
procedure rather than a property of the lexicon.
|
|
100
|
+
"""
|
|
101
|
+
return "".join(ch for ch in word.casefold() if ch.isalpha())
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def pack() -> GermanPack:
|
|
105
|
+
"""The best German pack this install can supply. The `de` entry point.
|
|
106
|
+
|
|
107
|
+
A factory rather than a class, because three distributions carry German data
|
|
108
|
+
under three licences and only one of them may register the language.
|
|
109
|
+
`denckring/lang/__init__.py` raises `DuplicatePack` for a second `de` entry
|
|
110
|
+
point — deliberately, so no installer has to choose between packs — and ADR
|
|
111
|
+
0013 forbids merging their licences. A factory satisfies both: one entry
|
|
112
|
+
point, and the richest pack whose data is installed wins.
|
|
113
|
+
|
|
114
|
+
`entry.load()()` is what the registry calls, so a function and a class are
|
|
115
|
+
interchangeable there; nothing in core learns that German is special.
|
|
116
|
+
|
|
117
|
+
**Four combinations, each naming a class rather than composing one at
|
|
118
|
+
runtime.** ADR 0030 replaced a computed-capabilities probe with a named
|
|
119
|
+
factory once already, and its reason still holds: a reader should be able to
|
|
120
|
+
see what a class carries without running it. Two optional distributions made
|
|
121
|
+
two branches; a third (ADR 0038) makes four, and four named branches are
|
|
122
|
+
still cheaper to read than one clever line.
|
|
123
|
+
|
|
124
|
+
The subclasses are imported here rather than at module scope because this
|
|
125
|
+
distribution depends on neither of them — the dependencies run the other way.
|
|
126
|
+
"""
|
|
127
|
+
try:
|
|
128
|
+
from denckring_de_frequency import (
|
|
129
|
+
GermanFrequencyPack,
|
|
130
|
+
GermanWiktionaryFrequencyPack,
|
|
131
|
+
)
|
|
132
|
+
except ImportError:
|
|
133
|
+
# No frequency data. `anagram` can still check in German and will raise
|
|
134
|
+
# `MissingCapability` naming `lexicon.graded_words` if asked to generate,
|
|
135
|
+
# which is the honest failure rather than an unranked pile of covers.
|
|
136
|
+
try:
|
|
137
|
+
from denckring_de_wiktionary import GermanWiktionaryPack
|
|
138
|
+
except ImportError:
|
|
139
|
+
return GermanDataPack()
|
|
140
|
+
return GermanWiktionaryPack()
|
|
141
|
+
try:
|
|
142
|
+
import denckring_de_wiktionary # noqa: F401
|
|
143
|
+
except ImportError:
|
|
144
|
+
# Not installed. A procedure wanting `phonemes` will raise
|
|
145
|
+
# `MissingCapability` naming it, which is the honest failure rather than
|
|
146
|
+
# a guessed pronunciation.
|
|
147
|
+
return GermanFrequencyPack()
|
|
148
|
+
return GermanWiktionaryFrequencyPack()
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"counts": {
|
|
3
|
+
"nouns.txt.gz": 184040,
|
|
4
|
+
"words.txt.gz": 668580
|
|
5
|
+
},
|
|
6
|
+
"generated": "2026-08-16",
|
|
7
|
+
"note": "This metadata file was added after the .gz files already existed, so 'generated' is not from a fresh run of build_lexicon.py. It is recovered from the mtime field embedded in each file's gzip header (nouns.txt.gz: 2026-08-16T23:19:58Z, words.txt.gz: 2026-08-16T23:27:34Z), which predates this fix's mtime=0 change. Future regenerations write this field from the actual run date.",
|
|
8
|
+
"source": "Wikidata Lexemes (Q188, German), CC0 - see LICENSE-WIKIDATA and scripts/build_lexicon.py"
|
|
9
|
+
}
|
|
Binary file
|
|
Binary file
|
|
File without changes
|