wikipron 2.0.0__tar.gz → 2.2.0__tar.gz

{wikipron-2.0.0 → wikipron-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wikipron
-Version: 2.0.0
+Version: 2.2.0
 Summary: Scraping grapheme-to-phoneme data from Wiktionary
 Author-email: WikiPron authors <kylebgorman@gmail.com>
 License: Apache 2.0
@@ -30,16 +30,6 @@ Requires-Dist: regex>=2019.12.9
 Requires-Dist: requests>=2.32.3
 Requires-Dist: segments<3,>=2.2.0
 Requires-Dist: unicodedataplus>=15.0.0
-Provides-Extra: dev
-Requires-Dist: black; extra == "dev"
-Requires-Dist: build; extra == "dev"
-Requires-Dist: flake8; extra == "dev"
-Requires-Dist: mypy; extra == "dev"
-Requires-Dist: pytest; extra == "dev"
-Requires-Dist: setuptools; extra == "dev"
-Requires-Dist: twine; extra == "dev"
-Requires-Dist: types-lxml; extra == "dev"
-Requires-Dist: types-requests; extra == "dev"
 Dynamic: license-file
 
 # WikiPron
@@ -58,7 +48,7 @@ dictionaries mined using this tool.
 
 -   [Command-line tool](#command-line-tool)
 -   [Python API](#python-api)
--   [Data](#data)
+-   [Data](#data) of over 5 million word pronunciation pairs from 348 languages
 -   [Models](#models)
 -   [Development](#development)
 
@@ -174,9 +164,7 @@ for word, pron in wikipron.scrape(config):
 
 ## Data
 
-We also make available [a database of over 3 million word/pronunciation
-pairs](https://github.com/CUNY-CL/wikipron/tree/master/data) mined using
-WikiPron.
+We also make available [a database](https://github.com/CUNY-CL/wikipron/tree/master/data) mined using WikiPron.
 
 ## Models
 

{wikipron-2.0.0 → wikipron-2.2.0}/README.md

@@ -14,7 +14,7 @@ dictionaries mined using this tool.
 
 -   [Command-line tool](#command-line-tool)
 -   [Python API](#python-api)
--   [Data](#data)
+-   [Data](#data) of over 5 million word pronunciation pairs from 348 languages
 -   [Models](#models)
 -   [Development](#development)
 
@@ -130,9 +130,7 @@ for word, pron in wikipron.scrape(config):
 
 ## Data
 
-We also make available [a database of over 3 million word/pronunciation
-pairs](https://github.com/CUNY-CL/wikipron/tree/master/data) mined using
-WikiPron.
+We also make available [a database](https://github.com/CUNY-CL/wikipron/tree/master/data) mined using WikiPron.
 
 ## Models
 

{wikipron-2.0.0 → wikipron-2.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "wikipron"
-version = "2.0.0"
+version = "2.2.0"
 description = "Scraping grapheme-to-phoneme data from Wiktionary"
 readme = "README.md"
 requires-python = ">= 3.11"
@@ -45,10 +45,9 @@ classifiers = [
     "Topic :: Text Processing :: Linguistic",
 ]
 
-[project.optional-dependencies]
+[dependency-groups]
 dev = [
     'black',
-    'build',
     'flake8',
     'mypy',
     'pytest',

{wikipron-2.0.0 → wikipron-2.2.0}/src/wikipron/config.py

@@ -30,11 +30,24 @@ _PRON_XPATH_SELECTOR_TEMPLATE = """
   {dialect_selector}
 ]
 """
+# A pronunciation <li>/<p> is kept for a dialect if a matching accent label
+# (an <a> link, e.g. "US" -> American English) sits on the line itself or on an
+# enclosing <li> (Wiktionary nests accent-specific prons under an accent
+# header), or -- the "no accent" fallback -- if the line carries no linked
+# accent label of its own (ignoring labels inside nested <li> sub-variants) and
+# none on any ancestor <li>. That fallback keeps general prons whose only
+# labels are non-accent qualifiers ("strong form", "weak form", "colloquial"),
+# which are plain text rather than links. See GH-591 for the descendant-axis
+# (.//) requirement.
 _DIALECT_XPATH_SELECTOR_TEMPLATE = (
     "and\n"
-    '  (span[contains(@class, "ib-content")]//a[{dialects_text}]\n'
-    '   or span[contains(@class, "ib-content") and ({dialects_text})]\n'
-    '   or count(span[contains(@class, "ib-content")]) = 0)'
+    '  (.//span[contains(@class, "ib-content")]//a[{dialects_text}]\n'
+    '   or .//span[contains(@class, "ib-content") and ({dialects_text})]\n'
+    '   or ancestor::li//span[contains(@class, "ib-content")]//a[{dialects_text}]\n'  # noqa: E501
+    '   or ancestor::li//span[contains(@class, "ib-content") and ({dialects_text})]\n'  # noqa: E501
+    '   or (count(.//span[contains(@class, "ib-content")][.//a])\n'
+    '       - count(.//li//span[contains(@class, "ib-content")][.//a]) = 0\n'
+    '       and count(ancestor::li//span[contains(@class, "ib-content")][.//a]) = 0))'  # noqa: E501
 )
 _PHONEMES_REGEX = r"/(.+?)/"
 _PHONES_REGEX = r"\[(.+?)\]"

{wikipron-2.0.0 → wikipron-2.2.0}/src/wikipron/extract/default.py

@@ -12,7 +12,11 @@ if typing.TYPE_CHECKING:
     from wikipron.typing import Iterator, WordPronPair
 
 
-IPA_XPATH_SELECTOR = './/span[contains(@class, "IPA")]'
+# Direct-child (not descendant) so a general-outer <li> does not also scoop the
+# IPA of accent variants nested in sub-<li>s; each variant is matched on its
+# own line. Matches the base selector, which likewise keys on a direct-child
+# IPA.
+IPA_XPATH_SELECTOR = 'span[contains(@class, "IPA")]'
 
 
 def _yield_phn(request: HTMLResponse, config: "Config") -> "Iterator[str]":

{wikipron-2.0.0 → wikipron-2.2.0}/src/wikipron/extract/eng.py

@@ -14,7 +14,11 @@ if typing.TYPE_CHECKING:
     from wikipron.typing import Iterator, WordPronPair
 
 
-IPA_XPATH_SELECTOR = './/span[contains(@class, "IPA")]'
+# Direct-child (not descendant) so a general-outer <li> does not also scoop the
+# IPA of accent variants nested in sub-<li>s; each variant is matched on its
+# own line. Matches the base selector, which likewise keys on a direct-child
+# IPA.
+IPA_XPATH_SELECTOR = 'span[contains(@class, "IPA")]'
 
 
 def yield_eng_pron(request: HTMLResponse, config: "Config") -> "Iterator[str]":

wikipron-2.2.0/src/wikipron/extract/tha.py

@@ -0,0 +1,33 @@
+"""Word and pron extraction for Thai."""
+
+import itertools
+import typing
+
+from wikipron.html_utils import HTMLResponse
+
+from wikipron.extract.default import yield_pron
+
+if typing.TYPE_CHECKING:
+    from wikipron.config import Config
+    from wikipron.typing import Iterator, WordPronPair
+
+
+# Thai pronunciations now render inside a <table>: the IPA <span> lives in a
+# <td> whose row's <th> carries the pronunciation-appendix/phonology anchor.
+# Restrict to those rows so we scrape Thai only (other languages render
+# Thai-script IPA on the same page).
+_THAI_IPA_XPATH_SELECTOR = (
+    "//tr[.//sup/a["
+    '@title = "Appendix:Thai pronunciation" or '
+    '@title = "wikipedia:Thai phonology" or '
+    '@title = "w:Thai language"]]'
+    '//span[contains(@class, "IPA")]'
+)
+
+
+def extract_word_pron_thai(
+    word: str, request: HTMLResponse, config: "Config"
+) -> "Iterator[WordPronPair]":
+    words = itertools.repeat(word)
+    prons = yield_pron(request.html, _THAI_IPA_XPATH_SELECTOR, config)
+    yield from zip(words, prons)

{wikipron-2.0.0 → wikipron-2.2.0}/src/wikipron/extract/vie.py

@@ -22,8 +22,8 @@ _PRON_XPATH_SELECTOR_TEMPLATE = """
 
 _DIALECT_XPATH_SELECTOR_TEMPLATE = (
     "and\n"
-    '  (span[contains(@class, "ib-content")]//a[{dialects_text}]\n'
-    '   or count(span[contains(@class, "ib-content")]) = 0)'
+    '  (.//span[contains(@class, "ib-content")]//a[{dialects_text}]\n'
+    '   or count(.//span[contains(@class, "ib-content")]) = 0)'
 )
 
 if typing.TYPE_CHECKING:

{wikipron-2.0.0 → wikipron-2.2.0}/src/wikipron/languagecodes.py

@@ -326,6 +326,10 @@ LANGUAGE_CODES = {
     "bjb": "Barngarla",
     "barngarla": "Barngarla",
     "banggarla": "Barngarla",
+    # Blackfoot. Would be "Siksika" in ISO 639-3.
+    "bla": "Blackfoot",
+    "blackfoot": "Blackfoot",
+    "siksika": "Blackfoot",
     # Carpathian Rusyn. Would be "Rusyn" in ISO 639-3.
     "rue": "Carpathian Rusyn",
     "rusyn": "Carpathian Rusyn",
@@ -497,6 +501,9 @@ LANGUAGE_CODES = {
     "shi": "Tashelhit",
     "tashelhit": "Tashelhit",
     "tachelhit": "Tashelhit",
+    # Tewa. Would be "Tewa (USA)" in ISO 639-3.
+    "tew": "Tewa",
+    "tewa": "Tewa",
     # Tokelauan. Would be "Tokelau" in ISO 639-3.
     "tkl": "Tokelauan",
     "tokelauan": "Tokelauan",

{wikipron-2.0.0 → wikipron-2.2.0}/src/wikipron.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wikipron
-Version: 2.0.0
+Version: 2.2.0
 Summary: Scraping grapheme-to-phoneme data from Wiktionary
 Author-email: WikiPron authors <kylebgorman@gmail.com>
 License: Apache 2.0
@@ -30,16 +30,6 @@ Requires-Dist: regex>=2019.12.9
 Requires-Dist: requests>=2.32.3
 Requires-Dist: segments<3,>=2.2.0
 Requires-Dist: unicodedataplus>=15.0.0
-Provides-Extra: dev
-Requires-Dist: black; extra == "dev"
-Requires-Dist: build; extra == "dev"
-Requires-Dist: flake8; extra == "dev"
-Requires-Dist: mypy; extra == "dev"
-Requires-Dist: pytest; extra == "dev"
-Requires-Dist: setuptools; extra == "dev"
-Requires-Dist: twine; extra == "dev"
-Requires-Dist: types-lxml; extra == "dev"
-Requires-Dist: types-requests; extra == "dev"
 Dynamic: license-file
 
 # WikiPron
@@ -58,7 +48,7 @@ dictionaries mined using this tool.
 
 -   [Command-line tool](#command-line-tool)
 -   [Python API](#python-api)
--   [Data](#data)
+-   [Data](#data) of over 5 million word pronunciation pairs from 348 languages
 -   [Models](#models)
 -   [Development](#development)
 
@@ -174,9 +164,7 @@ for word, pron in wikipron.scrape(config):
 
 ## Data
 
-We also make available [a database of over 3 million word/pronunciation
-pairs](https://github.com/CUNY-CL/wikipron/tree/master/data) mined using
-WikiPron.
+We also make available [a database](https://github.com/CUNY-CL/wikipron/tree/master/data) mined using WikiPron.
 
 ## Models
 

{wikipron-2.0.0 → wikipron-2.2.0}/src/wikipron.egg-info/requires.txt

@@ -10,14 +10,3 @@ unicodedataplus>=15.0.0
 
 [:sys_platform == "linux"]
 pynini>=2.1.0
-
-[dev]
-black
-build
-flake8
-mypy
-pytest
-setuptools
-twine
-types-lxml
-types-requests

wikipron-2.0.0/src/wikipron/extract/tha.py

@@ -1,20 +0,0 @@
-"""Word and pron extraction for Thai."""
-
-import itertools
-import typing
-
-from wikipron.html_utils import HTMLResponse
-
-from wikipron.extract.default import yield_pron, IPA_XPATH_SELECTOR
-
-if typing.TYPE_CHECKING:
-    from wikipron.config import Config
-    from wikipron.typing import Iterator, WordPronPair
-
-
-def extract_word_pron_thai(
-    word: str, request: HTMLResponse, config: "Config"
-) -> "Iterator[WordPronPair]":
-    words = itertools.repeat(word)
-    prons = yield_pron(request.html, IPA_XPATH_SELECTOR, config)
-    yield from zip(words, prons)