numtag 0.0.1a2__tar.gz

numtag-0.0.1a2/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2025 TigreGotico
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

numtag-0.0.1a2/PKG-INFO

@@ -0,0 +1,37 @@
+Metadata-Version: 2.4
+Name: numtag
+Version: 0.0.1a2
+Summary: Find every number in a text: exact values as digit strings, kinds and character spans.
+Author: TigreGotico
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/TigreGotico/numtag
+Project-URL: Repository, https://github.com/TigreGotico/numtag
+Project-URL: Issues, https://github.com/TigreGotico/numtag/issues
+Keywords: numbers,numerals,nlp,tagger,parser
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Dynamic: license-file
+
+# numtag
+
+numtag finds every number in a text. For each number it gives the exact words, their character offsets, the value and the kind.
+
+```python
+from numtag import tag_numbers
+
+tag_numbers("comprou duas dúzias de ovos e um milhão de euros", "pt-PT")
+# [NumberMention(char_span=(8, 19), text='duas dúzias', value='24', kind=Kind.CARDINAL, ...),
+#  NumberMention(char_span=(30, 39), text='um milhão', value='1000000', kind=Kind.CARDINAL, ...)]
+```
+
+A value is a digit string with no limit on its length: "24", "-2.5" or "1/3". numtag never converts a value to `int` or `float`, so no rounding changes a digit. Convert it with `int()`, `decimal.Decimal()` or `fractions.Fraction()` when you need a number.
+
+The language rules are data files in `numtag/lexicon/`, one per language, with the source of each rule in the file.

numtag-0.0.1a2/README.md

@@ -0,0 +1,15 @@
+# numtag
+
+numtag finds every number in a text. For each number it gives the exact words, their character offsets, the value and the kind.
+
+```python
+from numtag import tag_numbers
+
+tag_numbers("comprou duas dúzias de ovos e um milhão de euros", "pt-PT")
+# [NumberMention(char_span=(8, 19), text='duas dúzias', value='24', kind=Kind.CARDINAL, ...),
+#  NumberMention(char_span=(30, 39), text='um milhão', value='1000000', kind=Kind.CARDINAL, ...)]
+```
+
+A value is a digit string with no limit on its length: "24", "-2.5" or "1/3". numtag never converts a value to `int` or `float`, so no rounding changes a digit. Convert it with `int()`, `decimal.Decimal()` or `fractions.Fraction()` when you need a number.
+
+The language rules are data files in `numtag/lexicon/`, one per language, with the source of each rule in the file.

numtag-0.0.1a2/numtag/__init__.py

@@ -0,0 +1,4 @@
+"""numtag: find every number in a text, with exact values and character spans."""
+from numtag.mention import Form, Kind, NumberMention, is_canonical_value
+
+__all__ = ["Form", "Kind", "NumberMention", "is_canonical_value"]

numtag-0.0.1a2/numtag/mention.py

@@ -0,0 +1,68 @@
+"""The result type of the tagger: one number found in a text."""
+import re
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Optional, Tuple
+
+
+class Kind(str, Enum):
+    """What the number counts or names."""
+    CARDINAL = "cardinal"   # "duas dúzias", "21"
+    ORDINAL = "ordinal"     # "vigésimo primeiro", "21st"
+    FRACTION = "fraction"   # "um terço", "três quartos"
+    DECIMAL = "decimal"     # "dois vírgula cinco", "2,5"
+
+
+class Form(str, Enum):
+    """How the number is written."""
+    WORDS = "words"         # "dois milhões"
+    DIGITS = "digits"       # "2000000"
+    MIXED = "mixed"         # "2 milhões"
+
+
+_WHOLE = r"-?(?:0|[1-9][0-9]*)"
+_CANONICAL = re.compile(
+    rf"{_WHOLE}"                          # whole number
+    rf"|-?(?:0|[1-9][0-9]*)\.[0-9]*[1-9]"  # decimal, no trailing zero
+    rf"|-?[1-9][0-9]*/[1-9][0-9]*")       # fraction
+_NEGATIVE_ZERO = re.compile(r"-0(?:\.0*)?$")
+
+
+def is_canonical_value(value: str) -> bool:
+    """True when ``value`` is in one of the three canonical forms.
+
+    A whole number has no leading zero and is never "-0"; a decimal has no
+    trailing zero in its fraction part; a fraction has no zero part. Lowest
+    terms and the rule that a fraction with a finite decimal form is written
+    as a decimal need arithmetic, so the tagger's own value code keeps them.
+    """
+    return bool(_CANONICAL.fullmatch(value)) and not _NEGATIVE_ZERO.match(value)
+
+
+@dataclass(frozen=True)
+class NumberMention:
+    """One number inside a longer text.
+
+    ``char_span`` is the half-open ``(start, end)`` character extent in the
+    original text, so ``text[start:end] == mention.text``. ``value`` is a
+    canonical digit string ("1002", "-2.5", "1/3"); ``int()``,
+    ``decimal.Decimal()`` and ``fractions.Fraction()`` read it. ``gender``
+    is set when the words show it ("duas" is feminine). ``confidence`` is a
+    deterministic score in ``(0, 1]`` that the words are a number at all
+    (low for an article such as "uma casa"); it is not a probability and it
+    is left out of equality.
+    """
+    char_span: Tuple[int, int]
+    text: str
+    value: str
+    kind: Kind
+    form: Form
+    gender: Optional[str] = None
+    confidence: float = field(default=1.0, compare=False)
+
+    def __post_init__(self):
+        start, end = self.char_span
+        if not 0 <= start < end or end - start != len(self.text):
+            raise ValueError(f"char_span {self.char_span} does not fit text {self.text!r}")
+        if not is_canonical_value(self.value):
+            raise ValueError(f"value {self.value!r} is not a canonical digit string")

numtag-0.0.1a2/numtag/version.py

@@ -0,0 +1,10 @@
+# START_VERSION_BLOCK
+VERSION_MAJOR = 0
+VERSION_MINOR = 0
+VERSION_BUILD = 1
+VERSION_ALPHA = 2
+# END_VERSION_BLOCK
+
+VERSION_STR = f"{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_BUILD}"
+if VERSION_ALPHA:
+    VERSION_STR += f"a{VERSION_ALPHA}"

numtag-0.0.1a2/numtag.egg-info/PKG-INFO

@@ -0,0 +1,37 @@
+Metadata-Version: 2.4
+Name: numtag
+Version: 0.0.1a2
+Summary: Find every number in a text: exact values as digit strings, kinds and character spans.
+Author: TigreGotico
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/TigreGotico/numtag
+Project-URL: Repository, https://github.com/TigreGotico/numtag
+Project-URL: Issues, https://github.com/TigreGotico/numtag/issues
+Keywords: numbers,numerals,nlp,tagger,parser
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Dynamic: license-file
+
+# numtag
+
+numtag finds every number in a text. For each number it gives the exact words, their character offsets, the value and the kind.
+
+```python
+from numtag import tag_numbers
+
+tag_numbers("comprou duas dúzias de ovos e um milhão de euros", "pt-PT")
+# [NumberMention(char_span=(8, 19), text='duas dúzias', value='24', kind=Kind.CARDINAL, ...),
+#  NumberMention(char_span=(30, 39), text='um milhão', value='1000000', kind=Kind.CARDINAL, ...)]
+```
+
+A value is a digit string with no limit on its length: "24", "-2.5" or "1/3". numtag never converts a value to `int` or `float`, so no rounding changes a digit. Convert it with `int()`, `decimal.Decimal()` or `fractions.Fraction()` when you need a number.
+
+The language rules are data files in `numtag/lexicon/`, one per language, with the source of each rule in the file.

numtag-0.0.1a2/numtag.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+LICENSE
+README.md
+pyproject.toml
+numtag/__init__.py
+numtag/mention.py
+numtag/py.typed
+numtag/version.py
+numtag.egg-info/PKG-INFO
+numtag.egg-info/SOURCES.txt
+numtag.egg-info/dependency_links.txt
+numtag.egg-info/requires.txt
+numtag.egg-info/top_level.txt
+tests/test_gold_format.py
+tests/test_mention.py

numtag-0.0.1a2/numtag.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

numtag-0.0.1a2/numtag.egg-info/requires.txt

@@ -0,0 +1,3 @@
+
+[test]
+pytest

numtag-0.0.1a2/numtag.egg-info/top_level.txt

@@ -0,0 +1 @@
+numtag

numtag-0.0.1a2/pyproject.toml

@@ -0,0 +1,38 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "numtag"
+dynamic = ["version"]
+description = "Find every number in a text: exact values as digit strings, kinds and character spans."
+readme = "README.md"
+license = { text = "Apache-2.0" }
+requires-python = ">=3.11"
+authors = [{ name = "TigreGotico" }]
+keywords = ["numbers", "numerals", "nlp", "tagger", "parser"]
+classifiers = [
+    "Development Status :: 2 - Pre-Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Text Processing :: Linguistic",
+]
+dependencies = []
+
+[project.urls]
+Homepage = "https://github.com/TigreGotico/numtag"
+Repository = "https://github.com/TigreGotico/numtag"
+Issues = "https://github.com/TigreGotico/numtag/issues"
+
+[project.optional-dependencies]
+test = ["pytest"]
+
+[tool.setuptools.packages.find]
+include = ["numtag*"]
+
+[tool.setuptools.dynamic]
+version = { attr = "numtag.version.VERSION_STR" }
+
+[tool.setuptools.package-data]
+numtag = ["py.typed", "lexicon/*.toml"]

numtag-0.0.1a2/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

numtag-0.0.1a2/tests/test_gold_format.py

@@ -0,0 +1,186 @@
+import contextlib
+import importlib.util
+import io
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+GOLD_DIR = Path(__file__).parent / "gold"
+_spec = importlib.util.spec_from_file_location("check_gold", GOLD_DIR / "check_gold.py")
+check_gold = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(check_gold)
+
+LANGUAGES = ("pt-PT", "pt-BR", "en")
+MIN_ROWS = 400
+
+
+def _row(text, *mentions, source="arithmetic"):
+    return {"text": text, "mentions": list(mentions), "source": source}
+
+
+def _m(start, end, value, kind="cardinal", **extra):
+    return {"start": start, "end": end, "value": value, "kind": kind, **extra}
+
+
+class TestGoldFiles(unittest.TestCase):
+    def test_every_language_file_passes(self):
+        for lang in LANGUAGES:
+            with self.subTest(lang=lang):
+                n_in, errors = check_gold.check_file(GOLD_DIR / f"{lang}.jsonl")
+                self.assertEqual(errors, [])
+                self.assertGreaterEqual(n_in, MIN_ROWS)
+
+    def test_no_text_is_repeated_in_a_file(self):
+        for lang in LANGUAGES:
+            with self.subTest(lang=lang):
+                lines = (GOLD_DIR / f"{lang}.jsonl").read_text(encoding="utf-8").splitlines()
+                texts = [json.loads(line)["text"] for line in lines]
+                self.assertEqual(len(texts), len(set(texts)))
+
+
+class TestCheckerRefusesBadRows(unittest.TestCase):
+    """The checker must fail on each kind of broken row, or a green run proves nothing."""
+
+    def assertRefused(self, row):
+        with self.assertRaises(AssertionError):
+            check_gold.check_row(row)
+
+    def test_a_good_row_passes(self):
+        check_gold.check_row(_row("comprou duas dúzias de ovos", _m(8, 19, "24")))
+        check_gold.check_row(_row("Estão -5 graus.", _m(6, 8, "-5")))
+        check_gold.check_row(_row("Comprei uma casa.", _m(8, 11, "1", low_confidence=True)))
+        check_gold.check_row(_row("Comeu um terço.", _m(6, 14, "1/3", "fraction")))
+        check_gold.check_row(_row("Nada aqui."))
+
+    def test_span_with_trailing_punctuation(self):
+        self.assertRefused(_row("Saímos às 7, depois.", _m(10, 12, "7")))
+
+    def test_span_with_leading_space(self):
+        self.assertRefused(_row("Saímos às 7, depois.", _m(9, 11, "7")))
+
+    def test_span_with_currency_sign(self):
+        self.assertRefused(_row("Custa €10.", _m(6, 9, "10")))
+
+    def test_span_outside_the_text(self):
+        self.assertRefused(_row("dez", _m(0, 4, "10")))
+
+    def test_empty_span(self):
+        self.assertRefused(_row("dez", _m(1, 1, "10")))
+
+    def test_overlapping_mentions(self):
+        self.assertRefused(_row("vinte e um", _m(0, 5, "20"), _m(3, 10, "21")))
+
+    def test_mentions_out_of_order(self):
+        self.assertRefused(_row("dois e três", _m(7, 11, "3"), _m(0, 4, "2")))
+
+    def test_non_canonical_values(self):
+        for value in ("2.50", "007", "-0", "1,5"):
+            with self.subTest(value=value):
+                self.assertRefused(_row("dois vírgula cinco", _m(0, 18, value, "decimal")))
+
+    def test_fraction_not_in_lowest_terms(self):
+        self.assertRefused(_row("dois quartos", _m(0, 12, "2/4", "fraction")))
+
+    def test_fraction_with_finite_decimal(self):
+        self.assertRefused(_row("um meio", _m(0, 7, "1/2", "fraction")))
+
+    def test_kind_that_contradicts_the_value(self):
+        self.assertRefused(_row("dois e meio", _m(0, 11, "2.5", "cardinal")))
+        self.assertRefused(_row("um milhão e meio", _m(0, 16, "1500000", "fraction")))
+
+    def test_unknown_kind(self):
+        self.assertRefused(_row("dez", _m(0, 3, "10", "number")))
+
+    def test_empty_source(self):
+        self.assertRefused(_row("dez", _m(0, 3, "10"), source=" "))
+
+    def test_low_confidence_false(self):
+        self.assertRefused(_row("uma casa", _m(0, 3, "1", low_confidence=False)))
+
+    def test_unknown_key(self):
+        self.assertRefused(_row("dez", _m(0, 3, "10", gender="masculine")))
+
+    def test_span_that_cuts_a_word(self):
+        self.assertRefused(_row("comprou duas dúzias", _m(8, 18, "24")))
+
+    def test_span_that_starts_inside_a_word(self):
+        self.assertRefused(_row("Custa 125 euros.", _m(7, 9, "25")))
+
+    def test_span_that_stops_before_a_digit_mark(self):
+        self.assertRefused(_row("Pesava 3,5 quilos.", _m(7, 8, "3")))
+        self.assertRefused(_row("It cost 1,000 dollars.", _m(8, 9, "1")))
+        self.assertRefused(_row("It is 3.5 m.", _m(6, 7, "3")))
+
+    def test_span_that_starts_after_a_digit_mark(self):
+        self.assertRefused(_row("Pesava 3,5 quilos.", _m(9, 10, "5")))
+        self.assertRefused(_row("Custa 1.000 euros.", _m(8, 11, "0")))
+        self.assertRefused(_row("It is 3.5 m.", _m(8, 9, "5")))
+
+    def test_a_mark_after_a_letter_is_no_digit_mark(self):
+        check_gold.check_row(_row("Sala n.5", _m(7, 8, "5")))
+        check_gold.check_row(_row("Ver p.12", _m(6, 8, "12")))
+
+    def test_a_list_after_the_digits_is_no_digit_mark(self):
+        check_gold.check_row(_row("1, 2 e 3", _m(0, 1, "1"), _m(3, 4, "2"), _m(7, 8, "3")))
+
+    def test_a_file_with_no_known_decimal_marks_fails(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "xx.jsonl"
+            path.write_text('{"text": "dez", "mentions": [], "source": "arithmetic"}\n', encoding="utf-8")
+            n_in, errors = check_gold.check_file(path)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("no decimal marks", errors[0])
+
+    def test_the_run_counts_good_rows_as_out(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "pt-PT.jsonl"
+            path.write_text('{"text": "dez", "mentions": [], "source": "arithmetic"}\n', encoding="utf-8")
+            out = io.StringIO()
+            with contextlib.redirect_stdout(out):
+                status = check_gold.main([str(path)])
+        self.assertEqual(status, 0)
+        self.assertIn("pt-PT.jsonl: 1 in, 1 out", out.getvalue())
+
+    def test_a_file_with_no_known_decimal_marks_reports_no_row_checked(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "xx.jsonl"
+            path.write_text('{"text": "dez", "mentions": [], "source": "arithmetic"}\n' * 2, encoding="utf-8")
+            out = io.StringIO()
+            with contextlib.redirect_stdout(out):
+                status = check_gold.main([str(path)])
+        self.assertEqual(status, 1)
+        self.assertIn("xx.jsonl: 2 in, 0 out", out.getvalue())
+
+    def test_a_unit_may_touch_the_digits(self):
+        check_gold.check_row(_row("Pesava 3,5kg.", _m(7, 10, "3.5", "decimal")), "pt-PT")
+
+    def test_decimal_kind_without_a_decimal_mark(self):
+        with self.assertRaises(AssertionError):
+            check_gold.check_row(_row("três quartos", _m(0, 12, "0.75", "decimal")), "pt-PT")
+
+    def test_fraction_kind_with_a_decimal_mark(self):
+        with self.assertRaises(AssertionError):
+            check_gold.check_row(_row("dois vírgula cinco", _m(0, 18, "2.5", "fraction")), "pt-PT")
+
+    def test_a_repeated_text_is_reported_by_the_file_check(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "pt-PT.jsonl"
+            line = '{"text": "dez", "mentions": [{"start": 0, "end": 3, "value": "10", "kind": "cardinal"}], "source": "arithmetic"}\n'
+            path.write_text(line * 2, encoding="utf-8")
+            n_in, errors = check_gold.check_file(path)
+        self.assertEqual((n_in, len(errors)), (2, 1))
+        self.assertIn("repeats line 1", errors[0])
+
+    def test_invalid_json_line_is_reported(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "pt-PT.jsonl"
+            path.write_text('{"text": "dez", "mentions": [], "source": "arithmetic"}\n{not json\n', encoding="utf-8")
+            n_in, errors = check_gold.check_file(path)
+        self.assertEqual(n_in, 2)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("pt-PT.jsonl:2", errors[0])
+
+
+if __name__ == "__main__":
+    unittest.main()

numtag-0.0.1a2/tests/test_mention.py

@@ -0,0 +1,50 @@
+import unittest
+from decimal import Decimal
+from fractions import Fraction
+
+from numtag import Form, Kind, NumberMention, is_canonical_value
+
+
+class TestCanonicalValue(unittest.TestCase):
+    def test_canonical_forms_are_accepted(self):
+        for value in ("0", "7", "24", "-21", "1" + "0" * 600,
+                      "2.5", "-0.25", "0.5", "1/3", "-2/3"):
+            with self.subTest(value=value):
+                self.assertTrue(is_canonical_value(value))
+
+    def test_other_forms_are_refused(self):
+        for value in ("", "007", "-0", "-0.0", "2.50", "2.", ".5", "+3",
+                      "0/3", "3/0", "1e6", "1 000", "1,5", "um"):
+            with self.subTest(value=value):
+                self.assertFalse(is_canonical_value(value))
+
+    def test_standard_types_read_every_form(self):
+        self.assertEqual(int("1" + "0" * 600), 10 ** 600)
+        self.assertEqual(Decimal("-0.25"), Decimal(-1) / 4)
+        self.assertEqual(Fraction("1/3"), Fraction(1, 3))
+        self.assertEqual(Fraction("2.5"), Fraction(5, 2))
+
+
+class TestNumberMention(unittest.TestCase):
+    def test_a_mention_holds_its_slice(self):
+        text = "comprou duas dúzias de ovos"
+        mention = NumberMention((8, 19), text[8:19], "24", Kind.CARDINAL,
+                                Form.WORDS, gender="feminine")
+        self.assertEqual(text[slice(*mention.char_span)], mention.text)
+
+    def test_a_span_that_does_not_fit_the_text_is_refused(self):
+        with self.assertRaises(ValueError):
+            NumberMention((0, 3), "duas", "2", Kind.CARDINAL, Form.WORDS)
+
+    def test_a_value_that_is_not_canonical_is_refused(self):
+        with self.assertRaises(ValueError):
+            NumberMention((0, 3), "2,5", "2,5", Kind.DECIMAL, Form.DIGITS)
+
+    def test_confidence_is_not_part_of_equality(self):
+        a = NumberMention((0, 3), "uma", "1", Kind.CARDINAL, Form.WORDS, confidence=0.3)
+        b = NumberMention((0, 3), "uma", "1", Kind.CARDINAL, Form.WORDS)
+        self.assertEqual(a, b)
+
+
+if __name__ == "__main__":
+    unittest.main()