oracc-parser 0.1.2__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/PKG-INFO +13 -7
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/README.md +12 -6
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/__init__.py +1 -1
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/cli.py +0 -25
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/download/fetch_data.py +4 -4
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/languages.csv +1 -1
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/models/config.py +0 -13
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/models/tablet.py +3 -2
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/parsing/translation.py +3 -4
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/pipeline.py +34 -57
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/settings.py +17 -12
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser.egg-info/PKG-INFO +13 -7
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser.egg-info/SOURCES.txt +0 -2
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/pyproject.toml +1 -1
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/tests/test_models.py +0 -2
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/tests/test_settings.py +2 -8
- oracc_parser-0.1.2/oracc_parser/cache.py +0 -251
- oracc_parser-0.1.2/tests/test_cache.py +0 -205
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/LICENSE +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/constants.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/download/__init__.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/download/extract_jsons.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/download/oracc_download.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/download/pleiades.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/__init__.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/grouped_oracc_metadata_columns.csv +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/period_mapping.csv +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/pos_tags.csv +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/projects_metadata.csv +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/provenience.csv +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/raw_archive_values.csv +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/sign_readings.csv +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/state_supergroup_mapping.csv +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/export/__init__.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/export/to_jsonl.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/io/__init__.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/io/word_csv.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/metadata/__init__.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/metadata/archive.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/metadata/populate.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/models/__init__.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/parsing/__init__.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/parsing/parse_content.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/parsing/parse_signs.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/parsing/parse_words.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/parsing/text_builder.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/utils/__init__.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/utils/logger.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/utils/paths.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/utils/unicode.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser.egg-info/dependency_links.txt +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser.egg-info/entry_points.txt +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser.egg-info/requires.txt +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser.egg-info/top_level.txt +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/setup.cfg +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/tests/test_constants.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/tests/test_export.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/tests/test_parsing.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.3}/tests/test_pipeline.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: oracc-parser
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Download and parse ORACC cuneiform text projects into ML-ready formats
|
|
5
5
|
Author: Avital Romach, Shahar Spencer, Claude Sonnet 4.6
|
|
6
6
|
License-Expression: MIT
|
|
@@ -47,6 +47,11 @@ A Python tool to download and parse [ORACC](http://oracc.museum.upenn.edu/) cune
|
|
|
47
47
|
|
|
48
48
|
## Installation
|
|
49
49
|
|
|
50
|
+
```bash
|
|
51
|
+
pip install oracc-parser
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Or for development:
|
|
50
55
|
|
|
51
56
|
```bash
|
|
52
57
|
git clone https://github.com/shaharspencer/oracc-parser.git
|
|
@@ -59,12 +64,13 @@ pip install -e ".[dev]"
|
|
|
59
64
|
The easiest way to explore oracc-parser is through the interactive notebooks.
|
|
60
65
|
Start with notebook 01 — it downloads all the data you need from Zenodo automatically.
|
|
61
66
|
|
|
62
|
-
| Notebook | What you'll learn |
|
|
63
|
-
|
|
64
|
-
| [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project
|
|
65
|
-
| [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) |
|
|
66
|
-
| [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets |
|
|
67
|
-
| [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset |
|
|
67
|
+
| Notebook | What you'll learn | Open in Colab |
|
|
68
|
+
|---|---|---|
|
|
69
|
+
| [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project → explore transliterations, translations, and metadata → export | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/01_quickstart.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
70
|
+
| [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/02_reference_data.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
71
|
+
| [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/03_configure_and_export.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
72
|
+
| [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/04_oracc_json_processing.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
73
|
+
| [`05_translations.ipynb`](notebooks/05_translations.ipynb) | Where translations come from, how to download the translation cache from Zenodo, and how to enable translations in the parsing pipeline | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/05_translations.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
68
74
|
|
|
69
75
|
```bash
|
|
70
76
|
pip install oracc-parser[notebooks]
|
|
@@ -11,6 +11,11 @@ A Python tool to download and parse [ORACC](http://oracc.museum.upenn.edu/) cune
|
|
|
11
11
|
|
|
12
12
|
## Installation
|
|
13
13
|
|
|
14
|
+
```bash
|
|
15
|
+
pip install oracc-parser
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Or for development:
|
|
14
19
|
|
|
15
20
|
```bash
|
|
16
21
|
git clone https://github.com/shaharspencer/oracc-parser.git
|
|
@@ -23,12 +28,13 @@ pip install -e ".[dev]"
|
|
|
23
28
|
The easiest way to explore oracc-parser is through the interactive notebooks.
|
|
24
29
|
Start with notebook 01 — it downloads all the data you need from Zenodo automatically.
|
|
25
30
|
|
|
26
|
-
| Notebook | What you'll learn |
|
|
27
|
-
|
|
28
|
-
| [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project
|
|
29
|
-
| [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) |
|
|
30
|
-
| [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets |
|
|
31
|
-
| [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset |
|
|
31
|
+
| Notebook | What you'll learn | Open in Colab |
|
|
32
|
+
|---|---|---|
|
|
33
|
+
| [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project → explore transliterations, translations, and metadata → export | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/01_quickstart.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
34
|
+
| [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/02_reference_data.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
35
|
+
| [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/03_configure_and_export.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
36
|
+
| [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/04_oracc_json_processing.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
37
|
+
| [`05_translations.ipynb`](notebooks/05_translations.ipynb) | Where translations come from, how to download the translation cache from Zenodo, and how to enable translations in the parsing pipeline | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/05_translations.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
32
38
|
|
|
33
39
|
```bash
|
|
34
40
|
pip install oracc-parser[notebooks]
|
|
@@ -6,8 +6,6 @@ Usage:
|
|
|
6
6
|
oracc-parser download --lang akkadian
|
|
7
7
|
oracc-parser parse --project saao/saa01 --format jsonl --output data.jsonl
|
|
8
8
|
oracc-parser parse --project saao/saa01 --limit 5 --format csv --output data.csv
|
|
9
|
-
oracc-parser clear-cache # clear all cached tablets
|
|
10
|
-
oracc-parser clear-cache --project saao # clear one project's cache
|
|
11
9
|
"""
|
|
12
10
|
from __future__ import annotations
|
|
13
11
|
|
|
@@ -71,7 +69,6 @@ def main(argv: list[str] | None = None):
|
|
|
71
69
|
default=[],
|
|
72
70
|
help="POS tags to mask (e.g. PN DN GN)",
|
|
73
71
|
)
|
|
74
|
-
ps.add_argument("--no-cache", action="store_true", help="Disable caching")
|
|
75
72
|
ps.add_argument("--no-download", action="store_true", help="Skip download step")
|
|
76
73
|
|
|
77
74
|
# --------------- fetch-data ---------------
|
|
@@ -82,13 +79,6 @@ def main(argv: list[str] | None = None):
|
|
|
82
79
|
# --------------- info ---------------
|
|
83
80
|
subparsers.add_parser("info", help="Show bundled reference data summary")
|
|
84
81
|
|
|
85
|
-
# --------------- clear-cache ---------------
|
|
86
|
-
cc = subparsers.add_parser("clear-cache", help="Delete cached parsed tablets")
|
|
87
|
-
cc.add_argument(
|
|
88
|
-
"--project", "-p",
|
|
89
|
-
help="Only clear cache for this project (default: clear all)",
|
|
90
|
-
)
|
|
91
|
-
|
|
92
82
|
args = parser.parse_args(argv)
|
|
93
83
|
|
|
94
84
|
if args.command == "download":
|
|
@@ -99,8 +89,6 @@ def main(argv: list[str] | None = None):
|
|
|
99
89
|
_cmd_parse(args)
|
|
100
90
|
elif args.command == "info":
|
|
101
91
|
_cmd_info()
|
|
102
|
-
elif args.command == "clear-cache":
|
|
103
|
-
_cmd_clear_cache(args)
|
|
104
92
|
else:
|
|
105
93
|
parser.print_help()
|
|
106
94
|
sys.exit(1)
|
|
@@ -144,7 +132,6 @@ def _cmd_parse(args):
|
|
|
144
132
|
drop_missing=args.drop_missing,
|
|
145
133
|
drop_damaged=args.drop_damaged,
|
|
146
134
|
mask_pos=args.mask_pos,
|
|
147
|
-
use_cache=not args.no_cache,
|
|
148
135
|
limit=args.limit,
|
|
149
136
|
)
|
|
150
137
|
|
|
@@ -185,17 +172,5 @@ def _cmd_info():
|
|
|
185
172
|
print(f" {name}: Error loading - {e}", file=sys.stderr)
|
|
186
173
|
|
|
187
174
|
|
|
188
|
-
def _cmd_clear_cache(args):
|
|
189
|
-
"""Handle the clear-cache command."""
|
|
190
|
-
from oracc_parser.cache import clear_project_cache
|
|
191
|
-
|
|
192
|
-
count = clear_project_cache(project=args.project)
|
|
193
|
-
if count:
|
|
194
|
-
scope = f"project '{args.project}'" if args.project else "all projects"
|
|
195
|
-
print(f"Cleared {count} cached tablet(s) for {scope}.")
|
|
196
|
-
else:
|
|
197
|
-
print("No cached tablets found.")
|
|
198
|
-
|
|
199
|
-
|
|
200
175
|
if __name__ == "__main__":
|
|
201
176
|
main()
|
|
@@ -22,7 +22,7 @@ from pathlib import Path
|
|
|
22
22
|
import requests
|
|
23
23
|
from tqdm import tqdm
|
|
24
24
|
|
|
25
|
-
from oracc_parser.settings import zenodo_url, data_dir, jsonzip_dir, word_csv_dir, catalogue_dir,
|
|
25
|
+
from oracc_parser.settings import zenodo_url, data_dir, jsonzip_dir, word_csv_dir, catalogue_dir, translations_dir
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
def download_file(url: str, dest: Path, desc: str = "") -> Path:
|
|
@@ -235,7 +235,7 @@ def fetch_data(
|
|
|
235
235
|
|
|
236
236
|
def _already_extracted(filename: str) -> bool:
|
|
237
237
|
if filename == "oracc_html_translations.zip":
|
|
238
|
-
return
|
|
238
|
+
return (translations_dir() / ".translations_complete").exists()
|
|
239
239
|
if filename == "catalogues.zip":
|
|
240
240
|
return catalogue_dir().exists() and any(catalogue_dir().glob("*.csv"))
|
|
241
241
|
if filename == "oracc_jsonzip_all.zip":
|
|
@@ -266,9 +266,9 @@ def fetch_data(
|
|
|
266
266
|
|
|
267
267
|
translations_zip = dest / "oracc_html_translations.zip"
|
|
268
268
|
if translations_zip.exists():
|
|
269
|
-
extract_data_archive(translations_zip,
|
|
269
|
+
extract_data_archive(translations_zip, translations_dir(), "HTML translations")
|
|
270
270
|
translations_zip.unlink()
|
|
271
|
-
(
|
|
271
|
+
(translations_dir() / ".translations_complete").touch()
|
|
272
272
|
|
|
273
273
|
catalogues_zip = dest / "catalogues.zip"
|
|
274
274
|
if catalogues_zip.exists():
|
|
@@ -20,7 +20,7 @@ akk-x-oldass,Akkadian,Old Assyrian,yes,,695,tcma-assur,1
|
|
|
20
20
|
uga,Ugaritic,,yes,Ugaritic loanwords in Akkadian,440,"aemw-amarna, aemw-ugarit",2
|
|
21
21
|
hit,Hittite,,yes,,224,"aemw-ugarit, contrib-amarna",2
|
|
22
22
|
qca,Canaanite,,yes,Canaanite loanwords in Akkadian,200,aemw-amarna,1
|
|
23
|
-
qcu-949
|
|
23
|
+
qcu-949,Assyrian Hieroglyphs,,no,"Assyrian ""Hieroglyphs""",176,"ribo-babylon6, rinap-rinap4",2
|
|
24
24
|
akk-x-oldakk,Akkadian,Old Akkadian,yes,,90,akklove,1
|
|
25
25
|
hit-946,Hittite,,yes,,72,"aemw-ugarit, contrib-amarna",2
|
|
26
26
|
qcu,Unclear,,yes,one broken text,68,cams-gkab,1
|
|
@@ -82,19 +82,6 @@ class RunConfig(BaseModel):
|
|
|
82
82
|
),
|
|
83
83
|
)
|
|
84
84
|
|
|
85
|
-
# --- Caching ---
|
|
86
|
-
use_cache: bool = Field(
|
|
87
|
-
default=True,
|
|
88
|
-
description="Use cached parsed results if available.",
|
|
89
|
-
)
|
|
90
|
-
cache_dir: Optional[str] = Field(
|
|
91
|
-
default=None,
|
|
92
|
-
description=(
|
|
93
|
-
"Directory for cached parsed results. "
|
|
94
|
-
"Defaults to './oracc_cache/' in the current working directory."
|
|
95
|
-
),
|
|
96
|
-
)
|
|
97
|
-
|
|
98
85
|
# --- Sample / limit mode ---
|
|
99
86
|
limit: Optional[int] = Field(
|
|
100
87
|
default=None,
|
|
@@ -36,7 +36,7 @@ class Language(BaseModel):
|
|
|
36
36
|
"""Language tag for a word, with cuneiform flag."""
|
|
37
37
|
|
|
38
38
|
is_cuneiform: bool = True
|
|
39
|
-
normalized_language: Literal[
|
|
39
|
+
normalized_language: Optional[Literal[
|
|
40
40
|
"Sumerian",
|
|
41
41
|
"Proto-cuneiform",
|
|
42
42
|
"Akkadian",
|
|
@@ -53,7 +53,8 @@ class Language(BaseModel):
|
|
|
53
53
|
"Proto-Elamite",
|
|
54
54
|
"Greek",
|
|
55
55
|
"Egyptian",
|
|
56
|
-
|
|
56
|
+
"Assyrian Hieroglyphs",
|
|
57
|
+
]] = None
|
|
57
58
|
dialect: str = ""
|
|
58
59
|
|
|
59
60
|
|
|
@@ -12,14 +12,14 @@ import requests
|
|
|
12
12
|
from bs4 import BeautifulSoup
|
|
13
13
|
|
|
14
14
|
from oracc_parser.utils.logger import get_logger
|
|
15
|
-
from oracc_parser.settings import
|
|
15
|
+
from oracc_parser.settings import TRANSLATIONS_DIR as _settings_TRANSLATIONS_DIR
|
|
16
16
|
|
|
17
17
|
logger = get_logger()
|
|
18
18
|
|
|
19
19
|
ORACC_HTML_URL = "https://oracc.museum.upenn.edu"
|
|
20
20
|
|
|
21
21
|
|
|
22
|
-
def get_translation(project: str, text_id: str
|
|
22
|
+
def get_translation(project: str, text_id: str) -> str:
|
|
23
23
|
"""Fetch the English translation for a specific tablet.
|
|
24
24
|
|
|
25
25
|
Tries to read from a cached HTML file first. If not found,
|
|
@@ -28,12 +28,11 @@ def get_translation(project: str, text_id: str, cache_dir: str | None = None) ->
|
|
|
28
28
|
Args:
|
|
29
29
|
project: Project path, e.g. ``"saao/saa01"``.
|
|
30
30
|
text_id: The P-number or text ID, e.g. ``"P313511"``.
|
|
31
|
-
cache_dir: Custom cache directory. Default: ``oracc_cache/html/``.
|
|
32
31
|
|
|
33
32
|
Returns:
|
|
34
33
|
Multi-line translation string, or empty string on failure.
|
|
35
34
|
"""
|
|
36
|
-
cache =
|
|
35
|
+
cache = _settings_TRANSLATIONS_DIR
|
|
37
36
|
cache.mkdir(parents=True, exist_ok=True)
|
|
38
37
|
|
|
39
38
|
project_path = project.replace("-", "/")
|
|
@@ -18,11 +18,6 @@ from pathlib import Path
|
|
|
18
18
|
import pandas as pd
|
|
19
19
|
from tqdm import tqdm
|
|
20
20
|
|
|
21
|
-
from oracc_parser.cache import (
|
|
22
|
-
clear_project_cache,
|
|
23
|
-
load_cached_tablet,
|
|
24
|
-
save_tablet_to_cache,
|
|
25
|
-
)
|
|
26
21
|
from oracc_parser.download.extract_jsons import extract_from_zip
|
|
27
22
|
from oracc_parser.download.oracc_download import download_projects, download_zip, get_live_projects_dataframe
|
|
28
23
|
from oracc_parser.export.to_jsonl import to_csv, to_jsonl
|
|
@@ -33,6 +28,7 @@ from oracc_parser.io.word_csv import (
|
|
|
33
28
|
load_word_csvs_from_zenodo,
|
|
34
29
|
record_to_word_dataframe,
|
|
35
30
|
save_catalogue_csv,
|
|
31
|
+
save_word_csv,
|
|
36
32
|
word_dataframe_to_record,
|
|
37
33
|
)
|
|
38
34
|
from oracc_parser.metadata.populate import populate_metadata
|
|
@@ -94,10 +90,14 @@ def parse_project(
|
|
|
94
90
|
) -> list[TabletRecord]:
|
|
95
91
|
"""Download, parse, and return all tablets from an ORACC project.
|
|
96
92
|
|
|
93
|
+
On first call, downloads the ORACC ZIP, parses each tablet from JSON,
|
|
94
|
+
and saves per-word CSVs to disk for fast future reloads. On subsequent
|
|
95
|
+
calls the word CSVs are used directly, skipping the JSON parsing step.
|
|
96
|
+
|
|
97
97
|
Args:
|
|
98
98
|
project: ORACC project path, e.g. ``"saao/saa01"``.
|
|
99
99
|
config: RunConfig with parsing options. Uses defaults if None.
|
|
100
|
-
download: If True, download the ZIP from ORACC
|
|
100
|
+
download: If True, download the ZIP from ORACC if not already present.
|
|
101
101
|
|
|
102
102
|
Returns:
|
|
103
103
|
List of TabletRecord objects.
|
|
@@ -105,62 +105,52 @@ def parse_project(
|
|
|
105
105
|
if config is None:
|
|
106
106
|
config = RunConfig()
|
|
107
107
|
|
|
108
|
-
|
|
108
|
+
from oracc_parser.settings import WORD_CSV_DIR
|
|
109
|
+
project_slug = project.replace("/", "-")
|
|
110
|
+
csv_dir = WORD_CSV_DIR / project_slug
|
|
111
|
+
|
|
112
|
+
# Fast path: word CSVs already on disk — skip JSON parsing entirely
|
|
113
|
+
if csv_dir.exists() and any(csv_dir.glob("*.csv")):
|
|
114
|
+
word_dfs = load_word_csvs_from_dir(csv_dir, project=project)
|
|
115
|
+
if config.limit is not None:
|
|
116
|
+
word_dfs = dict(list(word_dfs.items())[: config.limit])
|
|
117
|
+
return parse_project_from_word_csvs(project, word_dfs, config=config)
|
|
118
|
+
|
|
119
|
+
# Slow path: parse from JSON, then save word CSVs for future use
|
|
109
120
|
if download:
|
|
110
121
|
zip_path = download_zip(project)
|
|
111
122
|
if not zip_path:
|
|
112
123
|
logger.error(f"Failed to download {project}")
|
|
113
124
|
return []
|
|
114
125
|
|
|
115
|
-
# 2. Extract JSONs from ZIP
|
|
116
126
|
project_data = extract_from_zip(project)
|
|
117
127
|
if not project_data.json_files:
|
|
118
128
|
logger.warning(f"No JSON files found for {project}")
|
|
119
129
|
return []
|
|
120
130
|
|
|
121
|
-
# 3. Parse each text
|
|
122
131
|
catalogue = project_data.project_catalogue or {}
|
|
123
132
|
members = catalogue.get("members", {})
|
|
124
|
-
records = []
|
|
125
133
|
|
|
126
134
|
json_files = project_data.json_files
|
|
127
135
|
if config.limit is not None:
|
|
128
136
|
json_files = json_files[: config.limit]
|
|
129
137
|
|
|
130
|
-
|
|
138
|
+
records = []
|
|
131
139
|
for js in tqdm(json_files, desc=f"Parsing {project}"):
|
|
132
140
|
text_id = js.get("textid", "")
|
|
133
|
-
|
|
134
|
-
# --- Cache: instant if config matches, rebuild strings if not ---
|
|
135
|
-
if config.use_cache:
|
|
136
|
-
cached = load_cached_tablet(project, text_id, config, config.cache_dir)
|
|
137
|
-
if cached is not None:
|
|
138
|
-
records.append(cached)
|
|
139
|
-
cache_hits += 1
|
|
140
|
-
continue
|
|
141
|
-
|
|
142
|
-
# --- Parse from scratch ---
|
|
143
141
|
metadata_dict = members.get(text_id, {})
|
|
144
142
|
|
|
145
143
|
record = TabletRecord()
|
|
146
144
|
record.content = parse_json_text(js, config)
|
|
147
|
-
|
|
148
|
-
project, text_id
|
|
149
|
-
)
|
|
145
|
+
if config.fetch_translations:
|
|
146
|
+
record.content.english_translation = get_translation(project, text_id)
|
|
150
147
|
record.metadata = populate_metadata(metadata_dict, text_id, project)
|
|
151
148
|
records.append(record)
|
|
152
149
|
|
|
153
|
-
#
|
|
154
|
-
|
|
155
|
-
save_tablet_to_cache(record, project, text_id, config, config.cache_dir)
|
|
150
|
+
# Save word CSV so future calls skip JSON parsing
|
|
151
|
+
save_word_csv(record_to_word_dataframe(record))
|
|
156
152
|
|
|
157
|
-
|
|
158
|
-
logger.info(
|
|
159
|
-
f"Loaded {cache_hits}/{len(records)} tablets from cache, "
|
|
160
|
-
f"parsed {len(records) - cache_hits} new"
|
|
161
|
-
)
|
|
162
|
-
else:
|
|
163
|
-
logger.info(f"Parsed {len(records)} tablets from {project}")
|
|
153
|
+
logger.info(f"Parsed {len(records)} tablets from {project}")
|
|
164
154
|
return records
|
|
165
155
|
|
|
166
156
|
|
|
@@ -317,9 +307,7 @@ def parse_project_from_word_csvs(
|
|
|
317
307
|
catalogue_row = catalogue_lookup.get(text_id)
|
|
318
308
|
record = word_dataframe_to_record(df, config, catalogue_row=catalogue_row)
|
|
319
309
|
if config.fetch_translations:
|
|
320
|
-
record.content.english_translation = get_translation(
|
|
321
|
-
project, text_id, cache_dir=config.cache_dir
|
|
322
|
-
)
|
|
310
|
+
record.content.english_translation = get_translation(project, text_id)
|
|
323
311
|
records.append(record)
|
|
324
312
|
|
|
325
313
|
logger.info(f"Processed {len(records)} tablets for {project} from word CSVs")
|
|
@@ -493,11 +481,10 @@ def get_translations(records: list[TabletRecord]) -> pd.DataFrame:
|
|
|
493
481
|
def get_full_flat_table(records: list[TabletRecord]) -> pd.DataFrame:
|
|
494
482
|
"""Get everything in one flat DataFrame — ideal for releasing as a dataset.
|
|
495
483
|
|
|
496
|
-
Combines metadata
|
|
497
|
-
No nesting, no Pydantic objects
|
|
484
|
+
Combines all metadata columns (same as ``get_metadata_table``) with the
|
|
485
|
+
text string representations. No nesting, no Pydantic objects.
|
|
498
486
|
|
|
499
|
-
Returns columns
|
|
500
|
-
``provenance``, ``period``, ``start_year``, ``end_year``,
|
|
487
|
+
Returns all columns from ``get_metadata_table`` plus:
|
|
501
488
|
``transliteration``, ``normalization``, ``lemmatization``,
|
|
502
489
|
``unicode``, ``translation``, ``total_tokens``, ``tokens_without_broken``.
|
|
503
490
|
|
|
@@ -507,7 +494,9 @@ def get_full_flat_table(records: list[TabletRecord]) -> pd.DataFrame:
|
|
|
507
494
|
df = get_full_flat_table(records)
|
|
508
495
|
df.to_json("dataset.jsonl", orient="records", lines=True)
|
|
509
496
|
"""
|
|
510
|
-
|
|
497
|
+
meta_df = get_metadata_table(records)
|
|
498
|
+
|
|
499
|
+
text_rows = []
|
|
511
500
|
for r in records:
|
|
512
501
|
md = r.metadata
|
|
513
502
|
ct = r.content
|
|
@@ -515,21 +504,8 @@ def get_full_flat_table(records: list[TabletRecord]) -> pd.DataFrame:
|
|
|
515
504
|
n_rep = ct.normalized_str_representation
|
|
516
505
|
l_rep = ct.lemmatized_str_representation
|
|
517
506
|
u_rep = ct.unicode_str_representation
|
|
518
|
-
|
|
519
|
-
rows.append({
|
|
507
|
+
text_rows.append({
|
|
520
508
|
"id": md.identifier,
|
|
521
|
-
"project": md.project,
|
|
522
|
-
"text_id": md.id_text,
|
|
523
|
-
"genre": md.genre or "",
|
|
524
|
-
"archive": md.archive or "",
|
|
525
|
-
"provenance": md.geographical_information.city.city_name,
|
|
526
|
-
"period": (
|
|
527
|
-
md.chronological_information.tablet_period.period_name
|
|
528
|
-
if md.chronological_information.tablet_period
|
|
529
|
-
else ""
|
|
530
|
-
),
|
|
531
|
-
"start_year": md.chronological_information.start_year,
|
|
532
|
-
"end_year": md.chronological_information.end_year,
|
|
533
509
|
"transliteration": t_rep.text if t_rep else "",
|
|
534
510
|
"normalization": n_rep.text if n_rep else "",
|
|
535
511
|
"lemmatization": l_rep.text if l_rep else "",
|
|
@@ -538,4 +514,5 @@ def get_full_flat_table(records: list[TabletRecord]) -> pd.DataFrame:
|
|
|
538
514
|
"total_tokens": t_rep.total_tokens if t_rep else 0,
|
|
539
515
|
"tokens_without_broken": t_rep.tokens_without_broken if t_rep else 0,
|
|
540
516
|
})
|
|
541
|
-
|
|
517
|
+
text_df = pd.DataFrame(text_rows)
|
|
518
|
+
return meta_df.merge(text_df, on="id")
|
|
@@ -20,7 +20,7 @@ Or via environment variable (set before running Python)::
|
|
|
20
20
|
oracc_data/ <-- DATA_DIR
|
|
21
21
|
catalogues/ <-- CATALOGUE_DIR (project catalogue CSVs)
|
|
22
22
|
oracc_csvs/ <-- WORD_CSV_DIR (downloaded word-level CSVs)
|
|
23
|
-
|
|
23
|
+
translations/ <-- TRANSLATIONS_DIR (cached HTML translation pages)
|
|
24
24
|
jsonzip/ <-- JSONZIP_DIR (downloaded ORACC JSON ZIPs)
|
|
25
25
|
output/ <-- OUTPUT_DIR (exported CSVs, JSONL files)
|
|
26
26
|
|
|
@@ -57,7 +57,7 @@ def _default_data_dir() -> Path:
|
|
|
57
57
|
DATA_DIR: Path = _default_data_dir()
|
|
58
58
|
OUTPUT_DIR: Path = DATA_DIR / "output"
|
|
59
59
|
|
|
60
|
-
|
|
60
|
+
TRANSLATIONS_DIR: Path = DATA_DIR / "translations"
|
|
61
61
|
JSONZIP_DIR: Path = DATA_DIR / "jsonzip"
|
|
62
62
|
WORD_CSV_DIR: Path = DATA_DIR / "oracc_csvs"
|
|
63
63
|
CATALOGUE_DIR: Path = DATA_DIR / "catalogues"
|
|
@@ -65,9 +65,6 @@ CATALOGUE_DIR: Path = DATA_DIR / "catalogues"
|
|
|
65
65
|
# URL for downloading reference data (Zenodo)
|
|
66
66
|
ZENODO_RECORD_URL: str = "https://zenodo.org/records/20625379"
|
|
67
67
|
|
|
68
|
-
# Should the parser use cached output?
|
|
69
|
-
USE_CACHE: bool = True
|
|
70
|
-
|
|
71
68
|
# Logging
|
|
72
69
|
LOG_LEVEL: str = "INFO"
|
|
73
70
|
|
|
@@ -87,9 +84,9 @@ def output_dir() -> Path:
|
|
|
87
84
|
global OUTPUT_DIR
|
|
88
85
|
return OUTPUT_DIR
|
|
89
86
|
|
|
90
|
-
def
|
|
91
|
-
global
|
|
92
|
-
return
|
|
87
|
+
def translations_dir() -> Path:
|
|
88
|
+
global TRANSLATIONS_DIR
|
|
89
|
+
return TRANSLATIONS_DIR
|
|
93
90
|
|
|
94
91
|
def jsonzip_dir() -> Path:
|
|
95
92
|
global JSONZIP_DIR
|
|
@@ -107,10 +104,6 @@ def zenodo_url() -> str:
|
|
|
107
104
|
global ZENODO_RECORD_URL
|
|
108
105
|
return ZENODO_RECORD_URL
|
|
109
106
|
|
|
110
|
-
def use_cache() -> bool:
|
|
111
|
-
global USE_CACHE
|
|
112
|
-
return USE_CACHE
|
|
113
|
-
|
|
114
107
|
def log_level() -> str:
|
|
115
108
|
global LOG_LEVEL
|
|
116
109
|
return LOG_LEVEL
|
|
@@ -118,3 +111,15 @@ def log_level() -> str:
|
|
|
118
111
|
def pleiades_zip_path() -> Path | None:
|
|
119
112
|
global PLEIADES_ZIP
|
|
120
113
|
return PLEIADES_ZIP
|
|
114
|
+
|
|
115
|
+
def get_settings() -> dict:
|
|
116
|
+
"""Return current settings as a plain dict (useful for inspection/debugging)."""
|
|
117
|
+
return {
|
|
118
|
+
"data_dir": str(DATA_DIR),
|
|
119
|
+
"translations_dir": str(TRANSLATIONS_DIR),
|
|
120
|
+
"jsonzip_dir": str(JSONZIP_DIR),
|
|
121
|
+
"word_csv_dir": str(WORD_CSV_DIR),
|
|
122
|
+
"catalogue_dir": str(CATALOGUE_DIR),
|
|
123
|
+
"zenodo_record_url": ZENODO_RECORD_URL,
|
|
124
|
+
"log_level": LOG_LEVEL,
|
|
125
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: oracc-parser
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: Download and parse ORACC cuneiform text projects into ML-ready formats
|
|
5
5
|
Author: Avital Romach, Shahar Spencer, Claude Sonnet 4.6
|
|
6
6
|
License-Expression: MIT
|
|
@@ -47,6 +47,11 @@ A Python tool to download and parse [ORACC](http://oracc.museum.upenn.edu/) cune
|
|
|
47
47
|
|
|
48
48
|
## Installation
|
|
49
49
|
|
|
50
|
+
```bash
|
|
51
|
+
pip install oracc-parser
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Or for development:
|
|
50
55
|
|
|
51
56
|
```bash
|
|
52
57
|
git clone https://github.com/shaharspencer/oracc-parser.git
|
|
@@ -59,12 +64,13 @@ pip install -e ".[dev]"
|
|
|
59
64
|
The easiest way to explore oracc-parser is through the interactive notebooks.
|
|
60
65
|
Start with notebook 01 — it downloads all the data you need from Zenodo automatically.
|
|
61
66
|
|
|
62
|
-
| Notebook | What you'll learn |
|
|
63
|
-
|
|
64
|
-
| [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project
|
|
65
|
-
| [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) |
|
|
66
|
-
| [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets |
|
|
67
|
-
| [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset |
|
|
67
|
+
| Notebook | What you'll learn | Open in Colab |
|
|
68
|
+
|---|---|---|
|
|
69
|
+
| [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project → explore transliterations, translations, and metadata → export | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/01_quickstart.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
70
|
+
| [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/02_reference_data.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
71
|
+
| [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/03_configure_and_export.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
72
|
+
| [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/04_oracc_json_processing.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
73
|
+
| [`05_translations.ipynb`](notebooks/05_translations.ipynb) | Where translations come from, how to download the translation cache from Zenodo, and how to enable translations in the parsing pipeline | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/05_translations.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
68
74
|
|
|
69
75
|
```bash
|
|
70
76
|
pip install oracc-parser[notebooks]
|
|
@@ -2,7 +2,6 @@ LICENSE
|
|
|
2
2
|
README.md
|
|
3
3
|
pyproject.toml
|
|
4
4
|
oracc_parser/__init__.py
|
|
5
|
-
oracc_parser/cache.py
|
|
6
5
|
oracc_parser/cli.py
|
|
7
6
|
oracc_parser/constants.py
|
|
8
7
|
oracc_parser/pipeline.py
|
|
@@ -48,7 +47,6 @@ oracc_parser/utils/__init__.py
|
|
|
48
47
|
oracc_parser/utils/logger.py
|
|
49
48
|
oracc_parser/utils/paths.py
|
|
50
49
|
oracc_parser/utils/unicode.py
|
|
51
|
-
tests/test_cache.py
|
|
52
50
|
tests/test_constants.py
|
|
53
51
|
tests/test_export.py
|
|
54
52
|
tests/test_models.py
|
|
@@ -25,8 +25,6 @@ class TestRunConfig:
|
|
|
25
25
|
assert config.drop_damaged is False
|
|
26
26
|
assert config.keep_word_segmentation is True
|
|
27
27
|
assert config.mask_pos == []
|
|
28
|
-
assert config.use_cache is True
|
|
29
|
-
assert config.cache_dir is None
|
|
30
28
|
assert config.limit is None
|
|
31
29
|
assert config.languages == ["Akkadian"]
|
|
32
30
|
|
|
@@ -4,7 +4,7 @@ import os
|
|
|
4
4
|
|
|
5
5
|
import pytest
|
|
6
6
|
|
|
7
|
-
from oracc_parser.settings import get_settings, data_dir,
|
|
7
|
+
from oracc_parser.settings import get_settings, data_dir, log_level
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
class TestSettings:
|
|
@@ -17,18 +17,12 @@ class TestSettings:
|
|
|
17
17
|
d = data_dir()
|
|
18
18
|
assert hasattr(d, "exists") # It's a Path
|
|
19
19
|
|
|
20
|
-
def test_cache_dir_under_data(self):
|
|
21
|
-
c = cache_dir()
|
|
22
|
-
d = data_dir()
|
|
23
|
-
# Default cache dir should be under data dir
|
|
24
|
-
assert str(d) in str(c)
|
|
25
|
-
|
|
26
20
|
def test_settings_returns_dict(self):
|
|
27
21
|
s = get_settings()
|
|
28
22
|
assert isinstance(s, dict)
|
|
29
23
|
assert "zenodo_record_url" in s
|
|
30
24
|
assert "data_dir" in s
|
|
31
|
-
assert "
|
|
25
|
+
assert "translations_dir" in s
|
|
32
26
|
|
|
33
27
|
def test_env_override(self, monkeypatch):
|
|
34
28
|
"""Environment variables should override defaults."""
|
|
@@ -1,251 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
JSON caching for parsed TabletRecord objects.
|
|
3
|
-
|
|
4
|
-
Parsed tablets are expensive to produce (long runtimes due to CDL tree
|
|
5
|
-
traversal, sign parsing, and translation downloads). This module caches
|
|
6
|
-
the full result including a **config fingerprint**.
|
|
7
|
-
|
|
8
|
-
On reload:
|
|
9
|
-
- If the current config matches the cached fingerprint → **instant return**
|
|
10
|
-
(everything is reused, including string representations)
|
|
11
|
-
- If the config differs → the cached **words** are reused and string
|
|
12
|
-
representations are rebuilt (cheap, no re-parsing needed)
|
|
13
|
-
- If not cached at all → full parse from scratch
|
|
14
|
-
|
|
15
|
-
Cache layout::
|
|
16
|
-
|
|
17
|
-
{cache_dir}/tablets/{project}/{text_id}.json
|
|
18
|
-
|
|
19
|
-
Each file is a JSON wrapper::
|
|
20
|
-
|
|
21
|
-
{"config_fingerprint": "a1b2c3d4", "record": { ... TabletRecord ... }}
|
|
22
|
-
"""
|
|
23
|
-
from __future__ import annotations
|
|
24
|
-
|
|
25
|
-
import hashlib
|
|
26
|
-
import json
|
|
27
|
-
from pathlib import Path
|
|
28
|
-
|
|
29
|
-
from oracc_parser.utils.logger import get_logger
|
|
30
|
-
|
|
31
|
-
logger = get_logger()
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
# ---------------------------------------------------------------------------
|
|
35
|
-
# Config fingerprinting
|
|
36
|
-
# ---------------------------------------------------------------------------
|
|
37
|
-
|
|
38
|
-
# These RunConfig fields affect the parsed output.
|
|
39
|
-
# Everything else (USE_CACHE, CACHE_DIR, limit, languages) does NOT.
|
|
40
|
-
_OUTPUT_AFFECTING_FIELDS = (
|
|
41
|
-
"drop_missing",
|
|
42
|
-
"drop_damaged",
|
|
43
|
-
"keep_word_segmentation",
|
|
44
|
-
"mask_pos",
|
|
45
|
-
"max_break_fraction",
|
|
46
|
-
)
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
def config_fingerprint(config) -> str:
|
|
50
|
-
"""Compute a short, stable hash of the output-affecting config options.
|
|
51
|
-
|
|
52
|
-
Args:
|
|
53
|
-
config: A ``RunConfig`` instance.
|
|
54
|
-
|
|
55
|
-
Returns:
|
|
56
|
-
8-char hex string (e.g. ``"a1b2c3d4"``).
|
|
57
|
-
"""
|
|
58
|
-
key = {}
|
|
59
|
-
for field in _OUTPUT_AFFECTING_FIELDS:
|
|
60
|
-
val = getattr(config, field)
|
|
61
|
-
if isinstance(val, list):
|
|
62
|
-
val = sorted(val)
|
|
63
|
-
key[field] = val
|
|
64
|
-
|
|
65
|
-
raw = json.dumps(key, sort_keys=True)
|
|
66
|
-
return hashlib.sha256(raw.encode()).hexdigest()[:8]
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
# ---------------------------------------------------------------------------
|
|
70
|
-
# Path helpers
|
|
71
|
-
# ---------------------------------------------------------------------------
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
def _resolve_cache_dir(cache_dir: str | None = None) -> Path:
|
|
75
|
-
"""Return the base cache directory."""
|
|
76
|
-
if cache_dir:
|
|
77
|
-
return Path(cache_dir)
|
|
78
|
-
from oracc_parser.settings import CACHE_DIR as settings_CACHE_DIR
|
|
79
|
-
return settings_CACHE_DIR
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
def _tablet_path(
|
|
83
|
-
project: str,
|
|
84
|
-
text_id: str,
|
|
85
|
-
cache_dir: str | None = None,
|
|
86
|
-
) -> Path:
|
|
87
|
-
"""Return the JSON file path for a cached tablet."""
|
|
88
|
-
base = _resolve_cache_dir(cache_dir) / "tablets"
|
|
89
|
-
project_dir = project.replace("/", "-")
|
|
90
|
-
return base / project_dir / f"{text_id}.json"
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
# ---------------------------------------------------------------------------
|
|
94
|
-
# Load / Save
|
|
95
|
-
# ---------------------------------------------------------------------------
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
def load_cached_tablet(
|
|
99
|
-
project: str,
|
|
100
|
-
text_id: str,
|
|
101
|
-
config,
|
|
102
|
-
cache_dir: str | None = None,
|
|
103
|
-
) -> "TabletRecord | None":
|
|
104
|
-
"""Load a cached tablet, rebuilding string reps only if config changed.
|
|
105
|
-
|
|
106
|
-
Two fast paths:
|
|
107
|
-
|
|
108
|
-
1. **Config match** — the cached fingerprint matches the current config.
|
|
109
|
-
The full record (including string representations) is returned as-is.
|
|
110
|
-
This is the fastest path.
|
|
111
|
-
|
|
112
|
-
2. **Config mismatch** — the words and metadata are reused, but string
|
|
113
|
-
representations are rebuilt with the current config. This avoids
|
|
114
|
-
the expensive CDL parsing + translation download.
|
|
115
|
-
|
|
116
|
-
Args:
|
|
117
|
-
project: ORACC project path, e.g. ``"saao/saa01"``.
|
|
118
|
-
text_id: Text identifier, e.g. ``"P334189"``.
|
|
119
|
-
config: ``RunConfig`` instance.
|
|
120
|
-
cache_dir: Custom cache directory (overrides settings).
|
|
121
|
-
|
|
122
|
-
Returns:
|
|
123
|
-
The TabletRecord (possibly with rebuilt strings), or ``None``.
|
|
124
|
-
"""
|
|
125
|
-
from oracc_parser.models.tablet import TabletRecord
|
|
126
|
-
from oracc_parser.parsing.parse_content import (
|
|
127
|
-
_add_word_level_representations,
|
|
128
|
-
_add_unicode_representation,
|
|
129
|
-
)
|
|
130
|
-
|
|
131
|
-
path = _tablet_path(project, text_id, cache_dir)
|
|
132
|
-
if not path.exists():
|
|
133
|
-
return None
|
|
134
|
-
|
|
135
|
-
try:
|
|
136
|
-
raw = path.read_text(encoding="utf-8")
|
|
137
|
-
wrapper = json.loads(raw)
|
|
138
|
-
|
|
139
|
-
# Handle both new wrapper format and legacy bare-record format
|
|
140
|
-
if "record" in wrapper and "config_fingerprint" in wrapper:
|
|
141
|
-
cached_fp = wrapper["config_fingerprint"]
|
|
142
|
-
record = TabletRecord.model_validate(wrapper["record"])
|
|
143
|
-
else:
|
|
144
|
-
# Legacy format (bare TabletRecord JSON) — always rebuild
|
|
145
|
-
cached_fp = None
|
|
146
|
-
record = TabletRecord.model_validate(wrapper)
|
|
147
|
-
|
|
148
|
-
current_fp = config_fingerprint(config)
|
|
149
|
-
|
|
150
|
-
if cached_fp == current_fp:
|
|
151
|
-
# Fast path: config matches → everything is valid
|
|
152
|
-
return record
|
|
153
|
-
|
|
154
|
-
# Config changed → rebuild string representations from cached words
|
|
155
|
-
record.content = _add_word_level_representations(
|
|
156
|
-
record.content, config.mask_pos, config.max_break_fraction
|
|
157
|
-
)
|
|
158
|
-
record.content = _add_unicode_representation(
|
|
159
|
-
record.content,
|
|
160
|
-
drop_missing=config.drop_missing,
|
|
161
|
-
drop_damaged=config.drop_damaged,
|
|
162
|
-
keep_segmentation=config.keep_word_segmentation,
|
|
163
|
-
)
|
|
164
|
-
return record
|
|
165
|
-
|
|
166
|
-
except Exception as e:
|
|
167
|
-
logger.warning(f"Corrupt cache file {path}, will re-parse: {e}")
|
|
168
|
-
path.unlink(missing_ok=True)
|
|
169
|
-
return None
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
def save_tablet_to_cache(
|
|
173
|
-
record: "TabletRecord",
|
|
174
|
-
project: str,
|
|
175
|
-
text_id: str,
|
|
176
|
-
config,
|
|
177
|
-
cache_dir: str | None = None,
|
|
178
|
-
) -> None:
|
|
179
|
-
"""Persist a TabletRecord to the JSON cache with a config fingerprint.
|
|
180
|
-
|
|
181
|
-
The saved file includes the config fingerprint so that on reload
|
|
182
|
-
we can skip string rebuilding when the config hasn't changed.
|
|
183
|
-
|
|
184
|
-
Args:
|
|
185
|
-
record: The parsed tablet to cache.
|
|
186
|
-
project: ORACC project path.
|
|
187
|
-
text_id: Text identifier.
|
|
188
|
-
config: ``RunConfig`` instance (its fingerprint is stored).
|
|
189
|
-
cache_dir: Custom cache directory.
|
|
190
|
-
"""
|
|
191
|
-
path = _tablet_path(project, text_id, cache_dir)
|
|
192
|
-
path.parent.mkdir(parents=True, exist_ok=True)
|
|
193
|
-
|
|
194
|
-
wrapper = {
|
|
195
|
-
"config_fingerprint": config_fingerprint(config),
|
|
196
|
-
"record": record.model_dump(mode="python"),
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
try:
|
|
200
|
-
path.write_text(
|
|
201
|
-
json.dumps(wrapper, indent=1, default=str, ensure_ascii=False),
|
|
202
|
-
encoding="utf-8",
|
|
203
|
-
)
|
|
204
|
-
except Exception as e:
|
|
205
|
-
logger.warning(f"Failed to write cache file {path}: {e}")
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
# ---------------------------------------------------------------------------
|
|
209
|
-
# Clear
|
|
210
|
-
# ---------------------------------------------------------------------------
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
def clear_project_cache(
|
|
214
|
-
project: str | None = None,
|
|
215
|
-
cache_dir: str | None = None,
|
|
216
|
-
) -> int:
|
|
217
|
-
"""Delete cached JSON files for a project (or all projects).
|
|
218
|
-
|
|
219
|
-
Args:
|
|
220
|
-
project: ORACC project path. ``None`` = clear everything.
|
|
221
|
-
cache_dir: Custom cache directory.
|
|
222
|
-
|
|
223
|
-
Returns:
|
|
224
|
-
Number of tablet JSON files deleted.
|
|
225
|
-
"""
|
|
226
|
-
base = _resolve_cache_dir(cache_dir) / "tablets"
|
|
227
|
-
if not base.exists():
|
|
228
|
-
return 0
|
|
229
|
-
|
|
230
|
-
if project:
|
|
231
|
-
target = base / project.replace("/", "-")
|
|
232
|
-
else:
|
|
233
|
-
target = base
|
|
234
|
-
|
|
235
|
-
if not target.exists():
|
|
236
|
-
return 0
|
|
237
|
-
|
|
238
|
-
count = 0
|
|
239
|
-
for f in target.rglob("*.json"):
|
|
240
|
-
f.unlink()
|
|
241
|
-
count += 1
|
|
242
|
-
|
|
243
|
-
# Clean up empty directories (bottom-up)
|
|
244
|
-
for d in sorted(target.rglob("*"), reverse=True):
|
|
245
|
-
if d.is_dir() and not any(d.iterdir()):
|
|
246
|
-
d.rmdir()
|
|
247
|
-
if project and target.exists() and not any(target.iterdir()):
|
|
248
|
-
target.rmdir()
|
|
249
|
-
|
|
250
|
-
logger.info(f"Cleared {count} cached tablet(s)")
|
|
251
|
-
return count
|
|
@@ -1,205 +0,0 @@
|
|
|
1
|
-
"""Tests for oracc_parser.cache — config-independent caching with on-load rebuild."""
|
|
2
|
-
|
|
3
|
-
from pathlib import Path
|
|
4
|
-
|
|
5
|
-
import pytest
|
|
6
|
-
|
|
7
|
-
from oracc_parser.cache import (
|
|
8
|
-
clear_project_cache,
|
|
9
|
-
config_fingerprint,
|
|
10
|
-
load_cached_tablet,
|
|
11
|
-
save_tablet_to_cache,
|
|
12
|
-
_tablet_path,
|
|
13
|
-
)
|
|
14
|
-
from oracc_parser.models.config import RunConfig
|
|
15
|
-
from oracc_parser.models.tablet import (
|
|
16
|
-
Sign,
|
|
17
|
-
TabletRecord,
|
|
18
|
-
TabletContent,
|
|
19
|
-
TabletMetadata,
|
|
20
|
-
TextStringRepresentation,
|
|
21
|
-
Word,
|
|
22
|
-
WordPOSInfo,
|
|
23
|
-
WordSigns,
|
|
24
|
-
)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
@pytest.fixture
|
|
28
|
-
def cache_dir(tmp_path):
|
|
29
|
-
return str(tmp_path / "test_cache")
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
@pytest.fixture
|
|
33
|
-
def default_config():
|
|
34
|
-
return RunConfig()
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
@pytest.fixture
|
|
38
|
-
def sample_record():
|
|
39
|
-
"""Create a TabletRecord with words that allow string regeneration."""
|
|
40
|
-
record = TabletRecord()
|
|
41
|
-
record.metadata = TabletMetadata(
|
|
42
|
-
identifier="test_tablet",
|
|
43
|
-
project="saao/saa01",
|
|
44
|
-
id_text="P334189",
|
|
45
|
-
genre="Letter",
|
|
46
|
-
)
|
|
47
|
-
|
|
48
|
-
# Create words with signs so unicode + text reps can be rebuilt
|
|
49
|
-
words = [
|
|
50
|
-
Word(
|
|
51
|
-
frag="a-na",
|
|
52
|
-
ref="P334189.1.1",
|
|
53
|
-
lemma_form="ana",
|
|
54
|
-
norm="ana",
|
|
55
|
-
raw_pos="PRP",
|
|
56
|
-
lang="akk-x-neoass",
|
|
57
|
-
line=1,
|
|
58
|
-
normalized_pos=WordPOSInfo(meaning="to", normalized_pos="PRP"),
|
|
59
|
-
sign_dictionaries=WordSigns(
|
|
60
|
-
reading="a-na",
|
|
61
|
-
break_percentage=0.0,
|
|
62
|
-
signs=[
|
|
63
|
-
Sign(role="phonetic", meaning="a", breakage="complete",
|
|
64
|
-
unicode_version="\U00012000"),
|
|
65
|
-
Sign(role="phonetic", meaning="na", breakage="complete",
|
|
66
|
-
unicode_version="\U0001223E"),
|
|
67
|
-
],
|
|
68
|
-
),
|
|
69
|
-
),
|
|
70
|
-
Word(
|
|
71
|
-
frag="{m}sargon",
|
|
72
|
-
ref="P334189.1.2",
|
|
73
|
-
lemma_form="sargon",
|
|
74
|
-
norm="sargon",
|
|
75
|
-
raw_pos="PN",
|
|
76
|
-
lang="akk-x-neoass",
|
|
77
|
-
line=1,
|
|
78
|
-
normalized_pos=WordPOSInfo(
|
|
79
|
-
meaning="Sargon", normalized_pos="PN",
|
|
80
|
-
to_mask=False, mask_as=""
|
|
81
|
-
),
|
|
82
|
-
sign_dictionaries=WordSigns(
|
|
83
|
-
reading="sargon", break_percentage=0.0, signs=[]
|
|
84
|
-
),
|
|
85
|
-
),
|
|
86
|
-
]
|
|
87
|
-
|
|
88
|
-
record.content = TabletContent(
|
|
89
|
-
english_translation="To Sargon",
|
|
90
|
-
words=words,
|
|
91
|
-
transliterated_str_representation=TextStringRepresentation(
|
|
92
|
-
text="a-na {m}sargon", total_tokens=2, tokens_without_broken=2,
|
|
93
|
-
),
|
|
94
|
-
)
|
|
95
|
-
return record
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
# ---------------------------------------------------------------------------
|
|
99
|
-
# Path structure — simple, no fingerprinting
|
|
100
|
-
# ---------------------------------------------------------------------------
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
class TestTabletPath:
|
|
104
|
-
def test_path_structure(self, cache_dir):
|
|
105
|
-
path = _tablet_path("saao/saa01", "P334189", cache_dir)
|
|
106
|
-
expected = Path(cache_dir) / "tablets" / "saao-saa01" / "P334189.json"
|
|
107
|
-
assert path == expected
|
|
108
|
-
|
|
109
|
-
def test_one_path_per_tablet(self, cache_dir):
|
|
110
|
-
"""Path is the same regardless of config."""
|
|
111
|
-
p1 = _tablet_path("blms", "P1", cache_dir)
|
|
112
|
-
p2 = _tablet_path("blms", "P1", cache_dir)
|
|
113
|
-
assert p1 == p2
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
# ---------------------------------------------------------------------------
|
|
117
|
-
# Save / Load roundtrip
|
|
118
|
-
# ---------------------------------------------------------------------------
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
class TestSaveAndLoad:
|
|
122
|
-
def test_roundtrip_preserves_words(self, cache_dir, default_config, sample_record):
|
|
123
|
-
"""Words and metadata survive cache roundtrip."""
|
|
124
|
-
save_tablet_to_cache(sample_record, "saao/saa01", "P334189", default_config, cache_dir)
|
|
125
|
-
loaded = load_cached_tablet("saao/saa01", "P334189", default_config, cache_dir)
|
|
126
|
-
|
|
127
|
-
assert loaded is not None
|
|
128
|
-
assert loaded.metadata.identifier == "test_tablet"
|
|
129
|
-
assert loaded.metadata.genre == "Letter"
|
|
130
|
-
assert loaded.content.english_translation == "To Sargon"
|
|
131
|
-
assert len(loaded.content.words) == 2
|
|
132
|
-
assert loaded.content.words[0].frag == "a-na"
|
|
133
|
-
assert loaded.content.words[1].frag == "{m}sargon"
|
|
134
|
-
|
|
135
|
-
def test_string_reps_rebuilt_on_load(self, cache_dir, sample_record):
|
|
136
|
-
"""String reps are regenerated from words, not loaded from cache blindly."""
|
|
137
|
-
save_tablet_to_cache(sample_record, "saao/saa01", "P1", RunConfig(), cache_dir)
|
|
138
|
-
|
|
139
|
-
# Load with mask_pos=["PN"] — should mask the PN word
|
|
140
|
-
config_masked = RunConfig(mask_pos=["PN"])
|
|
141
|
-
loaded = load_cached_tablet("saao/saa01", "P1", config_masked, cache_dir)
|
|
142
|
-
|
|
143
|
-
assert loaded is not None
|
|
144
|
-
# The transliteration should now contain [PN] mask
|
|
145
|
-
t_rep = loaded.content.transliterated_str_representation
|
|
146
|
-
assert t_rep is not None
|
|
147
|
-
assert "PN" in t_rep.text
|
|
148
|
-
|
|
149
|
-
def test_same_cache_different_configs(self, cache_dir, sample_record):
|
|
150
|
-
"""Same cached file serves different configs correctly."""
|
|
151
|
-
save_tablet_to_cache(sample_record, "proj", "P1", RunConfig(), cache_dir)
|
|
152
|
-
|
|
153
|
-
# Load with default (no masking)
|
|
154
|
-
loaded_default = load_cached_tablet("proj", "P1", RunConfig(), cache_dir)
|
|
155
|
-
# Load with PN masking
|
|
156
|
-
loaded_masked = load_cached_tablet("proj", "P1", RunConfig(mask_pos=["PN"]), cache_dir)
|
|
157
|
-
|
|
158
|
-
assert loaded_default is not None
|
|
159
|
-
assert loaded_masked is not None
|
|
160
|
-
|
|
161
|
-
# Both exist, but transliteration text should differ
|
|
162
|
-
text_default = loaded_default.content.transliterated_str_representation.text
|
|
163
|
-
text_masked = loaded_masked.content.transliterated_str_representation.text
|
|
164
|
-
assert text_default != text_masked
|
|
165
|
-
|
|
166
|
-
def test_cache_miss(self, cache_dir, default_config):
|
|
167
|
-
result = load_cached_tablet("saao/saa01", "NONEXISTENT", default_config, cache_dir)
|
|
168
|
-
assert result is None
|
|
169
|
-
|
|
170
|
-
def test_corrupt_file_returns_none(self, cache_dir, default_config):
|
|
171
|
-
path = _tablet_path("saao/saa01", "CORRUPT", cache_dir)
|
|
172
|
-
path.parent.mkdir(parents=True, exist_ok=True)
|
|
173
|
-
path.write_text("NOT VALID JSON {{{{", encoding="utf-8")
|
|
174
|
-
|
|
175
|
-
result = load_cached_tablet("saao/saa01", "CORRUPT", default_config, cache_dir)
|
|
176
|
-
assert result is None
|
|
177
|
-
assert not path.exists()
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
# ---------------------------------------------------------------------------
|
|
181
|
-
# Clear cache
|
|
182
|
-
# ---------------------------------------------------------------------------
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
class TestClearCache:
|
|
186
|
-
def test_clear_project(self, cache_dir, default_config, sample_record):
|
|
187
|
-
save_tablet_to_cache(sample_record, "saao/saa01", "P1", default_config, cache_dir)
|
|
188
|
-
save_tablet_to_cache(sample_record, "saao/saa01", "P2", default_config, cache_dir)
|
|
189
|
-
save_tablet_to_cache(sample_record, "blms", "P3", default_config, cache_dir)
|
|
190
|
-
|
|
191
|
-
deleted = clear_project_cache("saao/saa01", cache_dir)
|
|
192
|
-
assert deleted == 2
|
|
193
|
-
assert load_cached_tablet("blms", "P3", default_config, cache_dir) is not None
|
|
194
|
-
assert load_cached_tablet("saao/saa01", "P1", default_config, cache_dir) is None
|
|
195
|
-
|
|
196
|
-
def test_clear_all(self, cache_dir, default_config, sample_record):
|
|
197
|
-
save_tablet_to_cache(sample_record, "saao/saa01", "P1", default_config, cache_dir)
|
|
198
|
-
save_tablet_to_cache(sample_record, "blms", "P2", default_config, cache_dir)
|
|
199
|
-
|
|
200
|
-
deleted = clear_project_cache(project=None, cache_dir=cache_dir)
|
|
201
|
-
assert deleted == 2
|
|
202
|
-
assert load_cached_tablet("saao/saa01", "P1", default_config, cache_dir) is None
|
|
203
|
-
|
|
204
|
-
def test_clear_empty(self, cache_dir):
|
|
205
|
-
assert clear_project_cache(project=None, cache_dir=cache_dir) == 0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/state_supergroup_mapping.csv
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|