oracc-parser 0.1.2__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/PKG-INFO +45 -22
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/README.md +44 -21
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/__init__.py +1 -4
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/cli.py +27 -52
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/download/fetch_data.py +4 -4
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/download/oracc_download.py +0 -68
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/languages.csv +1 -1
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/io/word_csv.py +0 -69
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/metadata/populate.py +35 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/models/config.py +0 -25
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/models/tablet.py +7 -2
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/parsing/translation.py +3 -4
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/pipeline.py +73 -133
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/settings.py +17 -12
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser.egg-info/PKG-INFO +45 -22
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser.egg-info/SOURCES.txt +0 -2
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/pyproject.toml +1 -1
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/tests/test_models.py +0 -2
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/tests/test_settings.py +2 -8
- oracc_parser-0.1.2/oracc_parser/cache.py +0 -251
- oracc_parser-0.1.2/tests/test_cache.py +0 -205
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/LICENSE +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/constants.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/download/__init__.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/download/extract_jsons.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/download/pleiades.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/__init__.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/grouped_oracc_metadata_columns.csv +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/period_mapping.csv +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/pos_tags.csv +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/projects_metadata.csv +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/provenience.csv +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/raw_archive_values.csv +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/sign_readings.csv +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/state_supergroup_mapping.csv +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/export/__init__.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/export/to_jsonl.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/io/__init__.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/metadata/__init__.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/metadata/archive.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/models/__init__.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/parsing/__init__.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/parsing/parse_content.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/parsing/parse_signs.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/parsing/parse_words.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/parsing/text_builder.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/utils/__init__.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/utils/logger.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/utils/paths.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/utils/unicode.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser.egg-info/dependency_links.txt +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser.egg-info/entry_points.txt +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser.egg-info/requires.txt +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser.egg-info/top_level.txt +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/setup.cfg +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/tests/test_constants.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/tests/test_export.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/tests/test_parsing.py +0 -0
- {oracc_parser-0.1.2 → oracc_parser-0.1.4}/tests/test_pipeline.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: oracc-parser
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.4
|
|
4
4
|
Summary: Download and parse ORACC cuneiform text projects into ML-ready formats
|
|
5
5
|
Author: Avital Romach, Shahar Spencer, Claude Sonnet 4.6
|
|
6
6
|
License-Expression: MIT
|
|
@@ -40,13 +40,34 @@ A Python tool to download and parse [ORACC](http://oracc.museum.upenn.edu/) cune
|
|
|
40
40
|
|
|
41
41
|
## Features
|
|
42
42
|
|
|
43
|
-
- **Download** — Fetch project ZIPs directly from ORACC
|
|
44
|
-
- **Parse** — Convert raw ORACC JSON into structured data
|
|
43
|
+
- **Download** — Fetch project ZIPs from preprocessed files on Zenodo or directly from ORACC
|
|
44
|
+
- **Parse** — Convert raw ORACC JSON / preprocessed CSVs into structured data and create machine-learning-ready text representations (in transliteration, normalization, lemmatization, or Unicode cuneiform)
|
|
45
|
+
- **Configure** — Control handling of broken signs and POS masking
|
|
45
46
|
- **Export** — Save datasets as JSONL, CSV, or pandas DataFrames
|
|
46
|
-
|
|
47
|
+
|
|
48
|
+
## How to Cite
|
|
49
|
+
|
|
50
|
+
If you use oracc-parser in your research, please cite:
|
|
51
|
+
|
|
52
|
+
```bibtex
|
|
53
|
+
@software{romach_oracc_parser_2026,
|
|
54
|
+
author = {Romach, Avital and Spencer, Shahar},
|
|
55
|
+
title = {oracc-parser: A Python toolkit for downloading and parsing ORACC cuneiform text projects},
|
|
56
|
+
year = {2026},
|
|
57
|
+
publisher = {Zenodo},
|
|
58
|
+
version = {0.1.4},
|
|
59
|
+
doi = {10.5281/zenodo.18643122},
|
|
60
|
+
url = {https://doi.org/10.5281/zenodo.18643122}
|
|
61
|
+
}
|
|
62
|
+
```
|
|
47
63
|
|
|
48
64
|
## Installation
|
|
49
65
|
|
|
66
|
+
```bash
|
|
67
|
+
pip install oracc-parser
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Or for development:
|
|
50
71
|
|
|
51
72
|
```bash
|
|
52
73
|
git clone https://github.com/shaharspencer/oracc-parser.git
|
|
@@ -56,15 +77,15 @@ pip install -e ".[dev]"
|
|
|
56
77
|
|
|
57
78
|
## Getting Started — Notebooks
|
|
58
79
|
|
|
59
|
-
The easiest way to explore oracc-parser is through the interactive notebooks.
|
|
60
|
-
Start with notebook 01 — it downloads all the data you need from Zenodo automatically.
|
|
80
|
+
The easiest way to explore oracc-parser is through the interactive notebooks, that go through the main functions.
|
|
61
81
|
|
|
62
|
-
| Notebook | What you'll learn |
|
|
63
|
-
|
|
64
|
-
| [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project
|
|
65
|
-
| [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) |
|
|
66
|
-
| [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets |
|
|
67
|
-
| [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset |
|
|
82
|
+
| Notebook | What you'll learn | Open in Colab |
|
|
83
|
+
|---|---|---|
|
|
84
|
+
| [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project → explore transliterations, translations, and metadata → export | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/01_quickstart.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
85
|
+
| [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/02_reference_data.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
86
|
+
| [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/03_configure_and_export.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
87
|
+
| [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/04_oracc_json_processing.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
88
|
+
| [`05_translations.ipynb`](notebooks/05_translations.ipynb) | Where translations come from, how to download the translation cache from Zenodo, and how to enable translations in the parsing pipeline | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/05_translations.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
68
89
|
|
|
69
90
|
```bash
|
|
70
91
|
pip install oracc-parser[notebooks]
|
|
@@ -74,14 +95,22 @@ jupyter notebook notebooks/
|
|
|
74
95
|
## Quick Example
|
|
75
96
|
|
|
76
97
|
```python
|
|
98
|
+
from oracc_parser.download.fetch_data import fetch_data
|
|
77
99
|
from oracc_parser import parse_project, RunConfig, get_full_flat_table
|
|
78
100
|
|
|
79
|
-
#
|
|
101
|
+
# Step 1: download catalogues from Zenodo (run once)
|
|
102
|
+
fetch_data()
|
|
103
|
+
|
|
104
|
+
# Step 2: parse a project — word CSVs fetched from Zenodo on first call,
|
|
105
|
+
# read from disk on subsequent calls
|
|
80
106
|
records = parse_project("saao/saa01", config=RunConfig(limit=5))
|
|
81
107
|
|
|
82
108
|
# Get a flat DataFrame — no nesting, ready for analysis
|
|
83
109
|
df = get_full_flat_table(records)
|
|
84
110
|
df.to_json("dataset.jsonl", orient="records", lines=True)
|
|
111
|
+
|
|
112
|
+
# For a project not on Zenodo, download directly from ORACC servers:
|
|
113
|
+
records = parse_project("my/project", download_from_oracc_server=True)
|
|
85
114
|
```
|
|
86
115
|
|
|
87
116
|
## Configuration
|
|
@@ -113,9 +142,10 @@ operating at different granularities and affecting different outputs:
|
|
|
113
142
|
|
|
114
143
|
> **Note:** Because word-level and sign-level filtering use different thresholds
|
|
115
144
|
> and different granularities, **the text outputs and the Unicode cuneiform output
|
|
116
|
-
> are not
|
|
145
|
+
> are not aligned**. A word kept in the transliteration (because its
|
|
117
146
|
> average damage is below `max_break_fraction`) may still have individual signs
|
|
118
147
|
> dropped from the Unicode output if `drop_missing` / `drop_damaged` are enabled.
|
|
148
|
+
> None of these filters affect the translations.
|
|
119
149
|
|
|
120
150
|
### Other options
|
|
121
151
|
|
|
@@ -124,8 +154,6 @@ operating at different granularities and affecting different outputs:
|
|
|
124
154
|
| `limit` | `None` | Only parse the first N texts (useful for testing) |
|
|
125
155
|
| `keep_word_segmentation` | `True` | Preserve word boundaries in Unicode cuneiform output |
|
|
126
156
|
| `mask_pos` | `[]` | Replace words of certain POS tags with the tag name |
|
|
127
|
-
| `languages` | `["Akkadian"]` | Which languages to include when downloading projects |
|
|
128
|
-
| `use_cache` | `True` | Use cached results if available |
|
|
129
157
|
|
|
130
158
|
All reference data is bundled with the package, so you don't need to configure external paths unless you are customizing `oracc_parser.settings`.
|
|
131
159
|
|
|
@@ -138,7 +166,7 @@ oracc-parser parse --project saao/saa01 --limit 5 --format jsonl --output saa01.
|
|
|
138
166
|
|
|
139
167
|
## Heavy Data (Zenodo)
|
|
140
168
|
|
|
141
|
-
Large data files (ORACC ZIPs, cached translations, Pleiades data) are on Zenodo:
|
|
169
|
+
Large data files (ORACC ZIPs, word-level CSVs, cached translations, Pleiades data) are on Zenodo:
|
|
142
170
|
|
|
143
171
|
[](https://doi.org/10.5281/zenodo.18643122)
|
|
144
172
|
|
|
@@ -152,11 +180,6 @@ python scripts/download_zenodo_data.py
|
|
|
152
180
|
pytest tests/ -v # 98 tests
|
|
153
181
|
```
|
|
154
182
|
|
|
155
|
-
## Known Limitations
|
|
156
|
-
|
|
157
|
-
- **Chronology**: Period-to-year normalization is optimized for the **1st Millennium BCE**.
|
|
158
|
-
- **Language**: Parsing is primarily validated on **Akkadian** projects.
|
|
159
|
-
|
|
160
183
|
## License
|
|
161
184
|
|
|
162
185
|
MIT — see [LICENSE](LICENSE).
|
|
@@ -4,13 +4,34 @@ A Python tool to download and parse [ORACC](http://oracc.museum.upenn.edu/) cune
|
|
|
4
4
|
|
|
5
5
|
## Features
|
|
6
6
|
|
|
7
|
-
- **Download** — Fetch project ZIPs directly from ORACC
|
|
8
|
-
- **Parse** — Convert raw ORACC JSON into structured data
|
|
7
|
+
- **Download** — Fetch project ZIPs from preprocessed files on Zenodo or directly from ORACC
|
|
8
|
+
- **Parse** — Convert raw ORACC JSON / preprocessed CSVs into structured data and create machine-learning-ready text representations (in transliteration, normalization, lemmatization, or Unicode cuneiform)
|
|
9
|
+
- **Configure** — Control handling of broken signs and POS masking
|
|
9
10
|
- **Export** — Save datasets as JSONL, CSV, or pandas DataFrames
|
|
10
|
-
|
|
11
|
+
|
|
12
|
+
## How to Cite
|
|
13
|
+
|
|
14
|
+
If you use oracc-parser in your research, please cite:
|
|
15
|
+
|
|
16
|
+
```bibtex
|
|
17
|
+
@software{romach_oracc_parser_2026,
|
|
18
|
+
author = {Romach, Avital and Spencer, Shahar},
|
|
19
|
+
title = {oracc-parser: A Python toolkit for downloading and parsing ORACC cuneiform text projects},
|
|
20
|
+
year = {2026},
|
|
21
|
+
publisher = {Zenodo},
|
|
22
|
+
version = {0.1.4},
|
|
23
|
+
doi = {10.5281/zenodo.18643122},
|
|
24
|
+
url = {https://doi.org/10.5281/zenodo.18643122}
|
|
25
|
+
}
|
|
26
|
+
```
|
|
11
27
|
|
|
12
28
|
## Installation
|
|
13
29
|
|
|
30
|
+
```bash
|
|
31
|
+
pip install oracc-parser
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Or for development:
|
|
14
35
|
|
|
15
36
|
```bash
|
|
16
37
|
git clone https://github.com/shaharspencer/oracc-parser.git
|
|
@@ -20,15 +41,15 @@ pip install -e ".[dev]"
|
|
|
20
41
|
|
|
21
42
|
## Getting Started — Notebooks
|
|
22
43
|
|
|
23
|
-
The easiest way to explore oracc-parser is through the interactive notebooks.
|
|
24
|
-
Start with notebook 01 — it downloads all the data you need from Zenodo automatically.
|
|
44
|
+
The easiest way to explore oracc-parser is through the interactive notebooks, that go through the main functions.
|
|
25
45
|
|
|
26
|
-
| Notebook | What you'll learn |
|
|
27
|
-
|
|
28
|
-
| [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project
|
|
29
|
-
| [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) |
|
|
30
|
-
| [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets |
|
|
31
|
-
| [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset |
|
|
46
|
+
| Notebook | What you'll learn | Open in Colab |
|
|
47
|
+
|---|---|---|
|
|
48
|
+
| [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project → explore transliterations, translations, and metadata → export | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/01_quickstart.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
49
|
+
| [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/02_reference_data.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
50
|
+
| [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/03_configure_and_export.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
51
|
+
| [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/04_oracc_json_processing.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
52
|
+
| [`05_translations.ipynb`](notebooks/05_translations.ipynb) | Where translations come from, how to download the translation cache from Zenodo, and how to enable translations in the parsing pipeline | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/05_translations.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
|
|
32
53
|
|
|
33
54
|
```bash
|
|
34
55
|
pip install oracc-parser[notebooks]
|
|
@@ -38,14 +59,22 @@ jupyter notebook notebooks/
|
|
|
38
59
|
## Quick Example
|
|
39
60
|
|
|
40
61
|
```python
|
|
62
|
+
from oracc_parser.download.fetch_data import fetch_data
|
|
41
63
|
from oracc_parser import parse_project, RunConfig, get_full_flat_table
|
|
42
64
|
|
|
43
|
-
#
|
|
65
|
+
# Step 1: download catalogues from Zenodo (run once)
|
|
66
|
+
fetch_data()
|
|
67
|
+
|
|
68
|
+
# Step 2: parse a project — word CSVs fetched from Zenodo on first call,
|
|
69
|
+
# read from disk on subsequent calls
|
|
44
70
|
records = parse_project("saao/saa01", config=RunConfig(limit=5))
|
|
45
71
|
|
|
46
72
|
# Get a flat DataFrame — no nesting, ready for analysis
|
|
47
73
|
df = get_full_flat_table(records)
|
|
48
74
|
df.to_json("dataset.jsonl", orient="records", lines=True)
|
|
75
|
+
|
|
76
|
+
# For a project not on Zenodo, download directly from ORACC servers:
|
|
77
|
+
records = parse_project("my/project", download_from_oracc_server=True)
|
|
49
78
|
```
|
|
50
79
|
|
|
51
80
|
## Configuration
|
|
@@ -77,9 +106,10 @@ operating at different granularities and affecting different outputs:
|
|
|
77
106
|
|
|
78
107
|
> **Note:** Because word-level and sign-level filtering use different thresholds
|
|
79
108
|
> and different granularities, **the text outputs and the Unicode cuneiform output
|
|
80
|
-
> are not
|
|
109
|
+
> are not aligned**. A word kept in the transliteration (because its
|
|
81
110
|
> average damage is below `max_break_fraction`) may still have individual signs
|
|
82
111
|
> dropped from the Unicode output if `drop_missing` / `drop_damaged` are enabled.
|
|
112
|
+
> None of these filters affect the translations.
|
|
83
113
|
|
|
84
114
|
### Other options
|
|
85
115
|
|
|
@@ -88,8 +118,6 @@ operating at different granularities and affecting different outputs:
|
|
|
88
118
|
| `limit` | `None` | Only parse the first N texts (useful for testing) |
|
|
89
119
|
| `keep_word_segmentation` | `True` | Preserve word boundaries in Unicode cuneiform output |
|
|
90
120
|
| `mask_pos` | `[]` | Replace words of certain POS tags with the tag name |
|
|
91
|
-
| `languages` | `["Akkadian"]` | Which languages to include when downloading projects |
|
|
92
|
-
| `use_cache` | `True` | Use cached results if available |
|
|
93
121
|
|
|
94
122
|
All reference data is bundled with the package, so you don't need to configure external paths unless you are customizing `oracc_parser.settings`.
|
|
95
123
|
|
|
@@ -102,7 +130,7 @@ oracc-parser parse --project saao/saa01 --limit 5 --format jsonl --output saa01.
|
|
|
102
130
|
|
|
103
131
|
## Heavy Data (Zenodo)
|
|
104
132
|
|
|
105
|
-
Large data files (ORACC ZIPs, cached translations, Pleiades data) are on Zenodo:
|
|
133
|
+
Large data files (ORACC ZIPs, word-level CSVs, cached translations, Pleiades data) are on Zenodo:
|
|
106
134
|
|
|
107
135
|
[](https://doi.org/10.5281/zenodo.18643122)
|
|
108
136
|
|
|
@@ -116,11 +144,6 @@ python scripts/download_zenodo_data.py
|
|
|
116
144
|
pytest tests/ -v # 98 tests
|
|
117
145
|
```
|
|
118
146
|
|
|
119
|
-
## Known Limitations
|
|
120
|
-
|
|
121
|
-
- **Chronology**: Period-to-year normalization is optimized for the **1st Millennium BCE**.
|
|
122
|
-
- **Language**: Parsing is primarily validated on **Akkadian** projects.
|
|
123
|
-
|
|
124
147
|
## License
|
|
125
148
|
|
|
126
149
|
MIT — see [LICENSE](LICENSE).
|
|
@@ -3,14 +3,13 @@ oracc-parser: Download and parse ORACC cuneiform text projects.
|
|
|
3
3
|
"""
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
|
-
__version__ = "0.1.
|
|
6
|
+
__version__ = "0.1.4"
|
|
7
7
|
|
|
8
8
|
# Public re-exports for convenience
|
|
9
9
|
from oracc_parser.pipeline import ( # noqa: F401
|
|
10
10
|
export_to_csv,
|
|
11
11
|
export_to_jsonl,
|
|
12
12
|
parse_project,
|
|
13
|
-
parse_project_from_word_csvs,
|
|
14
13
|
records_to_word_dataframes,
|
|
15
14
|
save_project_catalogue,
|
|
16
15
|
load_project_catalogue,
|
|
@@ -24,8 +23,6 @@ from oracc_parser.pipeline import ( # noqa: F401
|
|
|
24
23
|
get_full_flat_table,
|
|
25
24
|
)
|
|
26
25
|
from oracc_parser.io.word_csv import ( # noqa: F401
|
|
27
|
-
load_word_csvs_from_dir,
|
|
28
|
-
load_word_csvs_from_zenodo,
|
|
29
26
|
save_word_csv,
|
|
30
27
|
)
|
|
31
28
|
from oracc_parser.models.config import RunConfig # noqa: F401
|
|
@@ -6,8 +6,6 @@ Usage:
|
|
|
6
6
|
oracc-parser download --lang akkadian
|
|
7
7
|
oracc-parser parse --project saao/saa01 --format jsonl --output data.jsonl
|
|
8
8
|
oracc-parser parse --project saao/saa01 --limit 5 --format csv --output data.csv
|
|
9
|
-
oracc-parser clear-cache # clear all cached tablets
|
|
10
|
-
oracc-parser clear-cache --project saao # clear one project's cache
|
|
11
9
|
"""
|
|
12
10
|
from __future__ import annotations
|
|
13
11
|
|
|
@@ -15,7 +13,6 @@ import argparse
|
|
|
15
13
|
import sys
|
|
16
14
|
|
|
17
15
|
from oracc_parser.pipeline import export_to_csv, export_to_jsonl, parse_project
|
|
18
|
-
from oracc_parser.download.oracc_download import download_projects
|
|
19
16
|
from oracc_parser.models.config import RunConfig
|
|
20
17
|
from oracc_parser.utils.logger import get_logger
|
|
21
18
|
|
|
@@ -31,15 +28,8 @@ def main(argv: list[str] | None = None):
|
|
|
31
28
|
subparsers = parser.add_subparsers(dest="command", help="Available commands")
|
|
32
29
|
|
|
33
30
|
# --------------- download ---------------
|
|
34
|
-
dl = subparsers.add_parser("download", help="Download ORACC project
|
|
35
|
-
dl.add_argument("--project", "-p", help="Project path, e.g. saao/saa01")
|
|
36
|
-
dl.add_argument(
|
|
37
|
-
"--lang",
|
|
38
|
-
"-l",
|
|
39
|
-
default="Akkadian",
|
|
40
|
-
help="Language filter for bulk download (default: Akkadian)",
|
|
41
|
-
)
|
|
42
|
-
dl.add_argument("--limit", "-n", type=int, help="Download only first N projects")
|
|
31
|
+
dl = subparsers.add_parser("download", help="Download a single ORACC project ZIP from the ORACC servers")
|
|
32
|
+
dl.add_argument("--project", "-p", required=True, help="Project path, e.g. saao/saa01")
|
|
43
33
|
|
|
44
34
|
# --------------- parse ---------------
|
|
45
35
|
ps = subparsers.add_parser("parse", help="Parse a project and export results")
|
|
@@ -71,8 +61,22 @@ def main(argv: list[str] | None = None):
|
|
|
71
61
|
default=[],
|
|
72
62
|
help="POS tags to mask (e.g. PN DN GN)",
|
|
73
63
|
)
|
|
74
|
-
ps.add_argument(
|
|
75
|
-
|
|
64
|
+
ps.add_argument(
|
|
65
|
+
"--max-break-fraction",
|
|
66
|
+
type=float,
|
|
67
|
+
default=1.0,
|
|
68
|
+
help="Word-level break filter: words with more than this fraction of broken signs are replaced with X (default: 1.0 = keep all)",
|
|
69
|
+
)
|
|
70
|
+
ps.add_argument(
|
|
71
|
+
"--fetch-translations",
|
|
72
|
+
action="store_true",
|
|
73
|
+
help="Fetch English translations (from cache or ORACC live)",
|
|
74
|
+
)
|
|
75
|
+
ps.add_argument(
|
|
76
|
+
"--from-oracc",
|
|
77
|
+
action="store_true",
|
|
78
|
+
help="Download from ORACC live servers instead of Zenodo (for projects not in the dataset)",
|
|
79
|
+
)
|
|
76
80
|
|
|
77
81
|
# --------------- fetch-data ---------------
|
|
78
82
|
fd = subparsers.add_parser("fetch-data", help="Download pre-processed data from Zenodo")
|
|
@@ -82,13 +86,6 @@ def main(argv: list[str] | None = None):
|
|
|
82
86
|
# --------------- info ---------------
|
|
83
87
|
subparsers.add_parser("info", help="Show bundled reference data summary")
|
|
84
88
|
|
|
85
|
-
# --------------- clear-cache ---------------
|
|
86
|
-
cc = subparsers.add_parser("clear-cache", help="Delete cached parsed tablets")
|
|
87
|
-
cc.add_argument(
|
|
88
|
-
"--project", "-p",
|
|
89
|
-
help="Only clear cache for this project (default: clear all)",
|
|
90
|
-
)
|
|
91
|
-
|
|
92
89
|
args = parser.parse_args(argv)
|
|
93
90
|
|
|
94
91
|
if args.command == "download":
|
|
@@ -99,8 +96,6 @@ def main(argv: list[str] | None = None):
|
|
|
99
96
|
_cmd_parse(args)
|
|
100
97
|
elif args.command == "info":
|
|
101
98
|
_cmd_info()
|
|
102
|
-
elif args.command == "clear-cache":
|
|
103
|
-
_cmd_clear_cache(args)
|
|
104
99
|
else:
|
|
105
100
|
parser.print_help()
|
|
106
101
|
sys.exit(1)
|
|
@@ -119,23 +114,14 @@ def _cmd_fetch_data(args):
|
|
|
119
114
|
|
|
120
115
|
def _cmd_download(args):
|
|
121
116
|
"""Handle the download command."""
|
|
122
|
-
|
|
123
|
-
languages=[args.lang] if args.lang else ["Akkadian"],
|
|
124
|
-
limit=args.limit,
|
|
125
|
-
)
|
|
126
|
-
|
|
127
|
-
if args.project:
|
|
128
|
-
from oracc_parser.download.oracc_download import download_zip
|
|
117
|
+
from oracc_parser.download.oracc_download import download_zip
|
|
129
118
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
else:
|
|
134
|
-
print("Download failed.", file=sys.stderr)
|
|
135
|
-
sys.exit(1)
|
|
119
|
+
path = download_zip(args.project)
|
|
120
|
+
if path:
|
|
121
|
+
print(f"Downloaded: {path}")
|
|
136
122
|
else:
|
|
137
|
-
|
|
138
|
-
|
|
123
|
+
print("Download failed.", file=sys.stderr)
|
|
124
|
+
sys.exit(1)
|
|
139
125
|
|
|
140
126
|
|
|
141
127
|
def _cmd_parse(args):
|
|
@@ -144,12 +130,13 @@ def _cmd_parse(args):
|
|
|
144
130
|
drop_missing=args.drop_missing,
|
|
145
131
|
drop_damaged=args.drop_damaged,
|
|
146
132
|
mask_pos=args.mask_pos,
|
|
147
|
-
use_cache=not args.no_cache,
|
|
148
133
|
limit=args.limit,
|
|
134
|
+
max_break_fraction=args.max_break_fraction,
|
|
135
|
+
fetch_translations=args.fetch_translations,
|
|
149
136
|
)
|
|
150
137
|
|
|
151
138
|
records = parse_project(
|
|
152
|
-
args.project, config=config,
|
|
139
|
+
args.project, config=config, download_from_oracc_server=args.from_oracc
|
|
153
140
|
)
|
|
154
141
|
|
|
155
142
|
if not records:
|
|
@@ -185,17 +172,5 @@ def _cmd_info():
|
|
|
185
172
|
print(f" {name}: Error loading - {e}", file=sys.stderr)
|
|
186
173
|
|
|
187
174
|
|
|
188
|
-
def _cmd_clear_cache(args):
|
|
189
|
-
"""Handle the clear-cache command."""
|
|
190
|
-
from oracc_parser.cache import clear_project_cache
|
|
191
|
-
|
|
192
|
-
count = clear_project_cache(project=args.project)
|
|
193
|
-
if count:
|
|
194
|
-
scope = f"project '{args.project}'" if args.project else "all projects"
|
|
195
|
-
print(f"Cleared {count} cached tablet(s) for {scope}.")
|
|
196
|
-
else:
|
|
197
|
-
print("No cached tablets found.")
|
|
198
|
-
|
|
199
|
-
|
|
200
175
|
if __name__ == "__main__":
|
|
201
176
|
main()
|
|
@@ -22,7 +22,7 @@ from pathlib import Path
|
|
|
22
22
|
import requests
|
|
23
23
|
from tqdm import tqdm
|
|
24
24
|
|
|
25
|
-
from oracc_parser.settings import zenodo_url, data_dir, jsonzip_dir, word_csv_dir, catalogue_dir,
|
|
25
|
+
from oracc_parser.settings import zenodo_url, data_dir, jsonzip_dir, word_csv_dir, catalogue_dir, translations_dir
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
def download_file(url: str, dest: Path, desc: str = "") -> Path:
|
|
@@ -235,7 +235,7 @@ def fetch_data(
|
|
|
235
235
|
|
|
236
236
|
def _already_extracted(filename: str) -> bool:
|
|
237
237
|
if filename == "oracc_html_translations.zip":
|
|
238
|
-
return
|
|
238
|
+
return (translations_dir() / ".translations_complete").exists()
|
|
239
239
|
if filename == "catalogues.zip":
|
|
240
240
|
return catalogue_dir().exists() and any(catalogue_dir().glob("*.csv"))
|
|
241
241
|
if filename == "oracc_jsonzip_all.zip":
|
|
@@ -266,9 +266,9 @@ def fetch_data(
|
|
|
266
266
|
|
|
267
267
|
translations_zip = dest / "oracc_html_translations.zip"
|
|
268
268
|
if translations_zip.exists():
|
|
269
|
-
extract_data_archive(translations_zip,
|
|
269
|
+
extract_data_archive(translations_zip, translations_dir(), "HTML translations")
|
|
270
270
|
translations_zip.unlink()
|
|
271
|
-
(
|
|
271
|
+
(translations_dir() / ".translations_complete").touch()
|
|
272
272
|
|
|
273
273
|
catalogues_zip = dest / "catalogues.zip"
|
|
274
274
|
if catalogues_zip.exists():
|
|
@@ -69,74 +69,6 @@ def download_zip(project: str, output_dir: Path | None = None) -> Path | None:
|
|
|
69
69
|
return None
|
|
70
70
|
|
|
71
71
|
|
|
72
|
-
def download_projects(
|
|
73
|
-
projects: list[str] | None = None,
|
|
74
|
-
config: RunConfig | None = None,
|
|
75
|
-
output_dir: Path | None = None,
|
|
76
|
-
) -> list[Path]:
|
|
77
|
-
"""Download multiple ORACC project ZIPs.
|
|
78
|
-
|
|
79
|
-
If ``projects`` is not specified, uses the metadata table to find
|
|
80
|
-
projects matching the language filter in ``config``.
|
|
81
|
-
|
|
82
|
-
Args:
|
|
83
|
-
projects: Explicit list of project names to download.
|
|
84
|
-
config: RunConfig controlling language filter and limit.
|
|
85
|
-
output_dir: Custom output directory for ZIPs.
|
|
86
|
-
|
|
87
|
-
Returns:
|
|
88
|
-
List of paths to successfully downloaded ZIP files.
|
|
89
|
-
"""
|
|
90
|
-
if config is None:
|
|
91
|
-
config = RunConfig()
|
|
92
|
-
|
|
93
|
-
if projects is None:
|
|
94
|
-
projects = _filter_projects_by_language(config.languages)
|
|
95
|
-
|
|
96
|
-
if config.limit is not None:
|
|
97
|
-
projects = projects[: config.limit]
|
|
98
|
-
|
|
99
|
-
logger.info(f"Downloading {len(projects)} project(s)...")
|
|
100
|
-
downloaded = []
|
|
101
|
-
for project in tqdm(projects, desc="Downloading projects"):
|
|
102
|
-
path = download_zip(project, output_dir)
|
|
103
|
-
if path:
|
|
104
|
-
downloaded.append(path)
|
|
105
|
-
|
|
106
|
-
logger.info(f"Downloaded {len(downloaded)}/{len(projects)} projects.")
|
|
107
|
-
return downloaded
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
def _filter_projects_by_language(languages: list[str]) -> list[str]:
|
|
111
|
-
"""Filter project list by language using the metadata CSV.
|
|
112
|
-
|
|
113
|
-
Args:
|
|
114
|
-
languages: List of language names (e.g. ``["Akkadian"]``).
|
|
115
|
-
Use ``["all"]`` to include everything.
|
|
116
|
-
|
|
117
|
-
Returns:
|
|
118
|
-
List of project names matching the language filter.
|
|
119
|
-
"""
|
|
120
|
-
import pandas as pd
|
|
121
|
-
|
|
122
|
-
df = get_projects_metadata()
|
|
123
|
-
|
|
124
|
-
# If "all", return all projects with non-empty text folders
|
|
125
|
-
if "all" in [lang.lower() for lang in languages]:
|
|
126
|
-
mask_text = df["Is_Text_Folder_Empty"].fillna("").str.lower() != "yes"
|
|
127
|
-
return df.loc[mask_text, "Project_Name"].tolist()
|
|
128
|
-
|
|
129
|
-
# Filter by language
|
|
130
|
-
mask_lang = df["Languages"].fillna("").apply(
|
|
131
|
-
lambda x: any(lang.lower() in x.lower() for lang in languages)
|
|
132
|
-
)
|
|
133
|
-
mask_text = df["Is_Text_Folder_Empty"].fillna("").str.lower() != "yes"
|
|
134
|
-
mask = mask_lang & mask_text
|
|
135
|
-
|
|
136
|
-
projects = df.loc[mask, "Project_Name"].tolist()
|
|
137
|
-
logger.info(f"Found {len(projects)} projects for languages: {languages}")
|
|
138
|
-
return projects
|
|
139
|
-
|
|
140
72
|
|
|
141
73
|
def get_live_project_list() -> list[str]:
|
|
142
74
|
"""Fetch the current list of public projects from ORACC servers.
|
|
@@ -20,7 +20,7 @@ akk-x-oldass,Akkadian,Old Assyrian,yes,,695,tcma-assur,1
|
|
|
20
20
|
uga,Ugaritic,,yes,Ugaritic loanwords in Akkadian,440,"aemw-amarna, aemw-ugarit",2
|
|
21
21
|
hit,Hittite,,yes,,224,"aemw-ugarit, contrib-amarna",2
|
|
22
22
|
qca,Canaanite,,yes,Canaanite loanwords in Akkadian,200,aemw-amarna,1
|
|
23
|
-
qcu-949
|
|
23
|
+
qcu-949,Assyrian Hieroglyphs,,no,"Assyrian ""Hieroglyphs""",176,"ribo-babylon6, rinap-rinap4",2
|
|
24
24
|
akk-x-oldakk,Akkadian,Old Akkadian,yes,,90,akklove,1
|
|
25
25
|
hit-946,Hittite,,yes,,72,"aemw-ugarit, contrib-amarna",2
|
|
26
26
|
qcu,Unclear,,yes,one broken text,68,cams-gkab,1
|
|
@@ -361,75 +361,6 @@ def load_catalogue_csv(path: str | Path) -> pd.DataFrame:
|
|
|
361
361
|
return pd.DataFrame()
|
|
362
362
|
|
|
363
363
|
|
|
364
|
-
def load_word_csvs_from_zenodo(
|
|
365
|
-
zenodo_url: str,
|
|
366
|
-
project: str,
|
|
367
|
-
) -> dict[str, pd.DataFrame]:
|
|
368
|
-
"""Download a project's word CSV ZIP from Zenodo into memory.
|
|
369
|
-
|
|
370
|
-
No files are written to disk. The ZIP is streamed into a BytesIO buffer
|
|
371
|
-
and each inner CSV is read directly into a DataFrame.
|
|
372
|
-
|
|
373
|
-
The expected filename in the Zenodo record is ``{project_slug}_words.zip``
|
|
374
|
-
(e.g. ``saao-saa01_words.zip``), containing one ``{text_id}.csv`` per text.
|
|
375
|
-
|
|
376
|
-
Args:
|
|
377
|
-
zenodo_url: Zenodo record URL (e.g. ``"https://zenodo.org/records/12345"``).
|
|
378
|
-
project: ORACC project path (e.g. ``"saao/saa01"``).
|
|
379
|
-
|
|
380
|
-
Returns:
|
|
381
|
-
Dict mapping text_id to DataFrame.
|
|
382
|
-
"""
|
|
383
|
-
import zipfile
|
|
384
|
-
|
|
385
|
-
import requests
|
|
386
|
-
from tqdm import tqdm
|
|
387
|
-
|
|
388
|
-
from oracc_parser.download.fetch_data import get_zenodo_files
|
|
389
|
-
|
|
390
|
-
project_slug = project.replace("/", "-")
|
|
391
|
-
target_filename = f"{project_slug}_words.zip"
|
|
392
|
-
|
|
393
|
-
files = get_zenodo_files(zenodo_url)
|
|
394
|
-
match = next((f for f in files if f["filename"] == target_filename), None)
|
|
395
|
-
if match is None:
|
|
396
|
-
available = [f["filename"] for f in files]
|
|
397
|
-
raise FileNotFoundError(
|
|
398
|
-
f"No word CSV archive for '{project}' found in Zenodo record. "
|
|
399
|
-
f"Expected '{target_filename}'. Available: {available}"
|
|
400
|
-
)
|
|
401
|
-
|
|
402
|
-
logger.info(
|
|
403
|
-
f"Downloading {target_filename} from Zenodo "
|
|
404
|
-
f"({match['size'] / 1e6:.1f} MB)..."
|
|
405
|
-
)
|
|
406
|
-
resp = requests.get(match["url"], stream=True, timeout=60)
|
|
407
|
-
resp.raise_for_status()
|
|
408
|
-
total = int(resp.headers.get("content-length", 0))
|
|
409
|
-
|
|
410
|
-
buffer = io.BytesIO()
|
|
411
|
-
with tqdm(total=total, unit="B", unit_scale=True, desc=target_filename) as bar:
|
|
412
|
-
for chunk in resp.iter_content(chunk_size=8192):
|
|
413
|
-
buffer.write(chunk)
|
|
414
|
-
bar.update(len(chunk))
|
|
415
|
-
buffer.seek(0)
|
|
416
|
-
|
|
417
|
-
result = {}
|
|
418
|
-
with zipfile.ZipFile(buffer, "r") as z:
|
|
419
|
-
for name in z.namelist():
|
|
420
|
-
if not name.endswith(".csv"):
|
|
421
|
-
continue
|
|
422
|
-
text_id = Path(name).stem
|
|
423
|
-
with z.open(name) as f:
|
|
424
|
-
result[text_id] = pd.read_csv(
|
|
425
|
-
io.TextIOWrapper(f, encoding="utf-8"),
|
|
426
|
-
dtype=str,
|
|
427
|
-
keep_default_na=False,
|
|
428
|
-
)
|
|
429
|
-
|
|
430
|
-
logger.info(f"Loaded {len(result)} word CSV(s) for '{project}' from Zenodo")
|
|
431
|
-
return result
|
|
432
|
-
|
|
433
364
|
|
|
434
365
|
# ---------------------------------------------------------------------------
|
|
435
366
|
# Private helpers
|