oracc-parser 0.1.2__tar.gz → 0.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/PKG-INFO +45 -22
  2. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/README.md +44 -21
  3. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/__init__.py +1 -4
  4. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/cli.py +27 -52
  5. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/download/fetch_data.py +4 -4
  6. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/download/oracc_download.py +0 -68
  7. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/languages.csv +1 -1
  8. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/io/word_csv.py +0 -69
  9. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/metadata/populate.py +35 -0
  10. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/models/config.py +0 -25
  11. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/models/tablet.py +7 -2
  12. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/parsing/translation.py +3 -4
  13. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/pipeline.py +73 -133
  14. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/settings.py +17 -12
  15. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser.egg-info/PKG-INFO +45 -22
  16. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser.egg-info/SOURCES.txt +0 -2
  17. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/pyproject.toml +1 -1
  18. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/tests/test_models.py +0 -2
  19. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/tests/test_settings.py +2 -8
  20. oracc_parser-0.1.2/oracc_parser/cache.py +0 -251
  21. oracc_parser-0.1.2/tests/test_cache.py +0 -205
  22. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/LICENSE +0 -0
  23. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/constants.py +0 -0
  24. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/download/__init__.py +0 -0
  25. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/download/extract_jsons.py +0 -0
  26. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/download/pleiades.py +0 -0
  27. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/__init__.py +0 -0
  28. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/grouped_oracc_metadata_columns.csv +0 -0
  29. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/period_mapping.csv +0 -0
  30. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/pos_tags.csv +0 -0
  31. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/projects_metadata.csv +0 -0
  32. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/provenience.csv +0 -0
  33. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/raw_archive_values.csv +0 -0
  34. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/sign_readings.csv +0 -0
  35. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/enriched_data/state_supergroup_mapping.csv +0 -0
  36. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/export/__init__.py +0 -0
  37. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/export/to_jsonl.py +0 -0
  38. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/io/__init__.py +0 -0
  39. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/metadata/__init__.py +0 -0
  40. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/metadata/archive.py +0 -0
  41. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/models/__init__.py +0 -0
  42. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/parsing/__init__.py +0 -0
  43. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/parsing/parse_content.py +0 -0
  44. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/parsing/parse_signs.py +0 -0
  45. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/parsing/parse_words.py +0 -0
  46. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/parsing/text_builder.py +0 -0
  47. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/utils/__init__.py +0 -0
  48. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/utils/logger.py +0 -0
  49. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/utils/paths.py +0 -0
  50. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser/utils/unicode.py +0 -0
  51. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser.egg-info/dependency_links.txt +0 -0
  52. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser.egg-info/entry_points.txt +0 -0
  53. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser.egg-info/requires.txt +0 -0
  54. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/oracc_parser.egg-info/top_level.txt +0 -0
  55. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/setup.cfg +0 -0
  56. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/tests/test_constants.py +0 -0
  57. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/tests/test_export.py +0 -0
  58. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/tests/test_parsing.py +0 -0
  59. {oracc_parser-0.1.2 → oracc_parser-0.1.4}/tests/test_pipeline.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: oracc-parser
3
- Version: 0.1.2
3
+ Version: 0.1.4
4
4
  Summary: Download and parse ORACC cuneiform text projects into ML-ready formats
5
5
  Author: Avital Romach, Shahar Spencer, Claude Sonnet 4.6
6
6
  License-Expression: MIT
@@ -40,13 +40,34 @@ A Python tool to download and parse [ORACC](http://oracc.museum.upenn.edu/) cune
40
40
 
41
41
  ## Features
42
42
 
43
- - **Download** — Fetch project ZIPs directly from ORACC or Zenodo
44
- - **Parse** — Convert raw ORACC JSON into structured data
43
+ - **Download** — Fetch project ZIPs from preprocessed files on Zenodo or directly from ORACC
44
+ - **Parse** — Convert raw ORACC JSON / preprocessed CSVs into structured data and create machine-learning-ready text representations (in transliteration, normalization, lemmatization, or Unicode cuneiform)
45
+ - **Configure** — Control handling of broken signs and POS masking
45
46
  - **Export** — Save datasets as JSONL, CSV, or pandas DataFrames
46
- - **Configure** — Control handling of broken signs and POS masking using `RunConfig`
47
+
48
+ ## How to Cite
49
+
50
+ If you use oracc-parser in your research, please cite:
51
+
52
+ ```bibtex
53
+ @software{romach_oracc_parser_2026,
54
+ author = {Romach, Avital and Spencer, Shahar},
55
+ title = {oracc-parser: A Python toolkit for downloading and parsing ORACC cuneiform text projects},
56
+ year = {2026},
57
+ publisher = {Zenodo},
58
+ version = {0.1.4},
59
+ doi = {10.5281/zenodo.18643122},
60
+ url = {https://doi.org/10.5281/zenodo.18643122}
61
+ }
62
+ ```
47
63
 
48
64
  ## Installation
49
65
 
66
+ ```bash
67
+ pip install oracc-parser
68
+ ```
69
+
70
+ Or for development:
50
71
 
51
72
  ```bash
52
73
  git clone https://github.com/shaharspencer/oracc-parser.git
@@ -56,15 +77,15 @@ pip install -e ".[dev]"
56
77
 
57
78
  ## Getting Started — Notebooks
58
79
 
59
- The easiest way to explore oracc-parser is through the interactive notebooks.
60
- Start with notebook 01 — it downloads all the data you need from Zenodo automatically.
80
+ The easiest way to explore oracc-parser is through the interactive notebooks, that go through the main functions.
61
81
 
62
- | Notebook | What you'll learn |
63
- |---|---|
64
- | [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project from pre-processed CSVs → explore transliterations, translations, and metadata → export |
65
- | [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) |
66
- | [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets |
67
- | [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset |
82
+ | Notebook | What you'll learn | Open in Colab |
83
+ |---|---|---|
84
+ | [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project → explore transliterations, translations, and metadata → export | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/01_quickstart.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
85
+ | [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/02_reference_data.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
86
+ | [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/03_configure_and_export.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
87
+ | [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/04_oracc_json_processing.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
88
+ | [`05_translations.ipynb`](notebooks/05_translations.ipynb) | Where translations come from, how to download the translation cache from Zenodo, and how to enable translations in the parsing pipeline | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/05_translations.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
68
89
 
69
90
  ```bash
70
91
  pip install oracc-parser[notebooks]
@@ -74,14 +95,22 @@ jupyter notebook notebooks/
74
95
  ## Quick Example
75
96
 
76
97
  ```python
98
+ from oracc_parser.download.fetch_data import fetch_data
77
99
  from oracc_parser import parse_project, RunConfig, get_full_flat_table
78
100
 
79
- # Parse 5 tablets from SAA 01 (Neo-Assyrian royal letters)
101
+ # Step 1: download catalogues from Zenodo (run once)
102
+ fetch_data()
103
+
104
+ # Step 2: parse a project — word CSVs fetched from Zenodo on first call,
105
+ # read from disk on subsequent calls
80
106
  records = parse_project("saao/saa01", config=RunConfig(limit=5))
81
107
 
82
108
  # Get a flat DataFrame — no nesting, ready for analysis
83
109
  df = get_full_flat_table(records)
84
110
  df.to_json("dataset.jsonl", orient="records", lines=True)
111
+
112
+ # For a project not on Zenodo, download directly from ORACC servers:
113
+ records = parse_project("my/project", download_from_oracc_server=True)
85
114
  ```
86
115
 
87
116
  ## Configuration
@@ -113,9 +142,10 @@ operating at different granularities and affecting different outputs:
113
142
 
114
143
  > **Note:** Because word-level and sign-level filtering use different thresholds
115
144
  > and different granularities, **the text outputs and the Unicode cuneiform output
116
- > are not necessarily aligned**. A word kept in the transliteration (because its
145
+ > are not aligned**. A word kept in the transliteration (because its
117
146
  > average damage is below `max_break_fraction`) may still have individual signs
118
147
  > dropped from the Unicode output if `drop_missing` / `drop_damaged` are enabled.
148
+ > None of these filters affect the translations.
119
149
 
120
150
  ### Other options
121
151
 
@@ -124,8 +154,6 @@ operating at different granularities and affecting different outputs:
124
154
  | `limit` | `None` | Only parse the first N texts (useful for testing) |
125
155
  | `keep_word_segmentation` | `True` | Preserve word boundaries in Unicode cuneiform output |
126
156
  | `mask_pos` | `[]` | Replace words of certain POS tags with the tag name |
127
- | `languages` | `["Akkadian"]` | Which languages to include when downloading projects |
128
- | `use_cache` | `True` | Use cached results if available |
129
157
 
130
158
  All reference data is bundled with the package, so you don't need to configure external paths unless you are customizing `oracc_parser.settings`.
131
159
 
@@ -138,7 +166,7 @@ oracc-parser parse --project saao/saa01 --limit 5 --format jsonl --output saa01.
138
166
 
139
167
  ## Heavy Data (Zenodo)
140
168
 
141
- Large data files (ORACC ZIPs, cached translations, Pleiades data) are on Zenodo:
169
+ Large data files (ORACC ZIPs, word-level CSVs, cached translations, Pleiades data) are on Zenodo:
142
170
 
143
171
  [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.18643122.svg)](https://doi.org/10.5281/zenodo.18643122)
144
172
 
@@ -152,11 +180,6 @@ python scripts/download_zenodo_data.py
152
180
  pytest tests/ -v # 98 tests
153
181
  ```
154
182
 
155
- ## Known Limitations
156
-
157
- - **Chronology**: Period-to-year normalization is optimized for the **1st Millennium BCE**.
158
- - **Language**: Parsing is primarily validated on **Akkadian** projects.
159
-
160
183
  ## License
161
184
 
162
185
  MIT — see [LICENSE](LICENSE).
@@ -4,13 +4,34 @@ A Python tool to download and parse [ORACC](http://oracc.museum.upenn.edu/) cune
4
4
 
5
5
  ## Features
6
6
 
7
- - **Download** — Fetch project ZIPs directly from ORACC or Zenodo
8
- - **Parse** — Convert raw ORACC JSON into structured data
7
+ - **Download** — Fetch project ZIPs from preprocessed files on Zenodo or directly from ORACC
8
+ - **Parse** — Convert raw ORACC JSON / preprocessed CSVs into structured data and create machine-learning-ready text representations (in transliteration, normalization, lemmatization, or Unicode cuneiform)
9
+ - **Configure** — Control handling of broken signs and POS masking
9
10
  - **Export** — Save datasets as JSONL, CSV, or pandas DataFrames
10
- - **Configure** — Control handling of broken signs and POS masking using `RunConfig`
11
+
12
+ ## How to Cite
13
+
14
+ If you use oracc-parser in your research, please cite:
15
+
16
+ ```bibtex
17
+ @software{romach_oracc_parser_2026,
18
+ author = {Romach, Avital and Spencer, Shahar},
19
+ title = {oracc-parser: A Python toolkit for downloading and parsing ORACC cuneiform text projects},
20
+ year = {2026},
21
+ publisher = {Zenodo},
22
+ version = {0.1.4},
23
+ doi = {10.5281/zenodo.18643122},
24
+ url = {https://doi.org/10.5281/zenodo.18643122}
25
+ }
26
+ ```
11
27
 
12
28
  ## Installation
13
29
 
30
+ ```bash
31
+ pip install oracc-parser
32
+ ```
33
+
34
+ Or for development:
14
35
 
15
36
  ```bash
16
37
  git clone https://github.com/shaharspencer/oracc-parser.git
@@ -20,15 +41,15 @@ pip install -e ".[dev]"
20
41
 
21
42
  ## Getting Started — Notebooks
22
43
 
23
- The easiest way to explore oracc-parser is through the interactive notebooks.
24
- Start with notebook 01 — it downloads all the data you need from Zenodo automatically.
44
+ The easiest way to explore oracc-parser is through the interactive notebooks, that go through the main functions.
25
45
 
26
- | Notebook | What you'll learn |
27
- |---|---|
28
- | [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project from pre-processed CSVs → explore transliterations, translations, and metadata → export |
29
- | [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) |
30
- | [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets |
31
- | [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset |
46
+ | Notebook | What you'll learn | Open in Colab |
47
+ |---|---|---|
48
+ | [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project → explore transliterations, translations, and metadata → export | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/01_quickstart.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
49
+ | [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/02_reference_data.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
50
+ | [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/03_configure_and_export.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
51
+ | [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/04_oracc_json_processing.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
52
+ | [`05_translations.ipynb`](notebooks/05_translations.ipynb) | Where translations come from, how to download the translation cache from Zenodo, and how to enable translations in the parsing pipeline | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/05_translations.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
32
53
 
33
54
  ```bash
34
55
  pip install oracc-parser[notebooks]
@@ -38,14 +59,22 @@ jupyter notebook notebooks/
38
59
  ## Quick Example
39
60
 
40
61
  ```python
62
+ from oracc_parser.download.fetch_data import fetch_data
41
63
  from oracc_parser import parse_project, RunConfig, get_full_flat_table
42
64
 
43
- # Parse 5 tablets from SAA 01 (Neo-Assyrian royal letters)
65
+ # Step 1: download catalogues from Zenodo (run once)
66
+ fetch_data()
67
+
68
+ # Step 2: parse a project — word CSVs fetched from Zenodo on first call,
69
+ # read from disk on subsequent calls
44
70
  records = parse_project("saao/saa01", config=RunConfig(limit=5))
45
71
 
46
72
  # Get a flat DataFrame — no nesting, ready for analysis
47
73
  df = get_full_flat_table(records)
48
74
  df.to_json("dataset.jsonl", orient="records", lines=True)
75
+
76
+ # For a project not on Zenodo, download directly from ORACC servers:
77
+ records = parse_project("my/project", download_from_oracc_server=True)
49
78
  ```
50
79
 
51
80
  ## Configuration
@@ -77,9 +106,10 @@ operating at different granularities and affecting different outputs:
77
106
 
78
107
  > **Note:** Because word-level and sign-level filtering use different thresholds
79
108
  > and different granularities, **the text outputs and the Unicode cuneiform output
80
- > are not necessarily aligned**. A word kept in the transliteration (because its
109
+ > are not aligned**. A word kept in the transliteration (because its
81
110
  > average damage is below `max_break_fraction`) may still have individual signs
82
111
  > dropped from the Unicode output if `drop_missing` / `drop_damaged` are enabled.
112
+ > None of these filters affect the translations.
83
113
 
84
114
  ### Other options
85
115
 
@@ -88,8 +118,6 @@ operating at different granularities and affecting different outputs:
88
118
  | `limit` | `None` | Only parse the first N texts (useful for testing) |
89
119
  | `keep_word_segmentation` | `True` | Preserve word boundaries in Unicode cuneiform output |
90
120
  | `mask_pos` | `[]` | Replace words of certain POS tags with the tag name |
91
- | `languages` | `["Akkadian"]` | Which languages to include when downloading projects |
92
- | `use_cache` | `True` | Use cached results if available |
93
121
 
94
122
  All reference data is bundled with the package, so you don't need to configure external paths unless you are customizing `oracc_parser.settings`.
95
123
 
@@ -102,7 +130,7 @@ oracc-parser parse --project saao/saa01 --limit 5 --format jsonl --output saa01.
102
130
 
103
131
  ## Heavy Data (Zenodo)
104
132
 
105
- Large data files (ORACC ZIPs, cached translations, Pleiades data) are on Zenodo:
133
+ Large data files (ORACC ZIPs, word-level CSVs, cached translations, Pleiades data) are on Zenodo:
106
134
 
107
135
  [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.18643122.svg)](https://doi.org/10.5281/zenodo.18643122)
108
136
 
@@ -116,11 +144,6 @@ python scripts/download_zenodo_data.py
116
144
  pytest tests/ -v # 98 tests
117
145
  ```
118
146
 
119
- ## Known Limitations
120
-
121
- - **Chronology**: Period-to-year normalization is optimized for the **1st Millennium BCE**.
122
- - **Language**: Parsing is primarily validated on **Akkadian** projects.
123
-
124
147
  ## License
125
148
 
126
149
  MIT — see [LICENSE](LICENSE).
@@ -3,14 +3,13 @@ oracc-parser: Download and parse ORACC cuneiform text projects.
3
3
  """
4
4
  from __future__ import annotations
5
5
 
6
- __version__ = "0.1.2"
6
+ __version__ = "0.1.4"
7
7
 
8
8
  # Public re-exports for convenience
9
9
  from oracc_parser.pipeline import ( # noqa: F401
10
10
  export_to_csv,
11
11
  export_to_jsonl,
12
12
  parse_project,
13
- parse_project_from_word_csvs,
14
13
  records_to_word_dataframes,
15
14
  save_project_catalogue,
16
15
  load_project_catalogue,
@@ -24,8 +23,6 @@ from oracc_parser.pipeline import ( # noqa: F401
24
23
  get_full_flat_table,
25
24
  )
26
25
  from oracc_parser.io.word_csv import ( # noqa: F401
27
- load_word_csvs_from_dir,
28
- load_word_csvs_from_zenodo,
29
26
  save_word_csv,
30
27
  )
31
28
  from oracc_parser.models.config import RunConfig # noqa: F401
@@ -6,8 +6,6 @@ Usage:
6
6
  oracc-parser download --lang akkadian
7
7
  oracc-parser parse --project saao/saa01 --format jsonl --output data.jsonl
8
8
  oracc-parser parse --project saao/saa01 --limit 5 --format csv --output data.csv
9
- oracc-parser clear-cache # clear all cached tablets
10
- oracc-parser clear-cache --project saao # clear one project's cache
11
9
  """
12
10
  from __future__ import annotations
13
11
 
@@ -15,7 +13,6 @@ import argparse
15
13
  import sys
16
14
 
17
15
  from oracc_parser.pipeline import export_to_csv, export_to_jsonl, parse_project
18
- from oracc_parser.download.oracc_download import download_projects
19
16
  from oracc_parser.models.config import RunConfig
20
17
  from oracc_parser.utils.logger import get_logger
21
18
 
@@ -31,15 +28,8 @@ def main(argv: list[str] | None = None):
31
28
  subparsers = parser.add_subparsers(dest="command", help="Available commands")
32
29
 
33
30
  # --------------- download ---------------
34
- dl = subparsers.add_parser("download", help="Download ORACC project ZIPs")
35
- dl.add_argument("--project", "-p", help="Project path, e.g. saao/saa01")
36
- dl.add_argument(
37
- "--lang",
38
- "-l",
39
- default="Akkadian",
40
- help="Language filter for bulk download (default: Akkadian)",
41
- )
42
- dl.add_argument("--limit", "-n", type=int, help="Download only first N projects")
31
+ dl = subparsers.add_parser("download", help="Download a single ORACC project ZIP from the ORACC servers")
32
+ dl.add_argument("--project", "-p", required=True, help="Project path, e.g. saao/saa01")
43
33
 
44
34
  # --------------- parse ---------------
45
35
  ps = subparsers.add_parser("parse", help="Parse a project and export results")
@@ -71,8 +61,22 @@ def main(argv: list[str] | None = None):
71
61
  default=[],
72
62
  help="POS tags to mask (e.g. PN DN GN)",
73
63
  )
74
- ps.add_argument("--no-cache", action="store_true", help="Disable caching")
75
- ps.add_argument("--no-download", action="store_true", help="Skip download step")
64
+ ps.add_argument(
65
+ "--max-break-fraction",
66
+ type=float,
67
+ default=1.0,
68
+ help="Word-level break filter: words with more than this fraction of broken signs are replaced with X (default: 1.0 = keep all)",
69
+ )
70
+ ps.add_argument(
71
+ "--fetch-translations",
72
+ action="store_true",
73
+ help="Fetch English translations (from cache or ORACC live)",
74
+ )
75
+ ps.add_argument(
76
+ "--from-oracc",
77
+ action="store_true",
78
+ help="Download from ORACC live servers instead of Zenodo (for projects not in the dataset)",
79
+ )
76
80
 
77
81
  # --------------- fetch-data ---------------
78
82
  fd = subparsers.add_parser("fetch-data", help="Download pre-processed data from Zenodo")
@@ -82,13 +86,6 @@ def main(argv: list[str] | None = None):
82
86
  # --------------- info ---------------
83
87
  subparsers.add_parser("info", help="Show bundled reference data summary")
84
88
 
85
- # --------------- clear-cache ---------------
86
- cc = subparsers.add_parser("clear-cache", help="Delete cached parsed tablets")
87
- cc.add_argument(
88
- "--project", "-p",
89
- help="Only clear cache for this project (default: clear all)",
90
- )
91
-
92
89
  args = parser.parse_args(argv)
93
90
 
94
91
  if args.command == "download":
@@ -99,8 +96,6 @@ def main(argv: list[str] | None = None):
99
96
  _cmd_parse(args)
100
97
  elif args.command == "info":
101
98
  _cmd_info()
102
- elif args.command == "clear-cache":
103
- _cmd_clear_cache(args)
104
99
  else:
105
100
  parser.print_help()
106
101
  sys.exit(1)
@@ -119,23 +114,14 @@ def _cmd_fetch_data(args):
119
114
 
120
115
  def _cmd_download(args):
121
116
  """Handle the download command."""
122
- config = RunConfig(
123
- languages=[args.lang] if args.lang else ["Akkadian"],
124
- limit=args.limit,
125
- )
126
-
127
- if args.project:
128
- from oracc_parser.download.oracc_download import download_zip
117
+ from oracc_parser.download.oracc_download import download_zip
129
118
 
130
- path = download_zip(args.project)
131
- if path:
132
- print(f"Downloaded: {path}")
133
- else:
134
- print("Download failed.", file=sys.stderr)
135
- sys.exit(1)
119
+ path = download_zip(args.project)
120
+ if path:
121
+ print(f"Downloaded: {path}")
136
122
  else:
137
- paths = download_projects(config=config)
138
- print(f"Downloaded {len(paths)} project(s).")
123
+ print("Download failed.", file=sys.stderr)
124
+ sys.exit(1)
139
125
 
140
126
 
141
127
  def _cmd_parse(args):
@@ -144,12 +130,13 @@ def _cmd_parse(args):
144
130
  drop_missing=args.drop_missing,
145
131
  drop_damaged=args.drop_damaged,
146
132
  mask_pos=args.mask_pos,
147
- use_cache=not args.no_cache,
148
133
  limit=args.limit,
134
+ max_break_fraction=args.max_break_fraction,
135
+ fetch_translations=args.fetch_translations,
149
136
  )
150
137
 
151
138
  records = parse_project(
152
- args.project, config=config, download=not args.no_download
139
+ args.project, config=config, download_from_oracc_server=args.from_oracc
153
140
  )
154
141
 
155
142
  if not records:
@@ -185,17 +172,5 @@ def _cmd_info():
185
172
  print(f" {name}: Error loading - {e}", file=sys.stderr)
186
173
 
187
174
 
188
- def _cmd_clear_cache(args):
189
- """Handle the clear-cache command."""
190
- from oracc_parser.cache import clear_project_cache
191
-
192
- count = clear_project_cache(project=args.project)
193
- if count:
194
- scope = f"project '{args.project}'" if args.project else "all projects"
195
- print(f"Cleared {count} cached tablet(s) for {scope}.")
196
- else:
197
- print("No cached tablets found.")
198
-
199
-
200
175
  if __name__ == "__main__":
201
176
  main()
@@ -22,7 +22,7 @@ from pathlib import Path
22
22
  import requests
23
23
  from tqdm import tqdm
24
24
 
25
- from oracc_parser.settings import zenodo_url, data_dir, jsonzip_dir, word_csv_dir, catalogue_dir, cache_dir
25
+ from oracc_parser.settings import zenodo_url, data_dir, jsonzip_dir, word_csv_dir, catalogue_dir, translations_dir
26
26
 
27
27
 
28
28
  def download_file(url: str, dest: Path, desc: str = "") -> Path:
@@ -235,7 +235,7 @@ def fetch_data(
235
235
 
236
236
  def _already_extracted(filename: str) -> bool:
237
237
  if filename == "oracc_html_translations.zip":
238
- return any((cache_dir() / "html").rglob("*.html"))
238
+ return (translations_dir() / ".translations_complete").exists()
239
239
  if filename == "catalogues.zip":
240
240
  return catalogue_dir().exists() and any(catalogue_dir().glob("*.csv"))
241
241
  if filename == "oracc_jsonzip_all.zip":
@@ -266,9 +266,9 @@ def fetch_data(
266
266
 
267
267
  translations_zip = dest / "oracc_html_translations.zip"
268
268
  if translations_zip.exists():
269
- extract_data_archive(translations_zip, cache_dir() / "html", "HTML translations")
269
+ extract_data_archive(translations_zip, translations_dir(), "HTML translations")
270
270
  translations_zip.unlink()
271
- (cache_dir() / "html" / ".translations_complete").touch()
271
+ (translations_dir() / ".translations_complete").touch()
272
272
 
273
273
  catalogues_zip = dest / "catalogues.zip"
274
274
  if catalogues_zip.exists():
@@ -69,74 +69,6 @@ def download_zip(project: str, output_dir: Path | None = None) -> Path | None:
69
69
  return None
70
70
 
71
71
 
72
- def download_projects(
73
- projects: list[str] | None = None,
74
- config: RunConfig | None = None,
75
- output_dir: Path | None = None,
76
- ) -> list[Path]:
77
- """Download multiple ORACC project ZIPs.
78
-
79
- If ``projects`` is not specified, uses the metadata table to find
80
- projects matching the language filter in ``config``.
81
-
82
- Args:
83
- projects: Explicit list of project names to download.
84
- config: RunConfig controlling language filter and limit.
85
- output_dir: Custom output directory for ZIPs.
86
-
87
- Returns:
88
- List of paths to successfully downloaded ZIP files.
89
- """
90
- if config is None:
91
- config = RunConfig()
92
-
93
- if projects is None:
94
- projects = _filter_projects_by_language(config.languages)
95
-
96
- if config.limit is not None:
97
- projects = projects[: config.limit]
98
-
99
- logger.info(f"Downloading {len(projects)} project(s)...")
100
- downloaded = []
101
- for project in tqdm(projects, desc="Downloading projects"):
102
- path = download_zip(project, output_dir)
103
- if path:
104
- downloaded.append(path)
105
-
106
- logger.info(f"Downloaded {len(downloaded)}/{len(projects)} projects.")
107
- return downloaded
108
-
109
-
110
- def _filter_projects_by_language(languages: list[str]) -> list[str]:
111
- """Filter project list by language using the metadata CSV.
112
-
113
- Args:
114
- languages: List of language names (e.g. ``["Akkadian"]``).
115
- Use ``["all"]`` to include everything.
116
-
117
- Returns:
118
- List of project names matching the language filter.
119
- """
120
- import pandas as pd
121
-
122
- df = get_projects_metadata()
123
-
124
- # If "all", return all projects with non-empty text folders
125
- if "all" in [lang.lower() for lang in languages]:
126
- mask_text = df["Is_Text_Folder_Empty"].fillna("").str.lower() != "yes"
127
- return df.loc[mask_text, "Project_Name"].tolist()
128
-
129
- # Filter by language
130
- mask_lang = df["Languages"].fillna("").apply(
131
- lambda x: any(lang.lower() in x.lower() for lang in languages)
132
- )
133
- mask_text = df["Is_Text_Folder_Empty"].fillna("").str.lower() != "yes"
134
- mask = mask_lang & mask_text
135
-
136
- projects = df.loc[mask, "Project_Name"].tolist()
137
- logger.info(f"Found {len(projects)} projects for languages: {languages}")
138
- return projects
139
-
140
72
 
141
73
  def get_live_project_list() -> list[str]:
142
74
  """Fetch the current list of public projects from ORACC servers.
@@ -20,7 +20,7 @@ akk-x-oldass,Akkadian,Old Assyrian,yes,,695,tcma-assur,1
20
20
  uga,Ugaritic,,yes,Ugaritic loanwords in Akkadian,440,"aemw-amarna, aemw-ugarit",2
21
21
  hit,Hittite,,yes,,224,"aemw-ugarit, contrib-amarna",2
22
22
  qca,Canaanite,,yes,Canaanite loanwords in Akkadian,200,aemw-amarna,1
23
- qcu-949,,,no,"Assyrian ""Hieroglyphs""",176,"ribo-babylon6, rinap-rinap4",2
23
+ qcu-949,Assyrian Hieroglyphs,,no,"Assyrian ""Hieroglyphs""",176,"ribo-babylon6, rinap-rinap4",2
24
24
  akk-x-oldakk,Akkadian,Old Akkadian,yes,,90,akklove,1
25
25
  hit-946,Hittite,,yes,,72,"aemw-ugarit, contrib-amarna",2
26
26
  qcu,Unclear,,yes,one broken text,68,cams-gkab,1
@@ -361,75 +361,6 @@ def load_catalogue_csv(path: str | Path) -> pd.DataFrame:
361
361
  return pd.DataFrame()
362
362
 
363
363
 
364
- def load_word_csvs_from_zenodo(
365
- zenodo_url: str,
366
- project: str,
367
- ) -> dict[str, pd.DataFrame]:
368
- """Download a project's word CSV ZIP from Zenodo into memory.
369
-
370
- No files are written to disk. The ZIP is streamed into a BytesIO buffer
371
- and each inner CSV is read directly into a DataFrame.
372
-
373
- The expected filename in the Zenodo record is ``{project_slug}_words.zip``
374
- (e.g. ``saao-saa01_words.zip``), containing one ``{text_id}.csv`` per text.
375
-
376
- Args:
377
- zenodo_url: Zenodo record URL (e.g. ``"https://zenodo.org/records/12345"``).
378
- project: ORACC project path (e.g. ``"saao/saa01"``).
379
-
380
- Returns:
381
- Dict mapping text_id to DataFrame.
382
- """
383
- import zipfile
384
-
385
- import requests
386
- from tqdm import tqdm
387
-
388
- from oracc_parser.download.fetch_data import get_zenodo_files
389
-
390
- project_slug = project.replace("/", "-")
391
- target_filename = f"{project_slug}_words.zip"
392
-
393
- files = get_zenodo_files(zenodo_url)
394
- match = next((f for f in files if f["filename"] == target_filename), None)
395
- if match is None:
396
- available = [f["filename"] for f in files]
397
- raise FileNotFoundError(
398
- f"No word CSV archive for '{project}' found in Zenodo record. "
399
- f"Expected '{target_filename}'. Available: {available}"
400
- )
401
-
402
- logger.info(
403
- f"Downloading {target_filename} from Zenodo "
404
- f"({match['size'] / 1e6:.1f} MB)..."
405
- )
406
- resp = requests.get(match["url"], stream=True, timeout=60)
407
- resp.raise_for_status()
408
- total = int(resp.headers.get("content-length", 0))
409
-
410
- buffer = io.BytesIO()
411
- with tqdm(total=total, unit="B", unit_scale=True, desc=target_filename) as bar:
412
- for chunk in resp.iter_content(chunk_size=8192):
413
- buffer.write(chunk)
414
- bar.update(len(chunk))
415
- buffer.seek(0)
416
-
417
- result = {}
418
- with zipfile.ZipFile(buffer, "r") as z:
419
- for name in z.namelist():
420
- if not name.endswith(".csv"):
421
- continue
422
- text_id = Path(name).stem
423
- with z.open(name) as f:
424
- result[text_id] = pd.read_csv(
425
- io.TextIOWrapper(f, encoding="utf-8"),
426
- dtype=str,
427
- keep_default_na=False,
428
- )
429
-
430
- logger.info(f"Loaded {len(result)} word CSV(s) for '{project}' from Zenodo")
431
- return result
432
-
433
364
 
434
365
  # ---------------------------------------------------------------------------
435
366
  # Private helpers