oracc-parser 0.1.2__tar.gz → 0.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/PKG-INFO +13 -7
  2. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/README.md +12 -6
  3. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/__init__.py +1 -1
  4. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/cli.py +0 -25
  5. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/download/fetch_data.py +4 -4
  6. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/languages.csv +1 -1
  7. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/models/config.py +0 -13
  8. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/models/tablet.py +3 -2
  9. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/parsing/translation.py +3 -4
  10. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/pipeline.py +34 -57
  11. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/settings.py +17 -12
  12. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser.egg-info/PKG-INFO +13 -7
  13. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser.egg-info/SOURCES.txt +0 -2
  14. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/pyproject.toml +1 -1
  15. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/tests/test_models.py +0 -2
  16. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/tests/test_settings.py +2 -8
  17. oracc_parser-0.1.2/oracc_parser/cache.py +0 -251
  18. oracc_parser-0.1.2/tests/test_cache.py +0 -205
  19. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/LICENSE +0 -0
  20. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/constants.py +0 -0
  21. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/download/__init__.py +0 -0
  22. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/download/extract_jsons.py +0 -0
  23. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/download/oracc_download.py +0 -0
  24. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/download/pleiades.py +0 -0
  25. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/__init__.py +0 -0
  26. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/grouped_oracc_metadata_columns.csv +0 -0
  27. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/period_mapping.csv +0 -0
  28. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/pos_tags.csv +0 -0
  29. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/projects_metadata.csv +0 -0
  30. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/provenience.csv +0 -0
  31. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/raw_archive_values.csv +0 -0
  32. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/sign_readings.csv +0 -0
  33. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/enriched_data/state_supergroup_mapping.csv +0 -0
  34. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/export/__init__.py +0 -0
  35. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/export/to_jsonl.py +0 -0
  36. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/io/__init__.py +0 -0
  37. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/io/word_csv.py +0 -0
  38. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/metadata/__init__.py +0 -0
  39. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/metadata/archive.py +0 -0
  40. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/metadata/populate.py +0 -0
  41. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/models/__init__.py +0 -0
  42. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/parsing/__init__.py +0 -0
  43. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/parsing/parse_content.py +0 -0
  44. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/parsing/parse_signs.py +0 -0
  45. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/parsing/parse_words.py +0 -0
  46. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/parsing/text_builder.py +0 -0
  47. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/utils/__init__.py +0 -0
  48. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/utils/logger.py +0 -0
  49. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/utils/paths.py +0 -0
  50. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser/utils/unicode.py +0 -0
  51. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser.egg-info/dependency_links.txt +0 -0
  52. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser.egg-info/entry_points.txt +0 -0
  53. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser.egg-info/requires.txt +0 -0
  54. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/oracc_parser.egg-info/top_level.txt +0 -0
  55. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/setup.cfg +0 -0
  56. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/tests/test_constants.py +0 -0
  57. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/tests/test_export.py +0 -0
  58. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/tests/test_parsing.py +0 -0
  59. {oracc_parser-0.1.2 → oracc_parser-0.1.3}/tests/test_pipeline.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: oracc-parser
3
- Version: 0.1.2
3
+ Version: 0.1.3
4
4
  Summary: Download and parse ORACC cuneiform text projects into ML-ready formats
5
5
  Author: Avital Romach, Shahar Spencer, Claude Sonnet 4.6
6
6
  License-Expression: MIT
@@ -47,6 +47,11 @@ A Python tool to download and parse [ORACC](http://oracc.museum.upenn.edu/) cune
47
47
 
48
48
  ## Installation
49
49
 
50
+ ```bash
51
+ pip install oracc-parser
52
+ ```
53
+
54
+ Or for development:
50
55
 
51
56
  ```bash
52
57
  git clone https://github.com/shaharspencer/oracc-parser.git
@@ -59,12 +64,13 @@ pip install -e ".[dev]"
59
64
  The easiest way to explore oracc-parser is through the interactive notebooks.
60
65
  Start with notebook 01 — it downloads all the data you need from Zenodo automatically.
61
66
 
62
- | Notebook | What you'll learn |
63
- |---|---|
64
- | [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project from pre-processed CSVs → explore transliterations, translations, and metadata → export |
65
- | [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) |
66
- | [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets |
67
- | [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset |
67
+ | Notebook | What you'll learn | Open in Colab |
68
+ |---|---|---|
69
+ | [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project → explore transliterations, translations, and metadata → export | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/01_quickstart.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
70
+ | [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/02_reference_data.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
71
+ | [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/03_configure_and_export.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
72
+ | [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/04_oracc_json_processing.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
73
+ | [`05_translations.ipynb`](notebooks/05_translations.ipynb) | Where translations come from, how to download the translation cache from Zenodo, and how to enable translations in the parsing pipeline | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/05_translations.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
68
74
 
69
75
  ```bash
70
76
  pip install oracc-parser[notebooks]
@@ -11,6 +11,11 @@ A Python tool to download and parse [ORACC](http://oracc.museum.upenn.edu/) cune
11
11
 
12
12
  ## Installation
13
13
 
14
+ ```bash
15
+ pip install oracc-parser
16
+ ```
17
+
18
+ Or for development:
14
19
 
15
20
  ```bash
16
21
  git clone https://github.com/shaharspencer/oracc-parser.git
@@ -23,12 +28,13 @@ pip install -e ".[dev]"
23
28
  The easiest way to explore oracc-parser is through the interactive notebooks.
24
29
  Start with notebook 01 — it downloads all the data you need from Zenodo automatically.
25
30
 
26
- | Notebook | What you'll learn |
27
- |---|---|
28
- | [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project from pre-processed CSVs → explore transliterations, translations, and metadata → export |
29
- | [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) |
30
- | [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets |
31
- | [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset |
31
+ | Notebook | What you'll learn | Open in Colab |
32
+ |---|---|---|
33
+ | [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project → explore transliterations, translations, and metadata → export | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/01_quickstart.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
34
+ | [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/02_reference_data.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
35
+ | [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/03_configure_and_export.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
36
+ | [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/04_oracc_json_processing.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
37
+ | [`05_translations.ipynb`](notebooks/05_translations.ipynb) | Where translations come from, how to download the translation cache from Zenodo, and how to enable translations in the parsing pipeline | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/05_translations.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
32
38
 
33
39
  ```bash
34
40
  pip install oracc-parser[notebooks]
@@ -3,7 +3,7 @@ oracc-parser: Download and parse ORACC cuneiform text projects.
3
3
  """
4
4
  from __future__ import annotations
5
5
 
6
- __version__ = "0.1.2"
6
+ __version__ = "0.1.3"
7
7
 
8
8
  # Public re-exports for convenience
9
9
  from oracc_parser.pipeline import ( # noqa: F401
@@ -6,8 +6,6 @@ Usage:
6
6
  oracc-parser download --lang akkadian
7
7
  oracc-parser parse --project saao/saa01 --format jsonl --output data.jsonl
8
8
  oracc-parser parse --project saao/saa01 --limit 5 --format csv --output data.csv
9
- oracc-parser clear-cache # clear all cached tablets
10
- oracc-parser clear-cache --project saao # clear one project's cache
11
9
  """
12
10
  from __future__ import annotations
13
11
 
@@ -71,7 +69,6 @@ def main(argv: list[str] | None = None):
71
69
  default=[],
72
70
  help="POS tags to mask (e.g. PN DN GN)",
73
71
  )
74
- ps.add_argument("--no-cache", action="store_true", help="Disable caching")
75
72
  ps.add_argument("--no-download", action="store_true", help="Skip download step")
76
73
 
77
74
  # --------------- fetch-data ---------------
@@ -82,13 +79,6 @@ def main(argv: list[str] | None = None):
82
79
  # --------------- info ---------------
83
80
  subparsers.add_parser("info", help="Show bundled reference data summary")
84
81
 
85
- # --------------- clear-cache ---------------
86
- cc = subparsers.add_parser("clear-cache", help="Delete cached parsed tablets")
87
- cc.add_argument(
88
- "--project", "-p",
89
- help="Only clear cache for this project (default: clear all)",
90
- )
91
-
92
82
  args = parser.parse_args(argv)
93
83
 
94
84
  if args.command == "download":
@@ -99,8 +89,6 @@ def main(argv: list[str] | None = None):
99
89
  _cmd_parse(args)
100
90
  elif args.command == "info":
101
91
  _cmd_info()
102
- elif args.command == "clear-cache":
103
- _cmd_clear_cache(args)
104
92
  else:
105
93
  parser.print_help()
106
94
  sys.exit(1)
@@ -144,7 +132,6 @@ def _cmd_parse(args):
144
132
  drop_missing=args.drop_missing,
145
133
  drop_damaged=args.drop_damaged,
146
134
  mask_pos=args.mask_pos,
147
- use_cache=not args.no_cache,
148
135
  limit=args.limit,
149
136
  )
150
137
 
@@ -185,17 +172,5 @@ def _cmd_info():
185
172
  print(f" {name}: Error loading - {e}", file=sys.stderr)
186
173
 
187
174
 
188
- def _cmd_clear_cache(args):
189
- """Handle the clear-cache command."""
190
- from oracc_parser.cache import clear_project_cache
191
-
192
- count = clear_project_cache(project=args.project)
193
- if count:
194
- scope = f"project '{args.project}'" if args.project else "all projects"
195
- print(f"Cleared {count} cached tablet(s) for {scope}.")
196
- else:
197
- print("No cached tablets found.")
198
-
199
-
200
175
  if __name__ == "__main__":
201
176
  main()
@@ -22,7 +22,7 @@ from pathlib import Path
22
22
  import requests
23
23
  from tqdm import tqdm
24
24
 
25
- from oracc_parser.settings import zenodo_url, data_dir, jsonzip_dir, word_csv_dir, catalogue_dir, cache_dir
25
+ from oracc_parser.settings import zenodo_url, data_dir, jsonzip_dir, word_csv_dir, catalogue_dir, translations_dir
26
26
 
27
27
 
28
28
  def download_file(url: str, dest: Path, desc: str = "") -> Path:
@@ -235,7 +235,7 @@ def fetch_data(
235
235
 
236
236
  def _already_extracted(filename: str) -> bool:
237
237
  if filename == "oracc_html_translations.zip":
238
- return any((cache_dir() / "html").rglob("*.html"))
238
+ return (translations_dir() / ".translations_complete").exists()
239
239
  if filename == "catalogues.zip":
240
240
  return catalogue_dir().exists() and any(catalogue_dir().glob("*.csv"))
241
241
  if filename == "oracc_jsonzip_all.zip":
@@ -266,9 +266,9 @@ def fetch_data(
266
266
 
267
267
  translations_zip = dest / "oracc_html_translations.zip"
268
268
  if translations_zip.exists():
269
- extract_data_archive(translations_zip, cache_dir() / "html", "HTML translations")
269
+ extract_data_archive(translations_zip, translations_dir(), "HTML translations")
270
270
  translations_zip.unlink()
271
- (cache_dir() / "html" / ".translations_complete").touch()
271
+ (translations_dir() / ".translations_complete").touch()
272
272
 
273
273
  catalogues_zip = dest / "catalogues.zip"
274
274
  if catalogues_zip.exists():
@@ -20,7 +20,7 @@ akk-x-oldass,Akkadian,Old Assyrian,yes,,695,tcma-assur,1
20
20
  uga,Ugaritic,,yes,Ugaritic loanwords in Akkadian,440,"aemw-amarna, aemw-ugarit",2
21
21
  hit,Hittite,,yes,,224,"aemw-ugarit, contrib-amarna",2
22
22
  qca,Canaanite,,yes,Canaanite loanwords in Akkadian,200,aemw-amarna,1
23
- qcu-949,,,no,"Assyrian ""Hieroglyphs""",176,"ribo-babylon6, rinap-rinap4",2
23
+ qcu-949,Assyrian Hieroglyphs,,no,"Assyrian ""Hieroglyphs""",176,"ribo-babylon6, rinap-rinap4",2
24
24
  akk-x-oldakk,Akkadian,Old Akkadian,yes,,90,akklove,1
25
25
  hit-946,Hittite,,yes,,72,"aemw-ugarit, contrib-amarna",2
26
26
  qcu,Unclear,,yes,one broken text,68,cams-gkab,1
@@ -82,19 +82,6 @@ class RunConfig(BaseModel):
82
82
  ),
83
83
  )
84
84
 
85
- # --- Caching ---
86
- use_cache: bool = Field(
87
- default=True,
88
- description="Use cached parsed results if available.",
89
- )
90
- cache_dir: Optional[str] = Field(
91
- default=None,
92
- description=(
93
- "Directory for cached parsed results. "
94
- "Defaults to './oracc_cache/' in the current working directory."
95
- ),
96
- )
97
-
98
85
  # --- Sample / limit mode ---
99
86
  limit: Optional[int] = Field(
100
87
  default=None,
@@ -36,7 +36,7 @@ class Language(BaseModel):
36
36
  """Language tag for a word, with cuneiform flag."""
37
37
 
38
38
  is_cuneiform: bool = True
39
- normalized_language: Literal[
39
+ normalized_language: Optional[Literal[
40
40
  "Sumerian",
41
41
  "Proto-cuneiform",
42
42
  "Akkadian",
@@ -53,7 +53,8 @@ class Language(BaseModel):
53
53
  "Proto-Elamite",
54
54
  "Greek",
55
55
  "Egyptian",
56
- ] = None
56
+ "Assyrian Hieroglyphs",
57
+ ]] = None
57
58
  dialect: str = ""
58
59
 
59
60
 
@@ -12,14 +12,14 @@ import requests
12
12
  from bs4 import BeautifulSoup
13
13
 
14
14
  from oracc_parser.utils.logger import get_logger
15
- from oracc_parser.settings import CACHE_DIR as _settings_CACHE_DIR
15
+ from oracc_parser.settings import TRANSLATIONS_DIR as _settings_TRANSLATIONS_DIR
16
16
 
17
17
  logger = get_logger()
18
18
 
19
19
  ORACC_HTML_URL = "https://oracc.museum.upenn.edu"
20
20
 
21
21
 
22
- def get_translation(project: str, text_id: str, cache_dir: str | None = None) -> str:
22
+ def get_translation(project: str, text_id: str) -> str:
23
23
  """Fetch the English translation for a specific tablet.
24
24
 
25
25
  Tries to read from a cached HTML file first. If not found,
@@ -28,12 +28,11 @@ def get_translation(project: str, text_id: str, cache_dir: str | None = None) ->
28
28
  Args:
29
29
  project: Project path, e.g. ``"saao/saa01"``.
30
30
  text_id: The P-number or text ID, e.g. ``"P313511"``.
31
- cache_dir: Custom cache directory. Default: ``oracc_cache/html/``.
32
31
 
33
32
  Returns:
34
33
  Multi-line translation string, or empty string on failure.
35
34
  """
36
- cache = Path(cache_dir) if cache_dir else _settings_CACHE_DIR / "html"
35
+ cache = _settings_TRANSLATIONS_DIR
37
36
  cache.mkdir(parents=True, exist_ok=True)
38
37
 
39
38
  project_path = project.replace("-", "/")
@@ -18,11 +18,6 @@ from pathlib import Path
18
18
  import pandas as pd
19
19
  from tqdm import tqdm
20
20
 
21
- from oracc_parser.cache import (
22
- clear_project_cache,
23
- load_cached_tablet,
24
- save_tablet_to_cache,
25
- )
26
21
  from oracc_parser.download.extract_jsons import extract_from_zip
27
22
  from oracc_parser.download.oracc_download import download_projects, download_zip, get_live_projects_dataframe
28
23
  from oracc_parser.export.to_jsonl import to_csv, to_jsonl
@@ -33,6 +28,7 @@ from oracc_parser.io.word_csv import (
33
28
  load_word_csvs_from_zenodo,
34
29
  record_to_word_dataframe,
35
30
  save_catalogue_csv,
31
+ save_word_csv,
36
32
  word_dataframe_to_record,
37
33
  )
38
34
  from oracc_parser.metadata.populate import populate_metadata
@@ -94,10 +90,14 @@ def parse_project(
94
90
  ) -> list[TabletRecord]:
95
91
  """Download, parse, and return all tablets from an ORACC project.
96
92
 
93
+ On first call, downloads the ORACC ZIP, parses each tablet from JSON,
94
+ and saves per-word CSVs to disk for fast future reloads. On subsequent
95
+ calls the word CSVs are used directly, skipping the JSON parsing step.
96
+
97
97
  Args:
98
98
  project: ORACC project path, e.g. ``"saao/saa01"``.
99
99
  config: RunConfig with parsing options. Uses defaults if None.
100
- download: If True, download the ZIP from ORACC first.
100
+ download: If True, download the ZIP from ORACC if not already present.
101
101
 
102
102
  Returns:
103
103
  List of TabletRecord objects.
@@ -105,62 +105,52 @@ def parse_project(
105
105
  if config is None:
106
106
  config = RunConfig()
107
107
 
108
- # 1. Download
108
+ from oracc_parser.settings import WORD_CSV_DIR
109
+ project_slug = project.replace("/", "-")
110
+ csv_dir = WORD_CSV_DIR / project_slug
111
+
112
+ # Fast path: word CSVs already on disk — skip JSON parsing entirely
113
+ if csv_dir.exists() and any(csv_dir.glob("*.csv")):
114
+ word_dfs = load_word_csvs_from_dir(csv_dir, project=project)
115
+ if config.limit is not None:
116
+ word_dfs = dict(list(word_dfs.items())[: config.limit])
117
+ return parse_project_from_word_csvs(project, word_dfs, config=config)
118
+
119
+ # Slow path: parse from JSON, then save word CSVs for future use
109
120
  if download:
110
121
  zip_path = download_zip(project)
111
122
  if not zip_path:
112
123
  logger.error(f"Failed to download {project}")
113
124
  return []
114
125
 
115
- # 2. Extract JSONs from ZIP
116
126
  project_data = extract_from_zip(project)
117
127
  if not project_data.json_files:
118
128
  logger.warning(f"No JSON files found for {project}")
119
129
  return []
120
130
 
121
- # 3. Parse each text
122
131
  catalogue = project_data.project_catalogue or {}
123
132
  members = catalogue.get("members", {})
124
- records = []
125
133
 
126
134
  json_files = project_data.json_files
127
135
  if config.limit is not None:
128
136
  json_files = json_files[: config.limit]
129
137
 
130
- cache_hits = 0
138
+ records = []
131
139
  for js in tqdm(json_files, desc=f"Parsing {project}"):
132
140
  text_id = js.get("textid", "")
133
-
134
- # --- Cache: instant if config matches, rebuild strings if not ---
135
- if config.use_cache:
136
- cached = load_cached_tablet(project, text_id, config, config.cache_dir)
137
- if cached is not None:
138
- records.append(cached)
139
- cache_hits += 1
140
- continue
141
-
142
- # --- Parse from scratch ---
143
141
  metadata_dict = members.get(text_id, {})
144
142
 
145
143
  record = TabletRecord()
146
144
  record.content = parse_json_text(js, config)
147
- record.content.english_translation = get_translation(
148
- project, text_id, cache_dir=config.cache_dir
149
- )
145
+ if config.fetch_translations:
146
+ record.content.english_translation = get_translation(project, text_id)
150
147
  record.metadata = populate_metadata(metadata_dict, text_id, project)
151
148
  records.append(record)
152
149
 
153
- # --- Save to cache (with config fingerprint) ---
154
- if config.use_cache:
155
- save_tablet_to_cache(record, project, text_id, config, config.cache_dir)
150
+ # Save word CSV so future calls skip JSON parsing
151
+ save_word_csv(record_to_word_dataframe(record))
156
152
 
157
- if cache_hits:
158
- logger.info(
159
- f"Loaded {cache_hits}/{len(records)} tablets from cache, "
160
- f"parsed {len(records) - cache_hits} new"
161
- )
162
- else:
163
- logger.info(f"Parsed {len(records)} tablets from {project}")
153
+ logger.info(f"Parsed {len(records)} tablets from {project}")
164
154
  return records
165
155
 
166
156
 
@@ -317,9 +307,7 @@ def parse_project_from_word_csvs(
317
307
  catalogue_row = catalogue_lookup.get(text_id)
318
308
  record = word_dataframe_to_record(df, config, catalogue_row=catalogue_row)
319
309
  if config.fetch_translations:
320
- record.content.english_translation = get_translation(
321
- project, text_id, cache_dir=config.cache_dir
322
- )
310
+ record.content.english_translation = get_translation(project, text_id)
323
311
  records.append(record)
324
312
 
325
313
  logger.info(f"Processed {len(records)} tablets for {project} from word CSVs")
@@ -493,11 +481,10 @@ def get_translations(records: list[TabletRecord]) -> pd.DataFrame:
493
481
  def get_full_flat_table(records: list[TabletRecord]) -> pd.DataFrame:
494
482
  """Get everything in one flat DataFrame — ideal for releasing as a dataset.
495
483
 
496
- Combines metadata + all string representations into a single table.
497
- No nesting, no Pydantic objects — just clean columns.
484
+ Combines all metadata columns (same as ``get_metadata_table``) with the
485
+ text string representations. No nesting, no Pydantic objects.
498
486
 
499
- Returns columns: ``id``, ``project``, ``text_id``, ``genre``, ``archive``,
500
- ``provenance``, ``period``, ``start_year``, ``end_year``,
487
+ Returns all columns from ``get_metadata_table`` plus:
501
488
  ``transliteration``, ``normalization``, ``lemmatization``,
502
489
  ``unicode``, ``translation``, ``total_tokens``, ``tokens_without_broken``.
503
490
 
@@ -507,7 +494,9 @@ def get_full_flat_table(records: list[TabletRecord]) -> pd.DataFrame:
507
494
  df = get_full_flat_table(records)
508
495
  df.to_json("dataset.jsonl", orient="records", lines=True)
509
496
  """
510
- rows = []
497
+ meta_df = get_metadata_table(records)
498
+
499
+ text_rows = []
511
500
  for r in records:
512
501
  md = r.metadata
513
502
  ct = r.content
@@ -515,21 +504,8 @@ def get_full_flat_table(records: list[TabletRecord]) -> pd.DataFrame:
515
504
  n_rep = ct.normalized_str_representation
516
505
  l_rep = ct.lemmatized_str_representation
517
506
  u_rep = ct.unicode_str_representation
518
-
519
- rows.append({
507
+ text_rows.append({
520
508
  "id": md.identifier,
521
- "project": md.project,
522
- "text_id": md.id_text,
523
- "genre": md.genre or "",
524
- "archive": md.archive or "",
525
- "provenance": md.geographical_information.city.city_name,
526
- "period": (
527
- md.chronological_information.tablet_period.period_name
528
- if md.chronological_information.tablet_period
529
- else ""
530
- ),
531
- "start_year": md.chronological_information.start_year,
532
- "end_year": md.chronological_information.end_year,
533
509
  "transliteration": t_rep.text if t_rep else "",
534
510
  "normalization": n_rep.text if n_rep else "",
535
511
  "lemmatization": l_rep.text if l_rep else "",
@@ -538,4 +514,5 @@ def get_full_flat_table(records: list[TabletRecord]) -> pd.DataFrame:
538
514
  "total_tokens": t_rep.total_tokens if t_rep else 0,
539
515
  "tokens_without_broken": t_rep.tokens_without_broken if t_rep else 0,
540
516
  })
541
- return pd.DataFrame(rows)
517
+ text_df = pd.DataFrame(text_rows)
518
+ return meta_df.merge(text_df, on="id")
@@ -20,7 +20,7 @@ Or via environment variable (set before running Python)::
20
20
  oracc_data/ <-- DATA_DIR
21
21
  catalogues/ <-- CATALOGUE_DIR (project catalogue CSVs)
22
22
  oracc_csvs/ <-- WORD_CSV_DIR (downloaded word-level CSVs)
23
- cache/ <-- CACHE_DIR (parsed tablet JSON cache)
23
+ translations/ <-- TRANSLATIONS_DIR (cached HTML translation pages)
24
24
  jsonzip/ <-- JSONZIP_DIR (downloaded ORACC JSON ZIPs)
25
25
  output/ <-- OUTPUT_DIR (exported CSVs, JSONL files)
26
26
 
@@ -57,7 +57,7 @@ def _default_data_dir() -> Path:
57
57
  DATA_DIR: Path = _default_data_dir()
58
58
  OUTPUT_DIR: Path = DATA_DIR / "output"
59
59
 
60
- CACHE_DIR: Path = DATA_DIR / "cache"
60
+ TRANSLATIONS_DIR: Path = DATA_DIR / "translations"
61
61
  JSONZIP_DIR: Path = DATA_DIR / "jsonzip"
62
62
  WORD_CSV_DIR: Path = DATA_DIR / "oracc_csvs"
63
63
  CATALOGUE_DIR: Path = DATA_DIR / "catalogues"
@@ -65,9 +65,6 @@ CATALOGUE_DIR: Path = DATA_DIR / "catalogues"
65
65
  # URL for downloading reference data (Zenodo)
66
66
  ZENODO_RECORD_URL: str = "https://zenodo.org/records/20625379"
67
67
 
68
- # Should the parser use cached output?
69
- USE_CACHE: bool = True
70
-
71
68
  # Logging
72
69
  LOG_LEVEL: str = "INFO"
73
70
 
@@ -87,9 +84,9 @@ def output_dir() -> Path:
87
84
  global OUTPUT_DIR
88
85
  return OUTPUT_DIR
89
86
 
90
- def cache_dir() -> Path:
91
- global CACHE_DIR
92
- return CACHE_DIR
87
+ def translations_dir() -> Path:
88
+ global TRANSLATIONS_DIR
89
+ return TRANSLATIONS_DIR
93
90
 
94
91
  def jsonzip_dir() -> Path:
95
92
  global JSONZIP_DIR
@@ -107,10 +104,6 @@ def zenodo_url() -> str:
107
104
  global ZENODO_RECORD_URL
108
105
  return ZENODO_RECORD_URL
109
106
 
110
- def use_cache() -> bool:
111
- global USE_CACHE
112
- return USE_CACHE
113
-
114
107
  def log_level() -> str:
115
108
  global LOG_LEVEL
116
109
  return LOG_LEVEL
@@ -118,3 +111,15 @@ def log_level() -> str:
118
111
  def pleiades_zip_path() -> Path | None:
119
112
  global PLEIADES_ZIP
120
113
  return PLEIADES_ZIP
114
+
115
+ def get_settings() -> dict:
116
+ """Return current settings as a plain dict (useful for inspection/debugging)."""
117
+ return {
118
+ "data_dir": str(DATA_DIR),
119
+ "translations_dir": str(TRANSLATIONS_DIR),
120
+ "jsonzip_dir": str(JSONZIP_DIR),
121
+ "word_csv_dir": str(WORD_CSV_DIR),
122
+ "catalogue_dir": str(CATALOGUE_DIR),
123
+ "zenodo_record_url": ZENODO_RECORD_URL,
124
+ "log_level": LOG_LEVEL,
125
+ }
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: oracc-parser
3
- Version: 0.1.2
3
+ Version: 0.1.3
4
4
  Summary: Download and parse ORACC cuneiform text projects into ML-ready formats
5
5
  Author: Avital Romach, Shahar Spencer, Claude Sonnet 4.6
6
6
  License-Expression: MIT
@@ -47,6 +47,11 @@ A Python tool to download and parse [ORACC](http://oracc.museum.upenn.edu/) cune
47
47
 
48
48
  ## Installation
49
49
 
50
+ ```bash
51
+ pip install oracc-parser
52
+ ```
53
+
54
+ Or for development:
50
55
 
51
56
  ```bash
52
57
  git clone https://github.com/shaharspencer/oracc-parser.git
@@ -59,12 +64,13 @@ pip install -e ".[dev]"
59
64
  The easiest way to explore oracc-parser is through the interactive notebooks.
60
65
  Start with notebook 01 — it downloads all the data you need from Zenodo automatically.
61
66
 
62
- | Notebook | What you'll learn |
63
- |---|---|
64
- | [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project from pre-processed CSVs → explore transliterations, translations, and metadata → export |
65
- | [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) |
66
- | [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets |
67
- | [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset |
67
+ | Notebook | What you'll learn | Open in Colab |
68
+ |---|---|---|
69
+ | [`01_quickstart.ipynb`](notebooks/01_quickstart.ipynb) | Download the dataset → parse a project → explore transliterations, translations, and metadata → export | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/01_quickstart.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
70
+ | [`02_reference_data.ipynb`](notebooks/02_reference_data.ipynb) | Browse all projects in the dataset, query catalogues, explore bundled reference data (provenance, periods, sign list, POS tags) | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/02_reference_data.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
71
+ | [`03_configure_and_export.ipynb`](notebooks/03_configure_and_export.ipynb) | All `RunConfig` options — word-level and sign-level break filtering, POS masking — combining multiple projects and exporting datasets | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/03_configure_and_export.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
72
+ | [`04_oracc_json_processing.ipynb`](notebooks/04_oracc_json_processing.ipynb) | Advanced: understand the raw ORACC JSON structure, the JSON → TabletRecord → CSV pipeline, and how to download and parse projects not in the dataset | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/04_oracc_json_processing.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
73
+ | [`05_translations.ipynb`](notebooks/05_translations.ipynb) | Where translations come from, how to download the translation cache from Zenodo, and how to enable translations in the parsing pipeline | <a href="https://colab.research.google.com/github/shaharspencer/oracc-parser/blob/main/notebooks/05_translations.ipynb" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a> |
68
74
 
69
75
  ```bash
70
76
  pip install oracc-parser[notebooks]
@@ -2,7 +2,6 @@ LICENSE
2
2
  README.md
3
3
  pyproject.toml
4
4
  oracc_parser/__init__.py
5
- oracc_parser/cache.py
6
5
  oracc_parser/cli.py
7
6
  oracc_parser/constants.py
8
7
  oracc_parser/pipeline.py
@@ -48,7 +47,6 @@ oracc_parser/utils/__init__.py
48
47
  oracc_parser/utils/logger.py
49
48
  oracc_parser/utils/paths.py
50
49
  oracc_parser/utils/unicode.py
51
- tests/test_cache.py
52
50
  tests/test_constants.py
53
51
  tests/test_export.py
54
52
  tests/test_models.py
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "oracc-parser"
3
- version = "0.1.2"
3
+ version = "0.1.3"
4
4
  description = "Download and parse ORACC cuneiform text projects into ML-ready formats"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.10"
@@ -25,8 +25,6 @@ class TestRunConfig:
25
25
  assert config.drop_damaged is False
26
26
  assert config.keep_word_segmentation is True
27
27
  assert config.mask_pos == []
28
- assert config.use_cache is True
29
- assert config.cache_dir is None
30
28
  assert config.limit is None
31
29
  assert config.languages == ["Akkadian"]
32
30
 
@@ -4,7 +4,7 @@ import os
4
4
 
5
5
  import pytest
6
6
 
7
- from oracc_parser.settings import get_settings, data_dir, cache_dir, log_level
7
+ from oracc_parser.settings import get_settings, data_dir, log_level
8
8
 
9
9
 
10
10
  class TestSettings:
@@ -17,18 +17,12 @@ class TestSettings:
17
17
  d = data_dir()
18
18
  assert hasattr(d, "exists") # It's a Path
19
19
 
20
- def test_cache_dir_under_data(self):
21
- c = cache_dir()
22
- d = data_dir()
23
- # Default cache dir should be under data dir
24
- assert str(d) in str(c)
25
-
26
20
  def test_settings_returns_dict(self):
27
21
  s = get_settings()
28
22
  assert isinstance(s, dict)
29
23
  assert "zenodo_record_url" in s
30
24
  assert "data_dir" in s
31
- assert "use_cache" in s
25
+ assert "translations_dir" in s
32
26
 
33
27
  def test_env_override(self, monkeypatch):
34
28
  """Environment variables should override defaults."""
@@ -1,251 +0,0 @@
1
- """
2
- JSON caching for parsed TabletRecord objects.
3
-
4
- Parsed tablets are expensive to produce (long runtimes due to CDL tree
5
- traversal, sign parsing, and translation downloads). This module caches
6
- the full result including a **config fingerprint**.
7
-
8
- On reload:
9
- - If the current config matches the cached fingerprint → **instant return**
10
- (everything is reused, including string representations)
11
- - If the config differs → the cached **words** are reused and string
12
- representations are rebuilt (cheap, no re-parsing needed)
13
- - If not cached at all → full parse from scratch
14
-
15
- Cache layout::
16
-
17
- {cache_dir}/tablets/{project}/{text_id}.json
18
-
19
- Each file is a JSON wrapper::
20
-
21
- {"config_fingerprint": "a1b2c3d4", "record": { ... TabletRecord ... }}
22
- """
23
- from __future__ import annotations
24
-
25
- import hashlib
26
- import json
27
- from pathlib import Path
28
-
29
- from oracc_parser.utils.logger import get_logger
30
-
31
- logger = get_logger()
32
-
33
-
34
- # ---------------------------------------------------------------------------
35
- # Config fingerprinting
36
- # ---------------------------------------------------------------------------
37
-
38
- # These RunConfig fields affect the parsed output.
39
- # Everything else (USE_CACHE, CACHE_DIR, limit, languages) does NOT.
40
- _OUTPUT_AFFECTING_FIELDS = (
41
- "drop_missing",
42
- "drop_damaged",
43
- "keep_word_segmentation",
44
- "mask_pos",
45
- "max_break_fraction",
46
- )
47
-
48
-
49
- def config_fingerprint(config) -> str:
50
- """Compute a short, stable hash of the output-affecting config options.
51
-
52
- Args:
53
- config: A ``RunConfig`` instance.
54
-
55
- Returns:
56
- 8-char hex string (e.g. ``"a1b2c3d4"``).
57
- """
58
- key = {}
59
- for field in _OUTPUT_AFFECTING_FIELDS:
60
- val = getattr(config, field)
61
- if isinstance(val, list):
62
- val = sorted(val)
63
- key[field] = val
64
-
65
- raw = json.dumps(key, sort_keys=True)
66
- return hashlib.sha256(raw.encode()).hexdigest()[:8]
67
-
68
-
69
- # ---------------------------------------------------------------------------
70
- # Path helpers
71
- # ---------------------------------------------------------------------------
72
-
73
-
74
- def _resolve_cache_dir(cache_dir: str | None = None) -> Path:
75
- """Return the base cache directory."""
76
- if cache_dir:
77
- return Path(cache_dir)
78
- from oracc_parser.settings import CACHE_DIR as settings_CACHE_DIR
79
- return settings_CACHE_DIR
80
-
81
-
82
- def _tablet_path(
83
- project: str,
84
- text_id: str,
85
- cache_dir: str | None = None,
86
- ) -> Path:
87
- """Return the JSON file path for a cached tablet."""
88
- base = _resolve_cache_dir(cache_dir) / "tablets"
89
- project_dir = project.replace("/", "-")
90
- return base / project_dir / f"{text_id}.json"
91
-
92
-
93
- # ---------------------------------------------------------------------------
94
- # Load / Save
95
- # ---------------------------------------------------------------------------
96
-
97
-
98
- def load_cached_tablet(
99
- project: str,
100
- text_id: str,
101
- config,
102
- cache_dir: str | None = None,
103
- ) -> "TabletRecord | None":
104
- """Load a cached tablet, rebuilding string reps only if config changed.
105
-
106
- Two fast paths:
107
-
108
- 1. **Config match** — the cached fingerprint matches the current config.
109
- The full record (including string representations) is returned as-is.
110
- This is the fastest path.
111
-
112
- 2. **Config mismatch** — the words and metadata are reused, but string
113
- representations are rebuilt with the current config. This avoids
114
- the expensive CDL parsing + translation download.
115
-
116
- Args:
117
- project: ORACC project path, e.g. ``"saao/saa01"``.
118
- text_id: Text identifier, e.g. ``"P334189"``.
119
- config: ``RunConfig`` instance.
120
- cache_dir: Custom cache directory (overrides settings).
121
-
122
- Returns:
123
- The TabletRecord (possibly with rebuilt strings), or ``None``.
124
- """
125
- from oracc_parser.models.tablet import TabletRecord
126
- from oracc_parser.parsing.parse_content import (
127
- _add_word_level_representations,
128
- _add_unicode_representation,
129
- )
130
-
131
- path = _tablet_path(project, text_id, cache_dir)
132
- if not path.exists():
133
- return None
134
-
135
- try:
136
- raw = path.read_text(encoding="utf-8")
137
- wrapper = json.loads(raw)
138
-
139
- # Handle both new wrapper format and legacy bare-record format
140
- if "record" in wrapper and "config_fingerprint" in wrapper:
141
- cached_fp = wrapper["config_fingerprint"]
142
- record = TabletRecord.model_validate(wrapper["record"])
143
- else:
144
- # Legacy format (bare TabletRecord JSON) — always rebuild
145
- cached_fp = None
146
- record = TabletRecord.model_validate(wrapper)
147
-
148
- current_fp = config_fingerprint(config)
149
-
150
- if cached_fp == current_fp:
151
- # Fast path: config matches → everything is valid
152
- return record
153
-
154
- # Config changed → rebuild string representations from cached words
155
- record.content = _add_word_level_representations(
156
- record.content, config.mask_pos, config.max_break_fraction
157
- )
158
- record.content = _add_unicode_representation(
159
- record.content,
160
- drop_missing=config.drop_missing,
161
- drop_damaged=config.drop_damaged,
162
- keep_segmentation=config.keep_word_segmentation,
163
- )
164
- return record
165
-
166
- except Exception as e:
167
- logger.warning(f"Corrupt cache file {path}, will re-parse: {e}")
168
- path.unlink(missing_ok=True)
169
- return None
170
-
171
-
172
- def save_tablet_to_cache(
173
- record: "TabletRecord",
174
- project: str,
175
- text_id: str,
176
- config,
177
- cache_dir: str | None = None,
178
- ) -> None:
179
- """Persist a TabletRecord to the JSON cache with a config fingerprint.
180
-
181
- The saved file includes the config fingerprint so that on reload
182
- we can skip string rebuilding when the config hasn't changed.
183
-
184
- Args:
185
- record: The parsed tablet to cache.
186
- project: ORACC project path.
187
- text_id: Text identifier.
188
- config: ``RunConfig`` instance (its fingerprint is stored).
189
- cache_dir: Custom cache directory.
190
- """
191
- path = _tablet_path(project, text_id, cache_dir)
192
- path.parent.mkdir(parents=True, exist_ok=True)
193
-
194
- wrapper = {
195
- "config_fingerprint": config_fingerprint(config),
196
- "record": record.model_dump(mode="python"),
197
- }
198
-
199
- try:
200
- path.write_text(
201
- json.dumps(wrapper, indent=1, default=str, ensure_ascii=False),
202
- encoding="utf-8",
203
- )
204
- except Exception as e:
205
- logger.warning(f"Failed to write cache file {path}: {e}")
206
-
207
-
208
- # ---------------------------------------------------------------------------
209
- # Clear
210
- # ---------------------------------------------------------------------------
211
-
212
-
213
- def clear_project_cache(
214
- project: str | None = None,
215
- cache_dir: str | None = None,
216
- ) -> int:
217
- """Delete cached JSON files for a project (or all projects).
218
-
219
- Args:
220
- project: ORACC project path. ``None`` = clear everything.
221
- cache_dir: Custom cache directory.
222
-
223
- Returns:
224
- Number of tablet JSON files deleted.
225
- """
226
- base = _resolve_cache_dir(cache_dir) / "tablets"
227
- if not base.exists():
228
- return 0
229
-
230
- if project:
231
- target = base / project.replace("/", "-")
232
- else:
233
- target = base
234
-
235
- if not target.exists():
236
- return 0
237
-
238
- count = 0
239
- for f in target.rglob("*.json"):
240
- f.unlink()
241
- count += 1
242
-
243
- # Clean up empty directories (bottom-up)
244
- for d in sorted(target.rglob("*"), reverse=True):
245
- if d.is_dir() and not any(d.iterdir()):
246
- d.rmdir()
247
- if project and target.exists() and not any(target.iterdir()):
248
- target.rmdir()
249
-
250
- logger.info(f"Cleared {count} cached tablet(s)")
251
- return count
@@ -1,205 +0,0 @@
1
- """Tests for oracc_parser.cache — config-independent caching with on-load rebuild."""
2
-
3
- from pathlib import Path
4
-
5
- import pytest
6
-
7
- from oracc_parser.cache import (
8
- clear_project_cache,
9
- config_fingerprint,
10
- load_cached_tablet,
11
- save_tablet_to_cache,
12
- _tablet_path,
13
- )
14
- from oracc_parser.models.config import RunConfig
15
- from oracc_parser.models.tablet import (
16
- Sign,
17
- TabletRecord,
18
- TabletContent,
19
- TabletMetadata,
20
- TextStringRepresentation,
21
- Word,
22
- WordPOSInfo,
23
- WordSigns,
24
- )
25
-
26
-
27
- @pytest.fixture
28
- def cache_dir(tmp_path):
29
- return str(tmp_path / "test_cache")
30
-
31
-
32
- @pytest.fixture
33
- def default_config():
34
- return RunConfig()
35
-
36
-
37
- @pytest.fixture
38
- def sample_record():
39
- """Create a TabletRecord with words that allow string regeneration."""
40
- record = TabletRecord()
41
- record.metadata = TabletMetadata(
42
- identifier="test_tablet",
43
- project="saao/saa01",
44
- id_text="P334189",
45
- genre="Letter",
46
- )
47
-
48
- # Create words with signs so unicode + text reps can be rebuilt
49
- words = [
50
- Word(
51
- frag="a-na",
52
- ref="P334189.1.1",
53
- lemma_form="ana",
54
- norm="ana",
55
- raw_pos="PRP",
56
- lang="akk-x-neoass",
57
- line=1,
58
- normalized_pos=WordPOSInfo(meaning="to", normalized_pos="PRP"),
59
- sign_dictionaries=WordSigns(
60
- reading="a-na",
61
- break_percentage=0.0,
62
- signs=[
63
- Sign(role="phonetic", meaning="a", breakage="complete",
64
- unicode_version="\U00012000"),
65
- Sign(role="phonetic", meaning="na", breakage="complete",
66
- unicode_version="\U0001223E"),
67
- ],
68
- ),
69
- ),
70
- Word(
71
- frag="{m}sargon",
72
- ref="P334189.1.2",
73
- lemma_form="sargon",
74
- norm="sargon",
75
- raw_pos="PN",
76
- lang="akk-x-neoass",
77
- line=1,
78
- normalized_pos=WordPOSInfo(
79
- meaning="Sargon", normalized_pos="PN",
80
- to_mask=False, mask_as=""
81
- ),
82
- sign_dictionaries=WordSigns(
83
- reading="sargon", break_percentage=0.0, signs=[]
84
- ),
85
- ),
86
- ]
87
-
88
- record.content = TabletContent(
89
- english_translation="To Sargon",
90
- words=words,
91
- transliterated_str_representation=TextStringRepresentation(
92
- text="a-na {m}sargon", total_tokens=2, tokens_without_broken=2,
93
- ),
94
- )
95
- return record
96
-
97
-
98
- # ---------------------------------------------------------------------------
99
- # Path structure — simple, no fingerprinting
100
- # ---------------------------------------------------------------------------
101
-
102
-
103
- class TestTabletPath:
104
- def test_path_structure(self, cache_dir):
105
- path = _tablet_path("saao/saa01", "P334189", cache_dir)
106
- expected = Path(cache_dir) / "tablets" / "saao-saa01" / "P334189.json"
107
- assert path == expected
108
-
109
- def test_one_path_per_tablet(self, cache_dir):
110
- """Path is the same regardless of config."""
111
- p1 = _tablet_path("blms", "P1", cache_dir)
112
- p2 = _tablet_path("blms", "P1", cache_dir)
113
- assert p1 == p2
114
-
115
-
116
- # ---------------------------------------------------------------------------
117
- # Save / Load roundtrip
118
- # ---------------------------------------------------------------------------
119
-
120
-
121
- class TestSaveAndLoad:
122
- def test_roundtrip_preserves_words(self, cache_dir, default_config, sample_record):
123
- """Words and metadata survive cache roundtrip."""
124
- save_tablet_to_cache(sample_record, "saao/saa01", "P334189", default_config, cache_dir)
125
- loaded = load_cached_tablet("saao/saa01", "P334189", default_config, cache_dir)
126
-
127
- assert loaded is not None
128
- assert loaded.metadata.identifier == "test_tablet"
129
- assert loaded.metadata.genre == "Letter"
130
- assert loaded.content.english_translation == "To Sargon"
131
- assert len(loaded.content.words) == 2
132
- assert loaded.content.words[0].frag == "a-na"
133
- assert loaded.content.words[1].frag == "{m}sargon"
134
-
135
- def test_string_reps_rebuilt_on_load(self, cache_dir, sample_record):
136
- """String reps are regenerated from words, not loaded from cache blindly."""
137
- save_tablet_to_cache(sample_record, "saao/saa01", "P1", RunConfig(), cache_dir)
138
-
139
- # Load with mask_pos=["PN"] — should mask the PN word
140
- config_masked = RunConfig(mask_pos=["PN"])
141
- loaded = load_cached_tablet("saao/saa01", "P1", config_masked, cache_dir)
142
-
143
- assert loaded is not None
144
- # The transliteration should now contain [PN] mask
145
- t_rep = loaded.content.transliterated_str_representation
146
- assert t_rep is not None
147
- assert "PN" in t_rep.text
148
-
149
- def test_same_cache_different_configs(self, cache_dir, sample_record):
150
- """Same cached file serves different configs correctly."""
151
- save_tablet_to_cache(sample_record, "proj", "P1", RunConfig(), cache_dir)
152
-
153
- # Load with default (no masking)
154
- loaded_default = load_cached_tablet("proj", "P1", RunConfig(), cache_dir)
155
- # Load with PN masking
156
- loaded_masked = load_cached_tablet("proj", "P1", RunConfig(mask_pos=["PN"]), cache_dir)
157
-
158
- assert loaded_default is not None
159
- assert loaded_masked is not None
160
-
161
- # Both exist, but transliteration text should differ
162
- text_default = loaded_default.content.transliterated_str_representation.text
163
- text_masked = loaded_masked.content.transliterated_str_representation.text
164
- assert text_default != text_masked
165
-
166
- def test_cache_miss(self, cache_dir, default_config):
167
- result = load_cached_tablet("saao/saa01", "NONEXISTENT", default_config, cache_dir)
168
- assert result is None
169
-
170
- def test_corrupt_file_returns_none(self, cache_dir, default_config):
171
- path = _tablet_path("saao/saa01", "CORRUPT", cache_dir)
172
- path.parent.mkdir(parents=True, exist_ok=True)
173
- path.write_text("NOT VALID JSON {{{{", encoding="utf-8")
174
-
175
- result = load_cached_tablet("saao/saa01", "CORRUPT", default_config, cache_dir)
176
- assert result is None
177
- assert not path.exists()
178
-
179
-
180
- # ---------------------------------------------------------------------------
181
- # Clear cache
182
- # ---------------------------------------------------------------------------
183
-
184
-
185
- class TestClearCache:
186
- def test_clear_project(self, cache_dir, default_config, sample_record):
187
- save_tablet_to_cache(sample_record, "saao/saa01", "P1", default_config, cache_dir)
188
- save_tablet_to_cache(sample_record, "saao/saa01", "P2", default_config, cache_dir)
189
- save_tablet_to_cache(sample_record, "blms", "P3", default_config, cache_dir)
190
-
191
- deleted = clear_project_cache("saao/saa01", cache_dir)
192
- assert deleted == 2
193
- assert load_cached_tablet("blms", "P3", default_config, cache_dir) is not None
194
- assert load_cached_tablet("saao/saa01", "P1", default_config, cache_dir) is None
195
-
196
- def test_clear_all(self, cache_dir, default_config, sample_record):
197
- save_tablet_to_cache(sample_record, "saao/saa01", "P1", default_config, cache_dir)
198
- save_tablet_to_cache(sample_record, "blms", "P2", default_config, cache_dir)
199
-
200
- deleted = clear_project_cache(project=None, cache_dir=cache_dir)
201
- assert deleted == 2
202
- assert load_cached_tablet("saao/saa01", "P1", default_config, cache_dir) is None
203
-
204
- def test_clear_empty(self, cache_dir):
205
- assert clear_project_cache(project=None, cache_dir=cache_dir) == 0
File without changes
File without changes