keybridge 1.0.0__py3-none-any.whl

keybridge/__init__.py

@@ -0,0 +1,6 @@
+"""Shared-space decipherment of simple and homophonic substitution ciphers.
+
+Train a model for a known key, read transcriptions with it, and test transcriptions
+against a collection of models. See the README for the notebooks and the command line.
+"""
+__version__ = "1.0.0"

keybridge/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

keybridge/cli.py

@@ -0,0 +1,329 @@
+"""Command-line interface: the same workflows as the notebooks, run on your own computer.
+
+    keybridge train   --key my_key.tsv --format space --plaintext text1.txt text2.txt --name my_cipher
+    keybridge read    --model my_cipher_model.pt --transcription letter.txt
+    keybridge scan    --transcription letter.txt --format char --models all
+    keybridge convert --key old_key.tsv --transcription letter.txt     (to the standard format)
+    keybridge models  list | download
+    keybridge languages
+    keybridge evaluate --model my_cipher_model.pt --transcription letter.txt   (research: F1 against the key)
+    keybridge index-entry my_cipher_model.pt --add-to index.json      (for maintainers)
+
+Run any command with --help for its options.
+"""
+import argparse
+import importlib.util
+import json
+import os
+import sys
+
+from . import __version__
+from .common import StopHere, resolve_format, safe_name
+from .config import PLAINTEXT_DATASETS
+
+
+def _load_custom_edits(path):
+    if not path:
+        return None, None
+    spec = importlib.util.spec_from_file_location("custom_edits", path)
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return (getattr(mod, "custom_edit_training_units", None),
+            getattr(mod, "custom_edit_transcription_line", None))
+
+
+def cmd_train(a):
+    from .data import (load_builtin_plaintext, load_plaintext_files, parse_replacements,
+                       resolve_plaintext_language)
+    from .keys import load_key
+    from .pipeline import plaintext_chars_needed, TrainSettings, describe_fit, fit_figures, prepare_data, preview, read_lines, \
+        save_training_outputs, train, write_reading
+    from .transcription import read_transcription
+    from .model import load_model
+
+    fmt = resolve_format(a.format)
+    name = safe_name(a.name)
+    s = TrainSettings(project=name, symbol_format=fmt,
+                      plaintext_source="own files" if a.plaintext else "English", plaintext_files=a.plaintext or [],
+                      letter_replacements=a.replacements, word_spaces=a.word_spaces,
+                      training_examples=a.examples, sequence_length=a.length, null_rate=a.null_rate,
+                      double_rate=a.double_rate, rare_unit_rate=a.rare_rate,
+                      search_trials=a.search_trials, seed=a.seed, max_epochs=a.max_epochs)
+    train_edit, line_edit = _load_custom_edits(a.custom_edits)
+    key = load_key(a.key, fmt)
+    print(key.summary())
+    for w in key.warnings:
+        print("  •", w)
+    reps = parse_replacements(a.replacements)
+    heldout = None
+    if a.plaintext:
+        docs = load_plaintext_files(a.plaintext, reps)
+    else:
+        # Built-in plaintext: English unless --language says otherwise.
+        lang = resolve_plaintext_language(a.language)
+        s.plaintext_source = lang
+        print(f"Using the built-in {lang} plaintext.")
+        train_chars, heldout_chars = plaintext_chars_needed(a.examples, a.length)
+        docs = load_builtin_plaintext(lang, reps, train_chars, "train")
+        heldout = load_builtin_plaintext(lang, reps, heldout_chars, "validation")
+    data = prepare_data(key, docs, s, custom_edit=train_edit, heldout_documents=heldout)
+    result = train(key, data, s)
+    out_dir = os.path.join(a.out, name)
+    model_path = save_training_outputs(out_dir, s, key, a.key, data, result)
+    print(f"\nSaved to {out_dir}")
+    if a.transcription:
+        model, ck = load_model(model_path)
+        lines = read_transcription(a.transcription, fmt, a.skip.split(","), line_edit, a.length)
+        readings = read_lines(model, ck, lines)
+        write_reading(out_dir, name, readings, fmt, f"Decipherment of {os.path.basename(a.transcription)}",
+                      checkpoint=ck)
+        preview(readings)
+        describe_fit(fit_figures(readings, ck))
+
+
+def _registry(a):
+    from .registry import ModelRegistry
+    return ModelRegistry(a.models_from)
+
+
+def cmd_read(a):
+    from .model import load_model
+    from .pipeline import describe_fit, fit_figures, preview, read_lines, write_reading
+    from .transcription import read_transcription
+
+    path = a.model if os.path.exists(a.model) else None
+    if path is None:
+        reg = _registry(a)
+        path = reg.model_path(reg.entries(ids=[a.model])[0])
+    model, ck = load_model(path)
+    fmt = resolve_format(a.format) if a.format else ck["settings"].get("symbol_format")
+    _, line_edit = _load_custom_edits(a.custom_edits)
+    lines = read_transcription(a.transcription, fmt, a.skip.split(","), line_edit,
+                               ck["settings"].get("sequence_length"))
+    readings = read_lines(model, ck, lines)
+    name = safe_name(os.path.splitext(os.path.basename(a.transcription))[0])
+    files = write_reading(a.out, name, readings, fmt,
+                          f"Decipherment of {os.path.basename(a.transcription)} with {os.path.basename(path)}",
+                          checkpoint=ck)
+    print(f"Read {len(readings)} lines:\n")
+    preview(readings)
+    print()
+    describe_fit(fit_figures(readings, ck))
+    print("\nWritten:", *files, sep="\n  ")
+
+
+def cmd_scan(a):
+    from .scan import describe_ranking, scan, top_group, write_scan_outputs
+    from .transcription import read_transcription
+
+    fmt = resolve_format(a.format)
+    _, line_edit = _load_custom_edits(a.custom_edits)
+    lines = read_transcription(a.transcription, fmt, a.skip.split(","), line_edit, a.max_length)
+    entries, reg = [], None
+    if a.models and a.models != "none":
+        reg = _registry(a)
+        if a.models in ("all", "digits", "characters", "named"):
+            entries = reg.entries(symbol_type=None if a.models == "all" else a.models)
+        else:
+            entries = reg.entries(ids=[m.strip() for m in a.models.split(",")])
+    if not entries and not a.model_file:
+        raise StopHere("No models to test. Give your own with --model-file.")
+    print(f"Testing {len(entries) + len(a.model_file or [])} model(s) on {len(lines)} lines...")
+    langs = set(a.languages.split(",")) if a.languages else None
+    results = scan(lines, fmt, reg, entries, a.model_file or [], langs, a.min_coverage)
+    top = top_group(results, a.margin, a.max_close)
+    print()
+    describe_ranking(results, top)
+    name = safe_name(os.path.splitext(os.path.basename(a.transcription))[0])
+    all_files, _ = write_scan_outputs(a.out, name, results, top, os.path.basename(a.transcription), a.all_outputs)
+    print("\nWritten:", *all_files, sep="\n  ")
+
+
+def cmd_evaluate(a):
+    from .evaluate import describe, evaluate, write_evaluation
+    from .keys import load_key
+    from .model import load_model
+    from .transcription import read_transcription
+
+    path = a.model if os.path.exists(a.model) else None
+    if path is None:
+        reg = _registry(a)
+        path = reg.model_path(reg.entries(ids=[a.model])[0])
+    model, ck = load_model(path)
+    fmt = resolve_format(a.format) if a.format else ck["settings"].get("symbol_format")
+    gold_key = load_key(a.key) if a.key else None
+    _, line_edit = _load_custom_edits(a.custom_edits)
+    lines = read_transcription(a.transcription, fmt, a.skip.split(","), line_edit,
+                               ck["settings"].get("sequence_length"))
+    exclude = [u.strip().upper() for u in a.exclude_units.split(",") if u.strip()] if a.exclude_units else []
+    result, units, per_line, readings = evaluate(model, ck, lines, gold_key, a.exclude_unknown, exclude,
+                                                 a.bootstrap, a.seed)
+    describe(result, units)
+    name = safe_name(a.name or os.path.splitext(os.path.basename(a.transcription))[0])
+    files = write_evaluation(a.out, name, result, units, per_line, readings)
+    if a.reference_plaintext:
+        from .evaluate import (describe_reference_check, read_reference_plaintext, reference_check,
+                               write_reference_check)
+        from .keys import CipherKey
+        ref_lines = read_reference_plaintext(a.reference_plaintext, a.skip.split(","))
+        summary, rows = reference_check(readings, gold_key or CipherKey.from_json(ck["key"]), ref_lines)
+        describe_reference_check(summary)
+        files += write_reference_check(a.out, name, summary, rows)
+    print("\nWritten:", *files, sep="\n  ")
+
+
+def cmd_convert(a):
+    from .convert import convert_key, convert_transcription
+    if not a.key and not a.transcription:
+        raise StopHere("Give --key and/or --transcription to convert.")
+    if a.key:
+        out, key = convert_key(a.key, a.key_out)
+        print(f"Key written in the standard format: {out}"
+              + (' (blank symbol " " written as "_")' if key.blank_converted else ""))
+    for path in a.transcription or []:
+        out, n = convert_transcription(path, None, a.skip.split(","))
+        print(f"Transcription converted ({n} lines): {out}")
+
+
+def cmd_models(a):
+    reg = _registry(a)
+    entries = reg.entries()
+    if a.action == "list":
+        print(f"{len(entries)} model(s) in {reg.source}:")
+        for e in entries:
+            print(f"  {e['id']:<28} {e.get('symbol_format', ''):<28} {e.get('symbol_type', ''):<10} {e.get('name', '')}")
+    else:
+        for e in entries:
+            print("  downloading", e["id"], "->", reg.model_path(e))
+        print("Done. To work offline from now on, set the environment variable HF_HUB_OFFLINE=1.")
+
+
+def cmd_languages(a):
+    from .languages import available_languages
+    langs = available_languages()
+    if not langs:
+        print("No language profiles installed.")
+    for code, (name, path) in sorted(langs.items()):
+        print(f"  {code:<4} {name:<12} {path}")
+
+
+def cmd_index_entry(a):
+    from .registry import make_index_entry, new_index
+    entry = make_index_entry(a.model, a.id)
+    if a.add_to:
+        index = new_index()
+        if os.path.exists(a.add_to):
+            with open(a.add_to, encoding="utf-8") as f:
+                index = json.load(f)
+        index["models"] = [m for m in index["models"] if m["id"] != entry["id"]] + [entry]
+        with open(a.add_to, "w", encoding="utf-8") as f:
+            json.dump(index, f, ensure_ascii=False, indent=2)
+        print(f"Added '{entry['id']}' to {a.add_to}. Complete the fields marked FILL IN.")
+    else:
+        print(json.dumps(entry, ensure_ascii=False, indent=2))
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(prog="keybridge", description=__doc__,
+                                formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--version", action="version", version=__version__)
+    sub = p.add_subparsers(dest="command", required=True)
+    fmt_help = "transcription format: 'space' (symbols separated by spaces) or 'char' (one character per symbol)"
+
+    t = sub.add_parser("train", help="train a model for a key")
+    t.add_argument("--key", required=True, help="key file (.tsv)")
+    t.add_argument("--format", required=True, help=fmt_help)
+    t.add_argument("--plaintext", nargs="+", help="your own plaintext .txt file(s); without this, built-in plaintext is used")
+    t.add_argument("--language", default="English",
+                   help="language of the built-in plaintext (default English): " + ", ".join(PLAINTEXT_DATASETS))
+    t.add_argument("--name", default="my_cipher")
+    t.add_argument("--replacements", default="", help="letter replacements, e.g. 'J=I, V=U'")
+    t.add_argument("--word-spaces", default="keep real word boundaries",
+                   choices=["keep real word boundaries", "insert at random", "remove"])
+    t.add_argument("--examples", type=int, default=5000, help="number of synthetic training examples (default 5000)")
+    t.add_argument("--length", type=int, default=50)
+    t.add_argument("--null-rate", type=float, default=0.05)
+    t.add_argument("--double-rate", type=float, default=0.5)
+    t.add_argument("--rare-rate", type=float, default=0.02,
+                   help="insert key units that are rare or missing in the plaintext at this rate per word gap (default 0.02; 0 = off)")
+    t.add_argument("--search-trials", type=int, default=20,
+                   help="short trials of the model-settings search (default 20; 0 = skip the search and use the fixed default settings)")
+    t.add_argument("--seed", type=int, default=9)
+    t.add_argument("--max-epochs", type=int, default=100, help="upper limit on training rounds")
+    t.add_argument("--transcription", help="also read this transcription after training")
+    t.add_argument("--skip", default="##", help="skip transcription lines starting with these (comma-separated)")
+    t.add_argument("--custom-edits", help="Python file defining custom_edit_training_units / custom_edit_transcription_line")
+    t.add_argument("--out", default="outputs")
+    t.set_defaults(func=cmd_train)
+
+    for name, func, helptext in (("read", cmd_read, "read a transcription with one model"),
+                                 ("scan", cmd_scan, "test a transcription against many models")):
+        r = sub.add_parser(name, help=helptext)
+        r.add_argument("--transcription", required=True)
+        r.add_argument("--skip", default="##")
+        r.add_argument("--custom-edits")
+        r.add_argument("--out", default="outputs")
+        r.add_argument("--models-from", help="Hugging Face repo id or local folder of the model collection")
+        r.set_defaults(func=func)
+        if name == "read":
+            r.add_argument("--model", required=True, help="model file (.pt) or model ID from the collection")
+            r.add_argument("--format", help=fmt_help + " (default: the model's own)")
+        else:
+            r.add_argument("--format", required=True, help=fmt_help)
+            r.add_argument("--models", default="all",
+                           help="'all', 'digits', 'characters', 'named', comma-separated model IDs, or 'none'")
+            r.add_argument("--model-file", action="append", help="your own model file (repeatable)")
+            r.add_argument("--languages", help="limit language scoring to these codes, e.g. de,la,it")
+            r.add_argument("--min-coverage", type=float, default=0.75)
+            r.add_argument("--margin", type=float, default=0.05)
+            r.add_argument("--max-close", type=int, default=3)
+            r.add_argument("--max-length", type=int, default=50, help="longer lines are read in pieces")
+            r.add_argument("--all-outputs", action="store_true", help="write decipherments for every model")
+
+    ev = sub.add_parser("evaluate", help="(research) score a model's reading against the key")
+    ev.add_argument("--model", required=True, help="model file (.pt) or model ID from the collection")
+    ev.add_argument("--transcription", required=True)
+    ev.add_argument("--format", help=fmt_help + " (default: the model's own)")
+    ev.add_argument("--key", help="key to derive the reference from (default: the key stored in the model)")
+    ev.add_argument("--reference-plaintext", help="the archive's plaintext transcription, line by line, to check "
+                                                   "the key-derived reference against")
+    ev.add_argument("--skip", default="##")
+    ev.add_argument("--custom-edits", help="Python file defining custom_edit_transcription_line")
+    ev.add_argument("--exclude-unknown", action="store_true", help="do not score symbols that are not in the key")
+    ev.add_argument("--exclude-units", help="comma-separated plaintext units not to score, e.g. 'MASTER,LODGE'")
+    ev.add_argument("--bootstrap", type=int, default=1000, help="line-bootstrap resamples for CIs (0: none)")
+    ev.add_argument("--seed", type=int, default=0)
+    ev.add_argument("--name", help="name for the output files (default: transcription file name)")
+    ev.add_argument("--models-from")
+    ev.add_argument("--out", default="outputs")
+    ev.set_defaults(func=cmd_evaluate)
+
+    cv = sub.add_parser("convert", help="convert a key and/or 'one character per symbol' transcriptions "
+                                        "to the standard format")
+    cv.add_argument("--key", help="key file to rewrite in the standard format")
+    cv.add_argument("--key-out", help="output file for the key (default: <name>_standard.tsv)")
+    cv.add_argument("--transcription", nargs="+", help="'one character per symbol' transcription(s) to convert "
+                                                       "(each written to <name>_standard.txt)")
+    cv.add_argument("--skip", default="##", help="note lines starting with these are copied unchanged")
+    cv.set_defaults(func=cmd_convert)
+
+    m = sub.add_parser("models", help="list or download the shared models")
+    m.add_argument("action", choices=["list", "download"])
+    m.add_argument("--models-from")
+    m.set_defaults(func=cmd_models)
+
+    sub.add_parser("languages", help="list installed language profiles").set_defaults(func=cmd_languages)
+
+    ie = sub.add_parser("index-entry", help="(maintainers) make an index.json entry for a model")
+    ie.add_argument("model")
+    ie.add_argument("--id")
+    ie.add_argument("--add-to", help="index.json to add the entry to (created if missing)")
+    ie.set_defaults(func=cmd_index_entry)
+
+    a = p.parse_args(argv)
+    try:
+        a.func(a)
+    except StopHere as e:
+        print(str(e), file=sys.stderr)
+        sys.exit(2)
+

keybridge/colab.py

@@ -0,0 +1,79 @@
+"""Helpers for the Colab notebooks (uploads, downloads, Google Drive). They also work
+in a local Jupyter notebook, where uploads are replaced by file paths."""
+import os
+import shutil
+import zipfile
+
+from .common import stop
+
+
+def in_colab():
+    try:
+        import google.colab  # noqa: F401
+        return True
+    except ImportError:
+        return False
+
+
+def find_or_upload(value, what, upload_dir, multiple=False, required=True):
+    """Returns path(s) given in `value` (comma-separated paths or folders), or asks for an upload."""
+    os.makedirs(upload_dir, exist_ok=True)
+    paths = [p.strip() for p in (value or "").split(",") if p.strip()]
+    found = []
+    for p in paths:
+        if os.path.isdir(p):
+            found += [os.path.join(p, f) for f in sorted(os.listdir(p)) if not f.startswith(".")]
+        elif os.path.exists(p):
+            found.append(p)
+        elif os.path.exists(os.path.join(upload_dir, p)):
+            found.append(os.path.join(upload_dir, p))
+        else:
+            stop(f"The {what} could not be found at: {p}\nLeave the box empty to upload it instead, or check the path.")
+    if found:
+        return found if multiple else found[0]
+    if not in_colab():
+        if required:
+            stop(f"Please give the path to your {what} in step ①.")
+        return [] if multiple else None
+    from google.colab import files
+    print(f"Please upload your {what}" + (" (you may select several files)" if multiple else "") + ":")
+    uploaded = files.upload()
+    if not uploaded:
+        if required:
+            stop(f"No {what} was uploaded. Run this step again to try once more.")
+        return [] if multiple else None
+    saved = []
+    for name, data in uploaded.items():
+        dest = os.path.join(upload_dir, name)
+        with open(dest, "wb") as f:
+            f.write(data)
+        saved.append(dest)
+    return saved if multiple else saved[0]
+
+
+def zip_files(paths, zip_path, folder_name):
+    with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as z:
+        for p in paths:
+            z.write(p, arcname=f"{folder_name}/{os.path.basename(p)}")
+    return zip_path
+
+
+def offer_download(path):
+    if in_colab():
+        from google.colab import files
+        print(f"Downloading {os.path.basename(path)} ... (if nothing happens: open the folder icon on the left, "
+              "find the file, right-click → Download)")
+        files.download(path)
+    else:
+        print("Saved:", path)
+
+
+def copy_to_drive(folder, subfolder):
+    if not in_colab():
+        return None
+    from google.colab import drive
+    drive.mount("/content/drive")
+    dest = f"/content/drive/MyDrive/{subfolder}"
+    shutil.copytree(folder, dest, dirs_exist_ok=True)
+    print(f"Copied to your Google Drive: My Drive/{subfolder}")
+    return dest

keybridge/common.py

@@ -0,0 +1,72 @@
+"""Shared constants, the friendly error type and small file helpers."""
+import re
+
+# Special tokens shared by both vocabularies (same index in both).
+PAD, SOS, EOS, UNK = "<PAD>", "<SOS>", "<EOS>", "<UNK>"
+SPECIAL_TOKENS = [PAD, SOS, EOS, UNK]
+
+# Special plaintext units a key may use.
+SPACE_UNIT = " "            # <SPACE> in a key: the gap between words
+NULL_UNIT = "<NULL>"        # meaningless filler symbols
+DOUBLE_UNIT = "<DOUBLE>"    # "repeat the previous letter" sign
+SPECIAL_UNIT_NAMES = {"<SPACE>": SPACE_UNIT, "<NULL>": NULL_UNIT, "<DOUBLE>": DOUBLE_UNIT}
+SPECIAL_UNITS = set(SPECIAL_UNIT_NAMES.values())
+
+# A blank space used as a cipher symbol (e.g. a word divider) is always written as this
+# visible symbol, in keys, transcriptions and outputs, so that every key and transcription
+# can use the standard "symbols separated by spaces" format.
+BLANK_SYMBOL = "_"
+
+# Transcription formats. "symbols separated by spaces" is the standard; "one character per
+# symbol" files are read character by character, with blanks turned into BLANK_SYMBOL.
+SPACE_SEPARATED = "symbols separated by spaces"
+ONE_CHAR = "one character per symbol"
+FORMATS = (SPACE_SEPARATED, ONE_CHAR)
+FORMAT_ALIASES = {"space": SPACE_SEPARATED, "spaces": SPACE_SEPARATED, "char": ONE_CHAR,
+                  "chars": ONE_CHAR, "character": ONE_CHAR}
+
+
+def resolve_format(name):
+    name = FORMAT_ALIASES.get(name, name)
+    if name not in FORMATS:
+        stop(f"Unknown transcription format {name!r}. Use 'space' ({SPACE_SEPARATED}) "
+             f"or 'char' ({ONE_CHAR}).")
+    return name
+
+
+class StopHere(Exception):
+    """Raised with a plain-language message when something needs fixing."""
+
+
+def stop(message):
+    raise StopHere("\n\n" + message.strip() + "\n")
+
+
+def read_text_file(path):
+    with open(path, "rb") as f:
+        raw = f.read()
+    if raw.startswith((b"\xff\xfe", b"\xfe\xff")):
+        return raw.decode("utf-16")
+    for enc in ("utf-8-sig", "cp1252"):
+        try:
+            return raw.decode(enc)
+        except UnicodeDecodeError:
+            continue
+    return raw.decode("latin-1")
+
+
+def split_symbols(text, symbol_format):
+    """Split one transcription line into cipher symbols. In 'one character per symbol' text,
+    a blank is a symbol and becomes BLANK_SYMBOL."""
+    if symbol_format == SPACE_SEPARATED:
+        return text.split()
+    return [BLANK_SYMBOL if c == " " else c for c in text]
+
+
+def safe_name(text, default="my_cipher"):
+    return re.sub(r"[^A-Za-z0-9_-]+", "_", (text or "").strip()).strip("_") or default
+
+
+def say(log, *parts):
+    if log:
+        log(" ".join(str(p) for p in parts))

keybridge/config.py

@@ -0,0 +1,36 @@
+"""Names and addresses that the maintainer sets once.
+
+Every value marked PLACEHOLDER must be replaced when the GitHub and Hugging Face
+repositories exist. See docs/MAINTAINER_SETUP.md for the full list.
+"""
+
+# GitHub repository holding this code (owner/name).
+GITHUB_REPO = "mbruton0426/keybridge"
+
+# Hugging Face model repository holding index.json and one folder per key (owner/name).
+# A local folder with the same layout also works (for testing, or fully offline use).
+HF_MODEL_REPO = "mbruton/keybridge-models"
+
+# Where people send shared-model bundles by email (empty: no address is shown).
+SHARE_CONTACT_EMAIL = 'micaella.bruton@ling.su.se'
+
+# Built-in plaintext for training: one public Hugging Face dataset per language, each with
+# "train" and "validation" splits and a "plaintext" column (historical text, HistCorp).
+# Load them with datasets.load_dataset(repo); texts are ds["train"]["plaintext"] etc.
+PLAINTEXT_DATASETS = {
+    "English": "mbruton/english_encrypted_HistCiph",
+    "Czech": "mbruton/czech_encrypted_HistCiph",
+    "Dutch": "mbruton/dutch_encrypted_HistCiph",
+    "French": "mbruton/french_encrypted_HistCiph",
+    "Hungarian": "mbruton/hungarian_encrypted_HistCiph",
+    "Icelandic": "mbruton/icelandic_encrypted_HistCiph",
+    "Italian": "mbruton/italian_encrypted_HistCiph",
+    "Polish": "mbruton/polish_encrypted_HistCiph",
+    "Spanish": "mbruton/spanish_encrypted_HistCiph",
+    "Swedish": "mbruton/swedish_encrypted_HistCiph",
+}
+DEFAULT_PLAINTEXT_LANGUAGE = "English"
+PLAINTEXT_FIELD = "plaintext"
+
+CITATION = "[citation to be added]"  # PLACEHOLDER
+LICENSE = "CC-BY-4.0"  # licence of the models and keys in the collection (the code is MIT)

keybridge/convert.py

@@ -0,0 +1,54 @@
+"""Converting keys and transcriptions to the standard format.
+
+Standard format: cipher symbols separated by spaces, and a blank space used as a cipher
+symbol (e.g. a word divider) written as "_" (BLANK_SYMBOL).
+"""
+import os
+
+from .common import ONE_CHAR, read_text_file, split_symbols
+from .keys import load_key
+
+
+def standard_path(path, ext=None):
+    stem, old_ext = os.path.splitext(path)
+    return f"{stem}_standard{ext or old_ext}"
+
+
+def convert_transcription(path, out_path=None, skip_prefixes=("##",)):
+    """Converts a 'one character per symbol' transcription: every character becomes a symbol,
+    separated by spaces, with blanks written as '_'. Empty lines and note lines (starting with
+    a skip marker) are copied unchanged. Returns (output path, lines converted)."""
+    out_path = out_path or standard_path(path)
+    out, converted = [], 0
+    for raw in read_text_file(path).splitlines():
+        line = raw.rstrip("\r\n")
+        if not line.strip() or any(line.lstrip().startswith(p) for p in skip_prefixes if p):
+            out.append(line)
+            continue
+        out.append(" ".join(split_symbols(line.strip(), ONE_CHAR)))
+        converted += 1
+    with open(out_path, "w", encoding="utf-8") as f:
+        f.write("\n".join(out) + "\n")
+    return out_path, converted
+
+
+def convert_key(path, out_path=None):
+    """Rewrites a key (.tsv or .json) in the standard .tsv format, with a blank symbol written
+    as '_'. Note lines at the top of a .tsv key are kept. Returns (output path, key)."""
+    out_path = out_path or standard_path(path, ".tsv")
+    key = load_key(path)
+    notes = []
+    if path.lower().endswith((".tsv", ".txt")):
+        for line in read_text_file(path).splitlines():
+            if line.lstrip().startswith("#"):
+                notes.append(line)
+            elif line.strip():
+                break
+    key.write_tsv(out_path)
+    if notes:
+        with open(out_path, encoding="utf-8") as f:
+            body = f.read()
+        with open(out_path, "w", encoding="utf-8") as f:
+            f.write("\n".join(notes) + "\n" + body)
+    return out_path, key
+