django-flashdeck 0.1.0__py3-none-any.whl

flashdeck/bundles.py

@@ -0,0 +1,346 @@
+"""Bundles: a subject's decks and cards, with their images, as a directory or zip
+file holding cards.yaml and the image files it names. Bundles are how decks are
+written, generated, moved between sites and passed on; the README describes the
+format.
+"""
+
+import io
+import zipfile
+from collections import Counter
+from pathlib import Path, PurePosixPath
+
+import yaml
+from django.core.exceptions import ValidationError
+from django.core.files import File
+from django.core.validators import validate_slug
+from django.db import transaction
+
+from .images import prepare_image
+from .manifest import FORMAT, MANIFEST, dump_manifest
+from .marking import normalise
+from .models import RESERVED_SLUGS, Answer, Card, Deck, Subject
+
+# The largest file a bundle may hold, uncompressed.
+MAX_FILE_SIZE = 20 * 1024 * 1024
+
+
+class BundleError(Exception):
+    pass
+
+
+class _Source:
+    """Reads files from a bundle directory or zip file."""
+
+    def __init__(self, path):
+        self.path = Path(path)
+        self.zip = None
+        self.prefix = PurePosixPath()
+        if self.path.is_file():
+            if not zipfile.is_zipfile(self.path):
+                raise BundleError(f"{self.path} is neither a directory nor a zip file.")
+            self.zip = zipfile.ZipFile(self.path)
+            # Allow the bundle to be one directory down, as when a directory is
+            # zipped as a whole.
+            manifests = [
+                PurePosixPath(n)
+                for n in self.zip.namelist()
+                if PurePosixPath(n).name == MANIFEST
+            ]
+            if not manifests:
+                raise BundleError(f"There is no {MANIFEST} in {self.path}.")
+            self.prefix = min(manifests, key=lambda p: len(p.parts)).parent
+        elif not self.path.is_dir():
+            raise BundleError(f"There is no bundle at {self.path}.")
+
+    def read(self, name):
+        path = PurePosixPath(str(name))
+        if path.is_absolute() or ".." in path.parts or not path.parts:
+            raise BundleError(f"{name!r} is not a path within the bundle.")
+        try:
+            if self.zip:
+                info = self.zip.getinfo(str(self.prefix / path))
+                if info.file_size > MAX_FILE_SIZE:
+                    raise BundleError(f"{name} is larger than 20 MB.")
+                return self.zip.read(info)
+            return self.path.joinpath(*path.parts).read_bytes()
+        except (KeyError, FileNotFoundError, IsADirectoryError) as error:
+            raise BundleError(f"{name} is not in the bundle.") from error
+
+    def close(self):
+        if self.zip:
+            self.zip.close()
+
+
+def import_bundle(path, *, update=False):
+    """Imports a bundle, adding the subject, decks and cards that are not on the
+    site (matched by slug and by key). With `update`, existing ones are updated
+    too, except that answers accepted on the site are kept, an image is replaced
+    only if it has changed, and whether a deck is public is left as it is. Returns
+    counts of what was done."""
+    source = _Source(path)
+    try:
+        try:
+            data = yaml.safe_load(source.read(MANIFEST))
+        except yaml.YAMLError as error:
+            raise BundleError(f"{MANIFEST} is not valid YAML: {error}") from error
+        if not isinstance(data, dict) or data.get("format") != FORMAT:
+            raise BundleError(f"{MANIFEST} must start with “format: {FORMAT}”.")
+        with transaction.atomic():
+            return _import(data, source, update)
+    finally:
+        source.close()
+
+
+def _import(data, source, update):
+    counts = Counter()
+    fields = _mapping(data.get("subject"), "subject")
+    slug = _slug(fields.get("slug"), "subject", page_name=True)
+    characters = fields.get("characters") or ""
+    if isinstance(characters, list):
+        characters = " ".join(_text(c, "subject characters") for c in characters)
+    values = {
+        "title": _text(fields.get("title"), "subject title", required=True),
+        "description": _text(fields.get("description"), "subject description"),
+        "reference": _text(fields.get("reference"), "subject reference"),
+        "language": _text(fields.get("language"), "subject language"),
+        "characters": _text(characters, "subject characters"),
+        "case_sensitive": bool(fields.get("case_sensitive", False)),
+    }
+    subject, created = Subject.objects.get_or_create(slug=slug, defaults=values)
+    _count(counts, "subject", created, update, subject, values)
+
+    for position, deck_data in enumerate(data.get("decks") or []):
+        deck_data = _mapping(deck_data, f"deck {position + 1}")
+        slug = _slug(deck_data.get("slug"), f"deck {position + 1}", page_name=True)
+        where = f"deck {slug}"
+        values = {
+            "title": _text(deck_data.get("title"), f"{where}: title", required=True),
+            "description": _text(deck_data.get("description"), f"{where}: description"),
+            "reference": _text(deck_data.get("reference"), f"{where}: reference"),
+            "is_public": bool(deck_data.get("public", False)),
+            "position": position,
+        }
+        deck, created = Deck.objects.get_or_create(
+            subject=subject, slug=slug, defaults=values
+        )
+        # Whether a deck is public is for the site to decide, so an update leaves
+        # it as it is.
+        values.pop("is_public")
+        _count(counts, "deck", created, update, deck, values)
+        for card_position, card_data in enumerate(deck_data.get("cards") or []):
+            _import_card(deck, card_data, card_position, source, update, counts)
+    return counts
+
+
+def _count(counts, kind, created, update, obj, values):
+    if created:
+        counts[f"{kind}s created"] += 1
+    elif update:
+        for name, value in values.items():
+            setattr(obj, name, value)
+        obj.save()
+        counts[f"{kind}s updated"] += 1
+
+
+def _import_card(deck, data, position, source, update, counts):
+    data = _mapping(data, f"deck {deck.slug}: card {position + 1}")
+    key = _slug(data.get("key"), f"deck {deck.slug}: card {position + 1}")
+    where = f"deck {deck.slug}: card {key}"
+    answers = _answers(data, where)
+    card = Card.objects.filter(deck=deck, key=key).first()
+    if card is not None and not update:
+        counts["cards unchanged"] += 1
+        return
+    new = card is None
+    if new:
+        card = Card(deck=deck, key=key)
+    card.prompt = _text(data.get("prompt"), f"{where}: prompt")
+    card.notes = _text(data.get("notes"), f"{where}: notes")
+    card.image_alt = _text(data.get("image_alt"), f"{where}: image_alt")
+    card.image_credit = _text(data.get("image_credit"), f"{where}: image_credit")
+    card.image_licence = _text(data.get("image_licence"), f"{where}: image_licence")
+    card.position = position
+    image = data.get("image")
+    if image:
+        content = source.read(image)
+        if not card.image or _image_changed(card, content, image):
+            card.image = File(io.BytesIO(content), name=PurePosixPath(image).name)
+    if not card.prompt and not card.image:
+        raise BundleError(f"{where}: a card needs a prompt, an image or both.")
+    try:
+        card.full_clean(exclude=["deck"])
+    except ValidationError as error:
+        raise BundleError(f"{where}: {'; '.join(error.messages)}") from error
+    card.save()
+
+    existing = [] if new else list(card.answers.all())
+    for sort_order, values in enumerate(answers):
+        if sort_order < len(existing):
+            answer = existing[sort_order]
+            kept = answer.accepted()[1:]
+        else:
+            answer, kept = Answer(card=card), []
+        answer.label = values["label"]
+        answer.text = values["text"]
+        answer.alternatives = "\n".join(
+            _distinct(values["alternatives"] + kept, exclude=values["text"])
+        )
+        answer.sort_order = sort_order
+        answer.save()
+    for answer in existing[len(answers) :]:
+        answer.delete()
+    counts["cards created" if new else "cards updated"] += 1
+
+
+def _image_changed(card, content, name):
+    """Whether an image in a bundle differs from the card's, as it would be stored
+    (reduced and stripped of metadata), so that importing the same image again
+    leaves the card's file alone."""
+    prepared = prepare_image(io.BytesIO(content), name).read()
+    try:
+        with card.image.open("rb") as stored:
+            return stored.read() != prepared
+    except FileNotFoundError:
+        return True
+
+
+def _distinct(forms, exclude):
+    seen = {normalise(exclude)}
+    out = []
+    for form in forms:
+        key = normalise(form)
+        if key not in seen:
+            seen.add(key)
+            out.append(form)
+    return out
+
+
+def _answers(data, where):
+    raw = data.get("answers")
+    if raw is None and "answer" in data:
+        raw = [data["answer"]]
+    if not isinstance(raw, list) or not raw:
+        raise BundleError(f"{where}: a card needs at least one answer.")
+    answers = []
+    for item in raw:
+        if not isinstance(item, dict):
+            item = {"text": item}
+        alternatives = item.get("alternatives") or []
+        if not isinstance(alternatives, list):
+            alternatives = [alternatives]
+        answers.append(
+            {
+                "label": _text(item.get("label"), f"{where}: label"),
+                "text": _text(item.get("text"), f"{where}: answer", required=True),
+                "alternatives": [
+                    t
+                    for t in (_text(a, f"{where}: alternative") for a in alternatives)
+                    if t
+                ],
+            }
+        )
+    return answers
+
+
+def _mapping(value, where):
+    if not isinstance(value, dict):
+        raise BundleError(f"{where}: expected a mapping of names to values.")
+    return value
+
+
+def _slug(value, where, *, page_name=False):
+    """A slug or key; a subject's or deck's slug (`page_name`), which is part of
+    its page's address, must not be the name of one of the app's own pages."""
+    value = _text(value, f"{where}: slug or key", required=True)
+    try:
+        validate_slug(value)
+    except ValidationError as error:
+        raise BundleError(
+            f"{where}: {value!r} must be letters, numbers, hyphens and underscores."
+        ) from error
+    if page_name and value in RESERVED_SLUGS:
+        raise BundleError(f"{where}: {value!r} is the name of one of the app's pages.")
+    return value
+
+
+def _text(value, where, *, required=False):
+    """A YAML value as text. YAML reads an unquoted yes, no, on or off as true or
+    false, which is never meant here (on is a French pronoun)."""
+    if isinstance(value, bool):
+        raise BundleError(f"{where}: put {str(value).lower()!r} in quotation marks.")
+    text = "" if value is None else str(value).strip()
+    if required and not text:
+        raise BundleError(f"{where}: missing.")
+    return text
+
+
+def export_bundle(subject, path):
+    """Writes a subject's decks and cards, with their images, as a bundle: a zip
+    file if `path` ends in .zip, otherwise a directory. Returns the number of
+    cards."""
+    path = Path(path)
+    files = {}
+    decks = []
+    count = 0
+    for deck in subject.decks.all():
+        cards = []
+        for card in deck.cards.prefetch_related("answers"):
+            entry = {"key": card.key}
+            if card.prompt:
+                entry["prompt"] = card.prompt
+            if card.image:
+                name = f"images/{deck.slug}/{card.key}{PurePosixPath(card.image.name).suffix}"
+                with card.image.open("rb") as image:
+                    files[name] = image.read()
+                entry["image"] = name
+                for field in ("image_alt", "image_credit", "image_licence"):
+                    if getattr(card, field):
+                        entry[field] = getattr(card, field)
+            entry["answers"] = [_export_answer(a) for a in card.answers.all()]
+            if card.notes:
+                entry["notes"] = card.notes
+            cards.append(entry)
+            count += 1
+        deck_entry = {"slug": deck.slug, "title": deck.title}
+        if deck.description:
+            deck_entry["description"] = deck.description
+        if deck.reference:
+            deck_entry["reference"] = deck.reference
+        deck_entry["public"] = deck.is_public
+        deck_entry["cards"] = cards
+        decks.append(deck_entry)
+    subject_entry = {"slug": subject.slug, "title": subject.title}
+    for field in ("description", "language", "characters", "reference"):
+        if getattr(subject, field):
+            subject_entry[field] = getattr(subject, field)
+    if subject.case_sensitive:
+        subject_entry["case_sensitive"] = True
+    manifest = dump_manifest(
+        {"format": FORMAT, "subject": subject_entry, "decks": decks}
+    )
+    if path.suffix == ".zip":
+        with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as bundle:
+            bundle.writestr(MANIFEST, manifest)
+            for name, content in files.items():
+                bundle.writestr(name, content)
+    else:
+        path.mkdir(parents=True, exist_ok=True)
+        (path / MANIFEST).write_text(manifest, encoding="utf-8")
+        for name, content in files.items():
+            target = path.joinpath(*PurePosixPath(name).parts)
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_bytes(content)
+    return count
+
+
+def _export_answer(answer):
+    alternatives = answer.accepted()[1:]
+    if not answer.label and not alternatives:
+        return answer.text
+    entry = {}
+    if answer.label:
+        entry["label"] = answer.label
+    entry["text"] = answer.text
+    if alternatives:
+        entry["alternatives"] = alternatives
+    return entry

flashdeck/conf.py

@@ -0,0 +1,19 @@
+"""The app's settings. A site overrides one by setting FLASHDECK_<NAME>."""
+
+from django.conf import settings
+
+DEFAULTS = {
+    # New cards introduced to each user per day, across all subjects.
+    "NEW_CARDS_PER_DAY": 20,
+    # Card images are reduced to fit a square of this size (in pixels) on upload.
+    "IMAGE_MAX_SIZE": 1200,
+    # The directory, within the site's media storage, that holds card images.
+    "UPLOAD_DIR": "flashdeck",
+    # The URL (or URL name) to log out at, by POST, for a link on the index; blank
+    # for none. Logging in uses Django's LOGIN_URL.
+    "LOGOUT_URL": "",
+}
+
+
+def setting(name):
+    return getattr(settings, f"FLASHDECK_{name}", DEFAULTS[name])

flashdeck/images.py

@@ -0,0 +1,85 @@
+"""Card images: checked on upload, reduced in size and stripped of metadata.
+
+Photographs often carry a caption or keywords (in EXIF, XMP or a JPEG comment)
+that name what they show, which would give the answer away to anyone who looks
+at the file, so only the pixels (and the colour profile) are kept.
+"""
+
+import io
+from pathlib import PurePosixPath
+
+from django.core.exceptions import ValidationError
+from django.core.files.base import ContentFile
+from PIL import Image, ImageOps, UnidentifiedImageError
+
+from .conf import setting
+
+IMAGE_EXTENSIONS = ["png", "jpg", "jpeg", "webp", "svg"]
+_FORMATS = {".png": "PNG", ".jpg": "JPEG", ".jpeg": "JPEG", ".webp": "WEBP"}
+
+
+def validate_image(value):
+    """Checks that a newly uploaded image is of the type its extension says.
+
+    An SVG file is only checked for an <svg> element: as it can contain scripts,
+    only trusted editors should be able to upload one.
+    """
+    if getattr(value, "_committed", True):
+        return  # already stored, so checked when it was uploaded
+    suffix = PurePosixPath(value.name).suffix.lower()
+    value.seek(0)
+    try:
+        if suffix == ".svg":
+            head = value.read(4096)
+            if isinstance(head, bytes):
+                head = head.decode("utf-8", "replace")
+            if "<svg" not in head:
+                raise ValidationError("This file is not an SVG image.")
+            return
+        try:
+            with Image.open(value) as image:
+                found = image.format
+                image.verify()
+        except (UnidentifiedImageError, OSError, SyntaxError) as error:
+            raise ValidationError(
+                "This file is not an image that can be read."
+            ) from error
+        if found != _FORMATS.get(suffix):
+            raise ValidationError(
+                f"This is a {found} image, but its name ends in {suffix}."
+            )
+    finally:
+        value.seek(0)
+
+
+def prepare_image(file, name: str) -> ContentFile:
+    """The image to store: reduced to fit IMAGE_MAX_SIZE, turned the right way up,
+    and without metadata. SVG images are stored as they are."""
+    suffix = PurePosixPath(name).suffix.lower()
+    file.seek(0)
+    data = file.read()
+    if suffix == ".svg":
+        return ContentFile(data, name="image.svg")
+    with Image.open(io.BytesIO(data)) as original:
+        fmt = original.format
+        icc_profile = original.info.get("icc_profile")
+        transparency = original.info.get("transparency")
+        image = ImageOps.exif_transpose(original)
+    image.info = {} if transparency is None else {"transparency": transparency}
+    size = setting("IMAGE_MAX_SIZE")
+    image.thumbnail((size, size))
+    options = {}
+    if fmt == "JPEG":
+        if image.mode not in ("RGB", "L"):
+            image = image.convert("RGB")
+            icc_profile = None  # it described the old colour space
+        options.update(quality=85, optimize=True)
+    elif fmt == "WEBP":
+        options.update(quality=85)
+    elif fmt == "PNG":
+        options.update(optimize=True)
+    if icc_profile:
+        options["icc_profile"] = icc_profile
+    out = io.BytesIO()
+    image.save(out, fmt, **options)
+    return ContentFile(out.getvalue(), name=f"image{suffix}")

flashdeck/management/commands/flashdeck_export.py

@@ -0,0 +1,23 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from flashdeck.bundles import export_bundle
+from flashdeck.models import Subject
+
+
+class Command(BaseCommand):
+    help = (
+        "Exports a subject's decks and cards, with their images, as a bundle: a zip "
+        "file if the name ends in .zip, otherwise a directory."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument("subject", help="The subject's slug.")
+        parser.add_argument("bundle", help="The directory or zip file to write.")
+
+    def handle(self, subject, bundle, **options):
+        try:
+            subject = Subject.objects.get(slug=subject)
+        except Subject.DoesNotExist as error:
+            raise CommandError(f"There is no subject {subject!r}.") from error
+        count = export_bundle(subject, bundle)
+        self.stdout.write(f"{count} cards written to {bundle}.")

flashdeck/management/commands/flashdeck_import.py

@@ -0,0 +1,29 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from flashdeck.bundles import BundleError, import_bundle
+
+
+class Command(BaseCommand):
+    help = (
+        "Imports a bundle of flashcards: a directory or zip file holding cards.yaml "
+        "and its images. Cards not on the site are added; existing ones are left "
+        "alone unless --update is given."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument("bundle", help="The bundle's directory or zip file.")
+        parser.add_argument(
+            "--update",
+            action="store_true",
+            help="Also update existing subjects, decks and cards from the bundle. "
+            "Answers accepted on the site are kept, an image is replaced only if it "
+            "has changed, and whether a deck is public is left as it is.",
+        )
+
+    def handle(self, bundle, update, **options):
+        try:
+            counts = import_bundle(bundle, update=update)
+        except BundleError as error:
+            raise CommandError(str(error)) from error
+        summary = ", ".join(f"{n} {what}" for what, n in counts.items())
+        self.stdout.write(summary or "Nothing to import.")

flashdeck/manifest.py

@@ -0,0 +1,33 @@
+"""The bundle format's manifest, cards.yaml. This module needs no Django set-up, so
+that scripts which generate bundles can use it."""
+
+import yaml
+
+FORMAT = "flashdeck/1"
+MANIFEST = "cards.yaml"
+
+
+class _Dumper(yaml.SafeDumper):
+    """Writes text of more than one line as a literal block (|-)."""
+
+
+def _represent_text(dumper, text):
+    style = "|" if "\n" in text else None
+    return dumper.represent_scalar("tag:yaml.org,2002:str", text, style=style)
+
+
+_Dumper.add_representer(str, _represent_text)
+
+
+def dump_manifest(data):
+    """The text of cards.yaml for a bundle's data, written to be easy to read and
+    edit: short lists and mappings on one line, and text of more than one line as
+    a literal block."""
+    return yaml.dump(
+        data,
+        Dumper=_Dumper,
+        allow_unicode=True,
+        sort_keys=False,
+        default_flow_style=None,
+        width=88,
+    )

flashdeck/marking.py

@@ -0,0 +1,107 @@
+"""Marking typed answers as right, nearly right or wrong.
+
+An answer is *right* if it matches an accepted answer once both are normalised (see
+`normalise`), and *nearly right* if it differs from one only in accents, capitals
+or punctuation (see `loosen`). Nothing else is forgiven.
+"""
+
+import re
+import unicodedata
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass
+
+from django.db import models
+
+
+class Verdict(models.TextChoices):
+    RIGHT = "right", "Right"
+    NEARLY = "nearly", "Nearly right"
+    WRONG = "wrong", "Wrong"
+    # Marked nearly right or wrong, then claimed as right by the user.
+    CLAIMED = "claimed", "Claimed as right"
+
+
+# Typographic variants that one keyboard produces and another does not.
+_EQUIVALENTS = str.maketrans(
+    {
+        "’": "'",  # right single quotation mark, the French apostrophe
+        "‘": "'",  # left single quotation mark
+        "ʼ": "'",  # modifier letter apostrophe
+        "“": '"',  # left double quotation mark
+        "”": '"',  # right double quotation mark
+        " ": " ",  # no-break space
+        " ": " ",  # narrow no-break space, before ; : ! ? in French
+        "‑": "-",  # non-breaking hyphen
+    }
+)
+_SPACE_BEFORE_PUNCTUATION = re.compile(r"\s+([;:!?])")
+_LETTERS_SPELT_OUT = str.maketrans(
+    {"ß": "ss", "ẞ": "ss", "æ": "ae", "Æ": "ae", "œ": "oe", "Œ": "oe"}
+)
+
+
+def normalise(text: str, *, case_sensitive: bool = False) -> str:
+    """The form of an answer that is compared for a right mark.
+
+    Differences in Unicode composition, typographic apostrophes and quotation
+    marks, runs of white space, a space before ; : ! or ? (which French typography
+    puts there), and a final full stop, exclamation mark or question mark are all
+    ignored, as are capitals unless `case_sensitive` is set.
+    """
+    text = unicodedata.normalize("NFC", text).translate(_EQUIVALENTS)
+    text = " ".join(text.split())
+    text = _SPACE_BEFORE_PUNCTUATION.sub(r"\1", text)
+    text = text.rstrip(".!?").rstrip()
+    # lower(), not casefold(), which would turn ß into ss.
+    return text if case_sensitive else text.lower()
+
+
+def loosen(text: str) -> str:
+    """The form of an answer that is compared for a nearly-right mark.
+
+    As `normalise`, but also without accents, capitals or punctuation (each mark
+    of punctuation counts as a space), and with ß, æ and œ spelt out.
+    """
+    text = normalise(text).translate(_LETTERS_SPELT_OUT)
+    text = "".join(
+        c for c in unicodedata.normalize("NFD", text) if not unicodedata.combining(c)
+    )
+    text = "".join(c if c.isalnum() or c.isspace() else " " for c in text)
+    return " ".join(text.split())
+
+
+@dataclass(frozen=True)
+class Mark:
+    verdict: Verdict
+    # The accepted answer that was matched or nearly matched, or else the first
+    # (the model answer).
+    expected: str
+
+
+def mark(given: str, accepted: Sequence[str], *, case_sensitive: bool = False) -> Mark:
+    """Marks one answer against the answers accepted for it."""
+    if not accepted:
+        raise ValueError("No accepted answers")
+    strict = normalise(given, case_sensitive=case_sensitive)
+    if not strict:
+        return Mark(Verdict.WRONG, accepted[0])
+    for form in accepted:
+        if normalise(form, case_sensitive=case_sensitive) == strict:
+            return Mark(Verdict.RIGHT, form)
+    loose = loosen(given)
+    if loose:
+        for form in accepted:
+            if loosen(form) == loose:
+                return Mark(Verdict.NEARLY, form)
+    return Mark(Verdict.WRONG, accepted[0])
+
+
+def combine(verdicts: Iterable[str]) -> Verdict:
+    """The verdict on a card from the verdicts on its answers: wrong if any is
+    wrong, otherwise nearly right if any is nearly right, otherwise right."""
+    verdicts = set(verdicts)
+    if Verdict.WRONG in verdicts:
+        return Verdict.WRONG
+    if Verdict.NEARLY in verdicts:
+        return Verdict.NEARLY
+    return Verdict.RIGHT