sslabdata 3.0.0__py3-none-any.whl

sslabdata/loaders.py

@@ -0,0 +1,298 @@
+"""
+YAML data loaders for people and projects.
+
+Copyright (c) 2024 Personal Robotics Laboratory, University of Washington
+Author: Siddhartha Srinivasa
+MIT License - see LICENSE file for details.
+"""
+
+import yaml
+from dataclasses import dataclass, field
+from typing import Dict, List
+from pathlib import Path
+
+from .config import (
+    CONTROL_CHARACTER, RepeatedKey, _kind, control_message, dotted,
+    read_yaml, repeated_message,
+)
+from .diagnostics import Diagnostic, diagnostic
+from .models import Person, Project
+
+
+# One code per condition and file; the RECORD-* codes cover all three files
+# (SPEC.md "Diagnostic codes").
+PEOPLE_YAML_INVALID = "PEOPLE-YAML-INVALID"
+PEOPLE_NOT_A_LIST = "PEOPLE-NOT-A-LIST"
+PEOPLE_FIELD_MISSING = "PEOPLE-FIELD-MISSING"
+PROJECTS_YAML_INVALID = "PROJECTS-YAML-INVALID"
+PROJECTS_NOT_A_LIST = "PROJECTS-NOT-A-LIST"
+PROJECTS_FIELD_MISSING = "PROJECTS-FIELD-MISSING"
+COLLABORATORS_YAML_INVALID = "COLLABORATORS-YAML-INVALID"
+COLLABORATORS_NOT_A_LIST = "COLLABORATORS-NOT-A-LIST"
+COLLABORATORS_FIELD_MISSING = "COLLABORATORS-FIELD-MISSING"
+PEOPLE_ID_DUPLICATE = "PEOPLE-ID-DUPLICATE"
+PEOPLE_ROLE_INVALID = "PEOPLE-ROLE-INVALID"
+PEOPLE_STATUS_INVALID = "PEOPLE-STATUS-INVALID"
+PROJECTS_ID_DUPLICATE = "PROJECTS-ID-DUPLICATE"
+PROJECTS_STATUS_INVALID = "PROJECTS-STATUS-INVALID"
+RECORD_KEY_UNKNOWN = "RECORD-KEY-UNKNOWN"
+RECORD_KEY_REPEATED = "RECORD-KEY-REPEATED"
+RECORD_TYPE_INVALID = "RECORD-TYPE-INVALID"
+
+# The keys each file's records are read for. Any other key is reported and
+# ignored, so a misspelt `webiste` is not silently dropped from the document.
+PERSON_KEYS = ("id", "name", "aliases", "role", "status", "photo", "website",
+               "email", "co_advisor", "start_year", "end_year", "degree",
+               "thesis_title", "current_position")
+PROJECT_KEYS = ("id", "title", "description", "website", "image", "status")
+COLLABORATOR_KEYS = ("name", "aliases")
+
+# The type each optional field accepts; `role` and `status` have codes of
+# their own. A value of another type is read as empty, so it never reaches the
+# document as the wrong type.
+STRING, INTEGER, ALIASES = ("a string", "an integer",
+                            "a list of non-empty strings")
+PERSON_TYPES = {**dict.fromkeys(("photo", "website", "email", "co_advisor",
+                                 "degree", "thesis_title",
+                                 "current_position"), STRING),
+                "start_year": INTEGER, "end_year": INTEGER,
+                "aliases": ALIASES}
+PROJECT_TYPES = dict.fromkeys(("description", "website", "image"), STRING)
+COLLABORATOR_TYPES = {"aliases": ALIASES}
+
+# There is no list of roles: any non-empty string is one, so that any lab's
+# roles fit (SPEC.md §5).
+PERSON_STATUSES = ("current", "alumni")
+PROJECT_STATUSES = ("active", "completed")
+
+
+def _has_type(value, expected: str) -> bool:
+    if expected == STRING:
+        return isinstance(value, str)
+    if expected == INTEGER:
+        return isinstance(value, int) and not isinstance(value, bool)
+    return isinstance(value, list) and all(
+        isinstance(v, str) and v.strip() for v in value)
+
+
+def _records(path: str, codes, required, known, optional,
+             diagnostics) -> List[dict]:
+    """The records of one people, projects or collaborators file that can be
+    emitted, every file checked the same way.
+
+    ``codes`` are the file's YAML-invalid, not-a-list and field-missing
+    codes, ``required`` the fields a record cannot be emitted without --
+    each a non-empty string, the first naming the record -- ``known`` the
+    keys the file's records are read for and ``optional`` the type each
+    optional field accepts. A missing file reads as no records: the assembler
+    reports it, naming the configuration key.
+    """
+    yaml_invalid, not_a_list, field_missing = codes
+    fail = diagnostics.append
+    if not Path(path).exists():
+        return []
+
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            data, repeated, controls = read_yaml(f)
+    except (yaml.YAMLError, UnicodeDecodeError) as error:
+        fail(diagnostic(yaml_invalid, path, None, None,
+                        " ".join(str(error).split())))
+        return []
+
+    # Each located as a repeat is: at the record, named by its first required
+    # field, and the path inside it; or, outside any record, at the path.
+    for found in controls:
+        index = found.path[0] if found.path else None
+        record = (data[index] if isinstance(data, list)
+                  and isinstance(index, int) else None)
+        key = record.get(required[0]) if isinstance(record, dict) else None
+        fail(diagnostic(CONTROL_CHARACTER, path,
+                        key if isinstance(key, str) else None,
+                        dotted(found.path[1:] if isinstance(record, dict)
+                               else found.path) or None,
+                        control_message(found.found)))
+
+    # Each repeat inside a record is reported with that record, below; one
+    # anywhere else is reported here, at the file and the path to the key.
+    in_record: Dict[int, List[RepeatedKey]] = {}
+    for repeat in repeated:
+        index = repeat.path[0] if repeat.path else None
+        if (isinstance(data, list) and isinstance(index, int)
+                and isinstance(data[index], dict)):
+            in_record.setdefault(index, []).append(repeat)
+        else:
+            fail(diagnostic(RECORD_KEY_REPEATED, path, None,
+                            dotted((*repeat.path, repeat.key)),
+                            repeated_message(repeat)))
+
+    if data is None:
+        return []
+    if not isinstance(data, list):
+        fail(diagnostic(not_a_list, path, None, None,
+                        "the file must be a list of records, one per entry; "
+                        f"it is a {type(data).__name__}"))
+        return []
+
+    records = []
+    for number, entry in enumerate(data, start=1):
+        if not isinstance(entry, dict):
+            fail(diagnostic(not_a_list, path, None, None,
+                            f"entry {number} is a {type(entry).__name__}, "
+                            "not a record"))
+            continue
+        if number - 1 in in_record:
+            # The record is named by its first required field, unless that
+            # is the key given twice.
+            repeats = in_record[number - 1]
+            key = entry.get(required[0])
+            if (not isinstance(key, str) or
+                    any(r.path == (number - 1,) and r.key == required[0]
+                        for r in repeats)):
+                key = None
+            for repeat in repeats:
+                fail(diagnostic(RECORD_KEY_REPEATED, path, key,
+                                dotted((*repeat.path[1:], repeat.key)),
+                                repeated_message(repeat)))
+            continue
+        missing = [name for name in required
+                   if not isinstance(entry.get(name), str)
+                   or not entry[name].strip()]
+        if missing:
+            name, value = missing[0], entry.get(missing[0])
+            key = entry.get('id')
+            fail(diagnostic(
+                field_missing, path, key if isinstance(key, str) else None,
+                name, f"entry {number} has no {name}"
+                if value is None or isinstance(value, str) else
+                f"entry {number}'s {name} is {_kind(value)}; it must be a "
+                "string"))
+            continue
+        # A YAML key need not be a string (`0:`, `true:`); it is named as
+        # text so that every unknown key has a location, even a falsy one.
+        for key in entry:
+            if key not in known:
+                diagnostics.append(diagnostic(
+                    RECORD_KEY_UNKNOWN, path, entry[required[0]], str(key),
+                    f"'{key}' is not a key sslabdata reads, and is ignored"))
+        for name, expected in optional.items():
+            if entry.get(name) is not None and not _has_type(entry[name],
+                                                             expected):
+                fail(diagnostic(
+                    RECORD_TYPE_INVALID, path, entry[required[0]], name,
+                    f"{name} is {_kind(entry[name])}; it must be {expected}, "
+                    "and is read as empty"))
+                entry[name] = None
+        records.append(entry)
+    return records
+
+
+def _repeated_ids(records: List[dict], path: str, code: str, report) -> None:
+    """Report each id declared again, at the record that repeats it; both
+    are kept, as the parser library keeps a repeated citation key."""
+    seen = set()
+    for entry in records:
+        key = entry['id']
+        if key in seen:
+            report(diagnostic(code, path, key, 'id',
+                              f"the id '{key}' is declared more than once"))
+        seen.add(key)
+
+
+def load_people(path: str, diagnostics: List[Diagnostic]) -> List[Person]:
+    """Load people from a YAML file (format: README.md); ``diagnostics``
+    receives what is wrong with it."""
+    warn = diagnostics.append
+    people = []
+    records = _records(path, (PEOPLE_YAML_INVALID, PEOPLE_NOT_A_LIST,
+                              PEOPLE_FIELD_MISSING), ('id', 'name'), PERSON_KEYS,
+                       PERSON_TYPES, diagnostics)
+    _repeated_ids(records, path, PEOPLE_ID_DUPLICATE, diagnostics.append)
+    for entry in records:
+        role = entry.get('role')
+        if not isinstance(role, str) or not role.strip():
+            warn(diagnostic(PEOPLE_ROLE_INVALID, path, entry['id'], 'role',
+                            "role is missing, empty or not a string; any "
+                            "non-empty string is accepted"))
+        status = entry.get('status', 'current')
+        if status not in PERSON_STATUSES:
+            warn(diagnostic(PEOPLE_STATUS_INVALID, path, entry['id'], 'status',
+                            f"'{status}' is not one of "
+                            f"{', '.join(PERSON_STATUSES)}"))
+            # A status that is not a string reads as the absent default: the
+            # schema requires a string, and the warning has said why.
+            if not isinstance(status, str):
+                status = 'current'
+        person = Person(
+            id=entry['id'],
+            name=entry['name'],
+            aliases=entry.get('aliases') or [],
+            role=role if isinstance(role, str) else None,
+            status=status,
+            photo=entry.get('photo'),
+            website=entry.get('website'),
+            email=entry.get('email'),
+            co_advisor=entry.get('co_advisor'),
+            start_year=entry.get('start_year'),
+            end_year=entry.get('end_year'),
+            degree=entry.get('degree'),
+            thesis_title=entry.get('thesis_title'),
+            current_position=entry.get('current_position'),
+        )
+        people.append(person)
+
+    return people
+
+
+def load_projects(path: str, diagnostics: List[Diagnostic]) -> List[Project]:
+    """Load projects from a YAML file, checked as `load_people()` checks
+    its own."""
+    warn = diagnostics.append
+    records = _records(path, (PROJECTS_YAML_INVALID, PROJECTS_NOT_A_LIST,
+                              PROJECTS_FIELD_MISSING), ('id', 'title'), PROJECT_KEYS,
+                       PROJECT_TYPES, diagnostics)
+    _repeated_ids(records, path, PROJECTS_ID_DUPLICATE, diagnostics.append)
+    projects = []
+    for entry in records:
+        status = entry.get('status', 'active')
+        if status not in PROJECT_STATUSES:
+            warn(diagnostic(PROJECTS_STATUS_INVALID, path, entry['id'],
+                            'status', f"'{status}' is not one of "
+                            f"{', '.join(PROJECT_STATUSES)}"))
+            if not isinstance(status, str):
+                status = 'active'
+        project = Project(
+            id=entry['id'],
+            title=entry['title'],
+            description=entry.get('description'),
+            website=entry.get('website'),
+            image=entry.get('image'),
+            status=status,
+        )
+        projects.append(project)
+
+    return projects
+
+
+@dataclass
+class DeclaredCollaborator:
+    """One external co-author declared in `collaborators_file`.
+
+    It only groups authorships into `collaborators`; it is never a person
+    and never produces a `person_id`.
+    """
+    name: str
+    aliases: List[str] = field(default_factory=list)
+
+
+def load_collaborators(path: str, diagnostics: List[Diagnostic]
+                       ) -> List[DeclaredCollaborator]:
+    """Load declared external co-authors from a YAML file, checked as
+    `load_people()` checks its own."""
+    records = _records(path, (COLLABORATORS_YAML_INVALID,
+                              COLLABORATORS_NOT_A_LIST,
+                              COLLABORATORS_FIELD_MISSING), ('name',),
+                       COLLABORATOR_KEYS, COLLABORATOR_TYPES, diagnostics)
+    return [DeclaredCollaborator(name=entry['name'],
+                                 aliases=entry.get('aliases') or [])
+            for entry in records]

sslabdata/models.py

@@ -0,0 +1,342 @@
+"""
+Data models for sslabdata.
+
+Defines the core entity types: Work, Author, Person, Project, Collaborator,
+and the assembled LabData output.
+
+Copyright (c) 2024 Personal Robotics Laboratory, University of Washington
+Author: Siddhartha Srinivasa
+MIT License - see LICENSE file for details.
+"""
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+# `config` owns these checks and their codes. `to_dict()` repeats them
+# because every document passes through it, including one built in Python
+# (SPEC.md §1).
+from .config import json_lab, reject_absolute_name
+
+
+# The document's schema version (schema/v5/output.schema.json). When it
+# changes is SPEC.md §6.
+SCHEMA_VERSION = 5
+
+GENERATOR_NAME = "sslabdata"
+
+# Every to_dict() emits every declared key, `null` when it does not apply;
+# the open maps carry only keys with values (SPEC.md §4).
+
+
+@dataclass
+class Venue:
+    """Where a work appeared (SPEC.md §5)."""
+    kind: str
+    name: str
+
+    def to_dict(self) -> dict:
+        return {'kind': self.kind, 'name': self.name}
+
+
+@dataclass
+class Link:
+    """One URL a work can be reached at, with its origin and verification
+    status (SPEC.md §5)."""
+    url: str
+    label: Optional[str] = None
+    origin: str = "input"
+    status: str = "unchecked"
+
+    def to_dict(self) -> dict:
+        return {
+            'url': self.url,
+            'label': self.label,
+            'origin': self.origin,
+            'verification': {'status': self.status},
+        }
+
+
+@dataclass
+class Contributor:
+    """One person named on a work: the parts of the name, and who it resolved
+    to (SPEC.md §5). The record ``work.editors`` carries.
+
+    ``literal`` holds a name written as one brace-protected unit, where the
+    other parts do not apply.
+    """
+    name: str
+    position: int = 0
+    person_id: Optional[str] = None
+    given: Optional[str] = None
+    von: Optional[str] = None
+    family: Optional[str] = None
+    suffix: Optional[str] = None
+    literal: Optional[str] = None
+    resolution_status: str = "unresolved"
+    resolution_method: Optional[str] = None
+    derived: Dict[str, object] = field(default_factory=dict)
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary for serialization."""
+        return {
+            'name': self.name,
+            'position': self.position,
+            'person_id': self.person_id,
+            'given': self.given,
+            'von': self.von,
+            'family': self.family,
+            'suffix': self.suffix,
+            'literal': self.literal,
+            'resolution': {'status': self.resolution_status,
+                           'method': self.resolution_method},
+            'derived': dict(self.derived),
+        }
+
+
+@dataclass
+class Author(Contributor):
+    """One authorship of a work, addressed by ``(work.bib_id, position)``
+    (SPEC.md §5).
+
+    Exactly one of ``person_id`` and ``collaborator_key`` is non-null.
+    """
+    collaborator_key: Optional[str] = None
+    equal_contribution: bool = False
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary for serialization."""
+        d = Contributor.to_dict(self)
+        d['collaborator_key'] = self.collaborator_key
+        d['equal_contribution'] = self.equal_contribution
+        return d
+
+
+@dataclass
+class Work:
+    """A single work with structured, renderer-agnostic data."""
+    bib_id: str
+    title: str
+    authors: List[Author]
+    year: Optional[int]
+    category: str
+    entry_type: str
+
+    # The configured `bib_files[].name`, relative to `bib_dir`: see `to_dict()`.
+    source_file: str = ""
+
+    editors: List[Contributor] = field(default_factory=list)
+    venue: Optional[Venue] = None
+
+    # The bibliographic parts, flat on the work rather than nested in the
+    # venue: they describe the work's placement, not the container.
+    volume: Optional[str] = None
+    number: Optional[str] = None
+    pages: Optional[str] = None
+    series: Optional[str] = None
+    edition: Optional[str] = None
+    publisher: Optional[str] = None
+    address: Optional[str] = None
+    organization: Optional[str] = None
+    chapter: Optional[str] = None
+    month: Optional[str] = None
+    howpublished: Optional[str] = None
+    type: Optional[str] = None
+
+    abstract: Optional[str] = None
+    note: Optional[str] = None
+
+    identifiers: Dict[str, List[str]] = field(default_factory=dict)
+    links: Dict[str, List[Link]] = field(default_factory=dict)
+
+    project_ids: List[str] = field(default_factory=list)
+    bibtex: Optional[str] = None
+    derived: Dict[str, object] = field(default_factory=dict)
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary for serialization.
+
+        ``source.file`` is checked here rather than only where it was set:
+        every serializer passes through this method, so a `Work` built by hand
+        cannot carry an absolute path into the document.
+        """
+        reject_absolute_name(self.source_file)
+        return {
+            'bib_id': self.bib_id,
+            'source': {'file': self.source_file, 'key': self.bib_id},
+            'title': self.title,
+            'authors': [a.to_dict() for a in self.authors],
+            'editors': [e.to_dict() for e in self.editors],
+            'year': self.year,
+            'venue': self.venue.to_dict() if self.venue else None,
+            'volume': self.volume,
+            'number': self.number,
+            'pages': self.pages,
+            'series': self.series,
+            'edition': self.edition,
+            'publisher': self.publisher,
+            'address': self.address,
+            'organization': self.organization,
+            'chapter': self.chapter,
+            'month': self.month,
+            'howpublished': self.howpublished,
+            'type': self.type,
+            'category': self.category,
+            'entry_type': self.entry_type,
+            'abstract': self.abstract,
+            'note': self.note,
+            'identifiers': {scheme: list(values)
+                            for scheme, values in self.identifiers.items()},
+            'links': {kind: [link.to_dict() for link in links]
+                      for kind, links in self.links.items()},
+            'project_ids': list(self.project_ids),
+            'bibtex': self.bibtex,
+            'derived': dict(self.derived),
+        }
+
+
+@dataclass
+class Person:
+    """A lab member (current or alumni)."""
+    id: str
+    name: str
+    aliases: List[str] = field(default_factory=list)
+    role: Optional[str] = None
+    status: str = "current"
+    photo: Optional[str] = None
+    website: Optional[str] = None
+    email: Optional[str] = None
+    co_advisor: Optional[str] = None
+    start_year: Optional[int] = None
+
+    end_year: Optional[int] = None
+    degree: Optional[str] = None
+    thesis_title: Optional[str] = None
+    current_position: Optional[str] = None
+
+    # Back-linked (computed, not from YAML input)
+    work_ids: List[str] = field(default_factory=list)
+    derived: Dict[str, object] = field(default_factory=dict)
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary for serialization; ``aliases`` are read for
+        matching and are not emitted."""
+        return {
+            'id': self.id,
+            'name': self.name,
+            'role': self.role,
+            'status': self.status,
+            'photo': self.photo,
+            'email': self.email,
+            'website': self.website,
+            'co_advisor': self.co_advisor,
+            'start_year': self.start_year,
+            'end_year': self.end_year,
+            'degree': self.degree,
+            'thesis_title': self.thesis_title,
+            'current_position': self.current_position,
+            'work_ids': list(self.work_ids),
+            'derived': dict(self.derived),
+        }
+
+
+@dataclass
+class Collaborator:
+    """A grouping over unresolved authorships, not an identity (SPEC.md §5)."""
+    key: str
+    name: str
+    grouped_by: str = "normalized_name"
+    name_kind: str = "personal"
+    given: Optional[str] = None
+    von: Optional[str] = None
+    family: Optional[str] = None
+    suffix: Optional[str] = None
+    literal: Optional[str] = None
+    name_variants: List[str] = field(default_factory=list)
+    authorships: List[Dict[str, object]] = field(default_factory=list)
+    work_ids: List[str] = field(default_factory=list)
+    last_year: Optional[int] = None
+    derived: Dict[str, object] = field(default_factory=dict)
+
+    def to_dict(self) -> dict:
+        return {
+            'key': self.key,
+            'grouped_by': self.grouped_by,
+            'name_kind': self.name_kind,
+            'name': self.name,
+            'given': self.given,
+            'von': self.von,
+            'family': self.family,
+            'suffix': self.suffix,
+            'literal': self.literal,
+            'name_variants': list(self.name_variants),
+            'authorships': [dict(a) for a in self.authorships],
+            'work_ids': list(self.work_ids),
+            'last_year': self.last_year,
+            'derived': dict(self.derived),
+        }
+
+
+@dataclass
+class Project:
+    """A research project."""
+    id: str
+    title: str
+    description: Optional[str] = None
+    website: Optional[str] = None
+    status: str = "active"
+
+    # Back-linked (computed)
+    work_ids: List[str] = field(default_factory=list)
+    people_ids: List[str] = field(default_factory=list)
+    derived: Dict[str, object] = field(default_factory=dict)
+
+    # Declared last so it takes no other field's position in a positional
+    # call; to_dict() emits it beside `website`.
+    image: Optional[str] = None
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary for serialization."""
+        return {
+            'id': self.id,
+            'title': self.title,
+            'description': self.description,
+            'website': self.website,
+            'image': self.image,
+            'status': self.status,
+            'work_ids': list(self.work_ids),
+            'people_ids': list(self.people_ids),
+            'derived': dict(self.derived),
+        }
+
+
+@dataclass
+class LabData:
+    """The fully resolved output: all entities with cross-references."""
+    works: List[Work] = field(default_factory=list)
+    people: List[Person] = field(default_factory=list)
+    projects: List[Project] = field(default_factory=list)
+    collaborators: List[Collaborator] = field(default_factory=list)
+    lab: Optional[Dict[str, object]] = None
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary for serialization.
+
+        ``generator`` carries no timestamp, so the document is deterministic
+        (SPEC.md §3). The version is imported here because the package imports
+        this module while defining it.
+        """
+        from . import __version__
+
+        return {
+            'schema_version': SCHEMA_VERSION,
+            'generator': {
+                'name': GENERATOR_NAME,
+                'version': __version__,
+                'schema_version': SCHEMA_VERSION,
+            },
+            'lab': json_lab(dict(self.lab or {})),
+            'works': [w.to_dict() for w in self.works],
+            'people': [p.to_dict() for p in self.people],
+            'projects': [p.to_dict() for p in self.projects],
+            'collaborators': [c.to_dict() for c in self.collaborators],
+        }