htruc 1.1.4__py3-none-any.whl

htruc/catalog.py

@@ -0,0 +1,301 @@
+from typing import Optional, Dict, Any, List, Tuple, Iterable, Union
+import os
+import logging
+import pandas
+import cffconvert
+import requests
+import re
+
+from htruc.repos import get_github_repo_yaml, get_htr_united_repos, get_github_repo_cff
+from htruc.utils import parse_yaml
+from htruc import validator
+from htruc.schemas import recursive_update
+from htruc.types import CatalogRecord, Catalog
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+_ZenodoRecord: re.Pattern = re.compile(r"zenodo\.org/record/([0-9]+)")
+
+
+def _clean_a_dict(catalog: Catalog) -> Catalog:
+    invalid: List[str] = []
+
+    for schema in catalog:
+        for status in validator.run([catalog[schema]], schema_path="auto"):
+            if not status.status:  # If the schema is invalid
+                invalid.append(schema)
+                logging.warning(f"Invalid schema file for {schema}")
+
+    for key in invalid:
+        del catalog[key]
+
+    return catalog
+
+
+def _upgrade_a_dict(catalog: Catalog) -> Catalog:
+    for schema in catalog:
+        new_version, nb_upgrade = recursive_update(catalog[schema])
+        if nb_upgrade:
+            catalog[schema] = new_version
+
+    return catalog
+
+
+def get_local_yaml(directory: str, keep_valid_only: bool = True) -> Catalog:
+    """ Reads all local YAML file in a given directory and parses them as Catalog Record.
+
+    :param directory: Directory to scan
+    :param keep_valid_only: Only keeps passing HTR-United files
+
+    """
+    out = {}
+    for root, dirs, files in os.walk(directory):
+        for file in files:
+            if file.endswith(".yml") or file.endswith(".yaml"):
+                try:
+                    data = parse_yaml(os.path.join(root, file))
+                    out[data.get("url")] = data
+                except Exception as E:
+                    logging.warning(f"Impossible to parse and understand {file}")
+                    logger.info(str(E))
+    if keep_valid_only:
+        _clean_a_dict(out)
+    return out
+
+
+def clever_catalog_update(catalog1: Dict, catalog2: Dict) -> Dict:
+    for repository in catalog2:
+        if repository in catalog1:
+            for key in ["characters", "volume"]:
+                if key in catalog2[repository]:
+                    catalog1[repository][key] = catalog2[repository][key]
+        else:
+            catalog1[repository] = catalog2[repository]
+    return catalog1
+
+
+def get_all_catalogs(
+    access_token: Optional[str] = None,
+    local_directory: Optional[str] = None,
+    get_distant: bool = True,
+    organizations: Optional[Union[str, Iterable[str]]] = "htr-united",
+    check_link: bool = False,
+    ignore_orgs_gits: List[str] = None,
+    keep_valid_only: bool = True,
+    auto_upgrade: bool = False,
+    citation_cff: bool = False
+) -> Catalog:
+    """ Retrieve repositories from various location (online, locally) and create a catalog out of the records.
+
+    :param access_token: Github Access Token to retrieve information ~ without limit from Github.com
+    :param local_directory: Local directory to scan for files
+    :param get_distant: Retrieves data from organisations on Github (Scan all their repositories)
+    :param organizations: Organizations to scan
+    :param check_link: If a local directory catalog record links to a github repository, scan the remote repository
+        for any updates on the catalog
+    :param ignore_orgs_gits: Ignore specific repositories in the scan
+    :param keep_valid_only: Only Keeps valid catalog record
+    :param auto_upgrade: Upgrade automatically all schemas to the latest version (Only applied if keep_valid_only is
+        True)
+    """
+    data: Catalog = {}
+    if local_directory:
+        data.update(get_local_yaml(directory=local_directory, keep_valid_only=False))
+        if check_link:
+            for uri in data:
+                # We update the catalog if needs be by checking each repo
+                if "github.com" in uri:
+                    print(f"Fetching {uri} remotely to update metrics")
+                    results = get_github_repo_yaml(address=uri, access_token=access_token)
+                    if results:
+                        data[uri] = results
+    if get_distant:
+        if isinstance(organizations, str):
+            organizations = (organizations, )
+        for orga in organizations:
+            data = clever_catalog_update(
+                data,
+                get_htr_united_repos(
+                    access_token=access_token,
+                    main_organization=orga,
+                    exclude=ignore_orgs_gits
+                )
+            )
+    if keep_valid_only:
+        _clean_a_dict(data)
+    if auto_upgrade and keep_valid_only:
+        _upgrade_a_dict(data)
+    if citation_cff:
+        for key in data:
+            up = _get_bibtex_and_apa(data[key], access_token=access_token)
+            if up:
+                logger.info(f"Successfully retrieved Bibtex or/and APA for {key}")
+                data[key].update(up)
+    return dict(sorted(data.items()))
+
+
+def get_statistics(repositories: Catalog) -> pandas.DataFrame:
+    """ Retrieve statistics from a diction of repositories
+
+    :param repositories: Dictionary of Repositories records
+
+    >>> #x = get_all_catalogs(local_directory="/home/thibault/dev/htr-united", check_link=False, get_distant=False)
+    >>> #get_statistics(x).groupby(by="metric").sum()
+    """
+    df = [
+
+    ]
+    for repository, entry in repositories.items():
+        try:
+            begin, end = entry["time"]["notBefore"], entry["time"]["notAfter"]
+            for a_volume in entry.get("volume", []):
+                df.append({
+                    "uri": repository,
+                    "title": entry["title"],
+                    "start": int(begin),
+                    "end": int(end),
+                    "metric": a_volume["metric"].lower(),
+                    "count": int(a_volume["count"]),
+                    "format": entry["format"],
+                    "script-type": entry["script-type"]
+                })
+        except KeyError:
+            logger.warning(f"Unable to parse {repository} for statistics")
+        except TypeError:
+            logger.warning(f"Unable to parse {repository} for statistics")
+    return pandas.DataFrame(df)
+
+
+def group_per_year(df: pandas.DataFrame, column: Optional[str] = "metric", period: int = 50):
+    """ Group a column per year
+
+    >>> group_per_year(pandas.DataFrame([
+    ... {"start": 1300, "end": 1399, "metric": "line", "count": 134},
+    ... {"start": 1300, "end": 1399, "metric": "characters", "count": 234},
+    ... {"start": 1350, "end": 1449, "metric": "line", "count": 34},
+    ... {"start": 1500, "end": 1551, "metric": "line", "count": 37},
+    ... {"start": 1503, "end": 1504, "metric": "line", "count": 37},
+    ... {"start": 1300, "end": 1499, "metric": "line", "count": 2}]))
+       year  characters  line
+    0  1300         234   136
+    1  1350         234   170
+    2  1400           0    36
+    3  1450           0     2
+    4  1500           0    74
+    5  1550           0    37
+
+    """
+    new_df = [
+        {
+            "year": range_start,
+            **df[
+                df.start.between(range_start, range_start+period-1) | \
+                df.end.between(range_start, range_start+period-1) | \
+                ((df.start < range_start) & (range_start+period-1 < df.end))
+            ].groupby(column)["count"].sum()
+        }
+        for range_start in range(
+            df.start.min() // period * period,
+            period * (df.end.max() // period) + int(bool(df.end.max() % period)),
+            period
+        )
+    ]
+    return pandas.DataFrame(new_df).fillna(0).astype(int)
+
+
+MetricLists = List[Dict[str, int]]
+
+
+def update_volume(original_volume: MetricLists, metrics: MetricLists) -> Tuple[MetricLists, MetricLists]:
+    """ Compute the new metrics for a catalog, returns a difference list as a second output
+
+    >>> old = [{"metric": "pages", "count": 5}, {"metric": "documents", "count": 5}]
+    >>> new = [{"metric": "pages", "count": 10}, {"metric": "line", "count": 105}]
+    >>> out = update_volume(old, new)
+    >>> out == (
+    ...    [{"metric": "documents", "count": 5}, {"metric": "line", "count": 105}, {"metric": "pages", "count": 10}],
+    ...    [{"metric": "pages", "count": 5}],
+    ... )
+    True
+
+    """
+    old = {vol["metric"]: vol["count"] for vol in original_volume}
+    new = {vol["metric"]: vol["count"] for vol in metrics}
+    all_keys = sorted(list(set(old.keys()).union(set(new.keys()))))
+    diff = {key: new.get(key) - old.get(key) for key in all_keys if key in old and key in new}
+    return (
+        [{"metric": key, "count": new.get(key, old.get(key))} for key in all_keys],
+        [{"metric": key, "count": diff.get(key)} for key in diff]
+    )
+
+
+def _get_bibtex_and_apa(catalog_record: CatalogRecord, access_token: Optional[str]=None) -> Dict[str, str]:
+    """
+
+    """
+    through_github = _get_github_citation_file(catalog_record, access_token)
+    if through_github:
+        return through_github
+
+    if _ZenodoRecord.search(catalog_record["url"]):
+        record = _ZenodoRecord.findall(catalog_record["url"])[0]
+        try:
+            req = requests.get(f"https://zenodo.org/api/records/{record}", headers={"Accept": "application/x-bibtex"})
+            req.raise_for_status()
+            return {"_bibtex": req.text}
+        except Exception as E:
+            logger.error(f"Unable to reach Zenodo for Bibtex ({catalog_record['url']}): {E}")
+
+    if "doi.org" in catalog_record["url"]:
+        try:
+            req = requests.get(catalog_record["url"], headers={"Accept": "application/x-bibtex"})
+            req.raise_for_status()
+            return {"_bibtex": req.text}
+        except Exception as E:
+            logger.error(f"Unable to reach the DOI API for Bibtex ({catalog_record['url']}): {E}")
+
+    return {}
+
+
+def _get_github_citation_file(catalog_record: CatalogRecord, access_token: Optional[str] = None) -> Dict[str, str]:
+    if "citation-file-link" not in catalog_record and "github.com" not in catalog_record["url"]:
+        return {}
+    elif "citation-file-link" not in catalog_record:
+        citation_file_content = get_github_repo_cff(catalog_record["url"], access_token=access_token)
+        if not citation_file_content:
+            return {}
+    else:  # We got a URI
+        try:
+            req = requests.get(catalog_record["citation-file-link"])
+            req.raise_for_status()
+            citation_file_content = req.text
+            if "</html>" in citation_file_content.lower():
+                raise Exception("CFF File link is wrong, it returns HTML.")
+            elif citation_file_content[0] == "{":
+                raise Exception("Got JSON at the given endpoint instead of YAML")
+        except Exception as E:
+            logger.error(f"Error retrieving CITATION File for {catalog_record['citation-file-link']}: {str(E)}")
+            if "github.com" in catalog_record["url"]:
+                logger.error(f"Trying to reach github directly")
+                return _get_github_citation_file({"url": catalog_record["url"]}, access_token=access_token)
+            return {}
+
+    try:
+        citation = cffconvert.Citation(citation_file_content)
+    except Exception as E:
+        logger.error(f"Unable to parse CFF for {catalog_record['url']} ({E})")
+        nl = "\n"
+        logger.error(f"Content: \n>>>    {citation_file_content.replace(nl, nl+'>>>    ')}")
+        return {}
+    return_obj = {}
+    try:
+        return_obj["_bibtex"] = citation.as_bibtex()
+    except Exception as E:
+        logger.error(f"Unable to parse as Bibtex {catalog_record['url']} ({E})")
+
+    try:
+        return_obj["_apa"] = citation.as_apalike()
+    except Exception as E:
+        logger.error(f"Unable to parse as APA {catalog_record['url']} ({E})")
+
+    return return_obj

htruc/cli.py

@@ -0,0 +1,210 @@
+import sys
+import click
+import os.path
+from ruamel.yaml import YAML
+import json
+from typing import Optional, List
+
+from htruc.validator import run
+from htruc.catalog import get_all_catalogs, get_statistics, group_per_year, update_volume, _get_bibtex_and_apa
+from htruc.utils import parse_yaml, create_json_catalog, get_local_or_download, dump_yaml
+
+
+def _error(message):
+    click.echo(
+        click.style(message, fg="red"),
+        color=True
+    )
+
+
+@click.group()
+def cli():
+    """ Interface for HTRUC """
+
+
+@cli.command("test")
+@click.argument("files", type=click.File(), nargs=-1)
+@click.option(
+    "--version", type=str, default="auto", show_default=True,
+    help="Date of the schema version"
+)
+@click.option("--force-download", is_flag=True, help="Download the schema using the version provided")
+def test(files, version: str, force_download: bool):
+    """ Test catalog files """
+    click.echo(f"{len(files)} to be tested")
+    statuses = []
+    if version != "auto":
+        version = get_local_or_download(version, force_download=force_download)
+    for status in run(files, schema_path=version):
+        statuses.append(status.status)
+        if status.status is False:
+            _error(f"☒ File `{status.filename}` testing failed")
+            for message in status.messages:
+                _error(f"  {message}")
+    click.echo()
+    click.echo(
+        click.style(
+            f"{statuses.count(True)/len(statuses)*100:.2f}% of schema passed ({statuses.count(True)}/{len(statuses)})",
+            fg="red" if False in statuses else "green"
+        )
+    )
+    sys.exit(-1 if False in statuses else 0)
+
+
+@cli.command("make")
+@click.argument("directory", default="./catalog/")
+@click.option("-o", "--organization", default=("htr-united", ), show_default=True, multiple=True,
+              help="Organization to retrieve repositories from")
+@click.option("--remote/--no-remote", is_flag=True, default=True, show_default=True,
+              help="Retrieve data from remote repositories in the organization's account")
+@click.option("--clean/--dirty", is_flag=True, default=True, show_default=True,
+              help="Keep only the valid catalog records")
+@click.option("--auto-upgrade/--no-auto-upgrade", is_flag=True, default=True, show_default=True,
+              help="Automatically upgrade to the latest schema")
+@click.option("--citation/--no-citation", is_flag=True, default=True, show_default=True,
+              help="Retrieve CITATION.CFF from repositories and creates unstandardized _apa and _bibtex properties "
+                   "for each record")
+@click.option("--check-link", is_flag=True, default=False, show_default=True,
+              help="For each github repository documented in the local files, tries to download a `htr-united.yaml`"
+                   " file from it.")
+@click.option("--output", default="catalog.yaml", show_default=True,
+              help="Dumps the agglutinated catalog as YAML")
+@click.option("--json", default=None, show_default=True,
+              help="Dumps the whole catalog as JSON too")
+@click.option("--graph", default=None, show_default=True,
+              help="Produce a graph at the path given (PNG Files please) with the amount of metrics"
+                   "at different times")
+@click.option("--graph-csv", default=None, show_default=True,
+              help="Outputs the data behind the graph into a CSV file")
+@click.option("--access_token", default=None, show_default=True,
+              help="Github Access token")
+@click.option("--statistics", default=None, show_default=True,
+              help="Produce a recap CSV file with different statistics about the period covered by the dataset")
+@click.option("--ignore-repo", default=["htr-united", "template-htr-united-datarepo", "template-depot"], multiple=True, show_default=True,
+              help="Repos of the main organization that can be ignored")
+@click.option("--ids", default="ids.json", type=click.Path(dir_okay=False), show_default=True,
+              help="JSON file with IDs that maps each repository URLs")
+def make(directory, organization: str, access_token: Optional[str] = None, remote: bool = True,
+         check_link: bool = False, output: str = "catalog.yaml",
+         json: Optional[str] = None,
+         graph: Optional[str] = None,
+         statistics: Optional[str] = None,
+         graph_csv: Optional[str] = None,
+         ignore_repo: List[str] = None,
+         ids: click.File = None,
+         auto_upgrade: bool = True,
+         clean: bool = True,
+         citation: bool = True):
+    """ Generate a catalog from a main repository and an organization
+
+    """
+    catalog = get_all_catalogs(
+        access_token=access_token,
+        organizations=organization,
+        local_directory=directory,
+        get_distant=remote,
+        check_link=check_link,
+        ignore_orgs_gits=ignore_repo,
+        keep_valid_only=clean,
+        auto_upgrade=auto_upgrade,
+        citation_cff=citation
+    )
+    click.echo(f"Dumping YAML output into {output}")
+    with open(output, "w") as f:
+        dump_yaml(list(catalog.values()), f, sort_keys=False)
+
+    if json:
+        click.echo(f"Dumping JSON output into {json}")
+        from json import dump
+        with open(json, "w") as f:
+            dump(create_json_catalog(catalog, ids_files=ids), f)
+    if graph or statistics or graph_csv:
+        stats = get_statistics(catalog)
+        if statistics:
+            click.echo(f"Writing stats to {statistics}")
+            stats.to_csv(statistics)
+        if graph or graph_csv:
+            data = group_per_year(stats)
+            if graph_csv:
+                click.echo(f"Plotting stats to {graph_csv}")
+                data.to_csv(graph_csv)
+            if graph:
+                click.echo(f"Plotting {len(data.columns)-1} files with {graph} basename")
+                basedir, basename = os.path.dirname(graph), os.path.basename(graph)
+                basename = ".".join(basename.split(".")[:-1])
+                import matplotlib.pyplot as plot
+
+                num_axes = len(data.columns) - 1
+                nrows = num_axes // 2 + int(bool(num_axes % 2))
+                fig, axes = plot.subplots(
+                    nrows=nrows,
+                    ncols=2,
+                    sharex=True,
+                    squeeze=True,
+                    figsize=(10, 5 * nrows),
+                    dpi=300
+                )
+                cols = [col for col in data.columns if col != "year"]
+                for metric, ax in zip(cols, [c for r in axes for c in r]):
+                    data.plot.line(x="year", y=metric, ax=ax)
+                fig.savefig(graph)
+                click.echo(f"Saved {graph}")
+
+
+@cli.command("update-volumes")
+@click.argument("catalog-file", type=click.File(), nargs=1)
+@click.argument("metrics-json", type=click.File(), nargs=1)
+@click.option(
+    "--inplace", type=bool, is_flag=True, default=False, show_default=True,
+    help="Saves the modified catalog inside the original file"
+)
+def catalog_volume_update(catalog_file, metrics_json, inplace):
+    """ Update the metrics of a file """
+    record = parse_yaml(catalog_file)
+    parsed_metrics = json.load(metrics_json)
+    metrics_volume = parsed_metrics["volume"]
+    updated, difference = update_volume(record.get("volume", []), metrics_volume)
+    record["volume"] = updated
+    for metric in difference:
+        if metric["count"] < 0:
+            click.echo(click.style(f"> The category `{metric['metric']}` decreased by {abs(metric['count'])}",
+                                   fg="yellow"))
+        else:
+            click.echo(click.style(f"> The category `{metric['metric']}` increased by {metric['count']}", fg="green"))
+
+    # Close the original file
+    catalog_file.close()
+
+    if record["schema"] != "2021-10-15" and "characters" in parsed_metrics:
+        if "characters" not in record:
+            record["characters"] = {}
+        record["characters"].update(parsed_metrics["characters"])
+
+    filename = f"{catalog_file.name}"
+    if not inplace:
+        filename = filename.split(".")
+        filename = ".".join([*filename[:-1], "auto-update", filename[-1]])
+    click.echo(f"Writing the update volumes in {filename}")
+    with open(filename, "w") as f:
+        dump_yaml(record, f, sort_keys=False)
+
+
+@cli.command("upgrade")
+@click.argument("files", type=click.File(), nargs=-1)
+def upgrade(files):
+    """ Upgrade [FILES] to the latest supported schema """
+    for file in files:
+        click.echo(click.style(f"Upgrading {file.name}", fg="green"))
+        catalog = parse_yaml(file)
+        from htruc.schemas import recursive_update
+        catalog, upgrade_order = recursive_update(catalog)
+        if not upgrade_order:
+            click.echo(click.style(f"--> No upgrade required", fg="yellow"))
+            continue
+        file.close()
+        with open(file.name, "w") as f:
+            dump_yaml(catalog, f, sort_keys=False)
+
+
+if __name__ == "__main__":
+    cli()

htruc/repos/__init__.py

@@ -0,0 +1,2 @@
+from ._generic import get_a_yaml
+from ._github import get_htr_united_repos, get_github_repo_yaml, get_github_repo_cff

htruc/repos/_generic.py

@@ -0,0 +1,21 @@
+from typing import Optional, Dict, Any
+import requests
+from htruc.utils import parse_yaml
+
+
+Catalog = Dict[str, Any]
+
+
+def get_a_yaml(address: str, raise_on_parse_error: bool = False) -> Optional[Catalog]:
+    req = requests.get(address)
+    if req.status_code >= 400:
+        return None
+
+    yaml = req.text
+    try:
+        return parse_yaml(yaml)
+    except Exception as E:
+        print(f"Parse error on {address}")
+        if raise_on_parse_error:
+            raise
+        return None

htruc/repos/_github.py

@@ -0,0 +1,86 @@
+from typing import Optional, Dict, Any, Iterable
+import re
+
+import github
+from ruamel.yaml import YAML, parser
+from github import Github
+from github.GithubException import UnknownObjectException
+from htruc.utils import parse_yaml
+
+
+Catalog = Dict[str, Any]
+
+
+def get_github_repo_yaml(
+        address: str,
+        access_token: Optional[str] = None,
+        raise_on_parse_error: bool = False) -> Optional[Catalog]:
+    """
+    >>> get_github_repo_yaml("github.com/htr-united/cremma-medieval.git")["title"]
+    'Cremma Medieval'
+    """
+
+    user, repo_name = re.findall("github.com/([^/]+)/([^/]+)", address)[0]
+    if repo_name.endswith(".git"):
+        repo_name = repo_name[:-4]
+    g = Github(access_token)
+    try:
+        repo = g.get_repo(f"{user}/{repo_name}")
+        text = repo.get_contents("htr-united.yml").decoded_content.decode()
+        print("--- Found htr-united.yml")
+    except UnknownObjectException as e:
+        return None
+    except github.GithubException as e:
+        return None
+
+    try:
+        return parse_yaml(text)
+    except parser.ParserError:
+        print(f"Parse error on {user}/{repo_name}")
+        if raise_on_parse_error:
+            raise
+        return None
+
+
+def get_github_repo_cff(
+        address: str,
+        access_token: Optional[str] = None) -> Optional[str]:
+    """
+
+    >>> get_github_repo_yaml("github.com/htr-united/cremma-medieval.git")["title"]
+    'Cremma Medieval'
+    """
+
+    user, repo_name = re.findall("github.com/([^/]+)/([^/]+)", address)[0]
+    if repo_name.endswith(".git"):
+        repo_name = repo_name[:-4]
+    g = Github(access_token)
+    repo = g.get_repo(f"{user}/{repo_name}")
+    for github_content in repo.get_contents(""):
+        if github_content.name.lower() == "citation.cff":
+            try:
+                text = repo.get_contents(github_content.name).decoded_content.decode()
+                return text
+            except UnknownObjectException as e:
+                return None
+
+
+def get_htr_united_repos(
+        access_token: Optional[str] = None,
+        main_organization: str = "htr-united",
+        exclude: Iterable[str] = ("htr-united", "template-htr-united-datarepo", )
+) -> Dict[str, Catalog]:
+    """ Get a single repo specific tokens
+
+    >>> get_htr_united_repos()
+    """
+    g = Github(access_token)
+    o = g.get_organization(main_organization)
+    out = {}
+    for repo in o.get_repos(type="public"):
+        if repo.name in exclude:
+            continue
+        data = get_github_repo_yaml(repo.clone_url, access_token=access_token)
+        if data:
+            out[repo.full_name] = data
+    return out

htruc/schemas/__init__.py

@@ -0,0 +1,34 @@
+from typing import Dict, Tuple, Callable, AnyStr, Iterable
+from htruc.schemas.upgrade_path import upgrade_2021_10_15_to_2022_04_15, upgrade_2022_04_15_to_2023_06_27
+from htruc.types import SchemaVersion, CatalogRecord
+
+
+# This Tuple is used to know which schema are supported
+UpgradeOrder: Tuple[SchemaVersion, ...] = (
+    "https://htr-united.github.io/schema/2021-10-15/schema.json",
+    "https://htr-united.github.io/schema/2022-04-15/schema.json",
+    "https://htr-united.github.io/schema/2023-06-27/schema.json",
+)
+
+UpgradeFunction: Dict[str, Callable[[CatalogRecord], CatalogRecord]] = {
+    UpgradeOrder[0]: upgrade_2021_10_15_to_2022_04_15,
+    UpgradeOrder[1]: upgrade_2022_04_15_to_2023_06_27
+}
+
+
+def recursive_update(catalog_record: CatalogRecord) -> Tuple[CatalogRecord, Iterable[SchemaVersion]]:
+    """ Automatically upgrade a schema
+
+    :returns: The catalog record upgrade to the latest schema, with the list of upgrade it went through
+
+    """
+    version = catalog_record["schema"]
+    # If the version is the latest, no need to upgrade
+    if version == UpgradeOrder[-1]:
+        return catalog_record, []
+    current_version_index = UpgradeOrder.index(version)
+
+    for version in UpgradeOrder[current_version_index:-1]:
+        catalog_record = UpgradeFunction[version](catalog_record)
+
+    return catalog_record, UpgradeOrder[current_version_index:-1]