commonmeta-py 0.22__py3-none-any.whl → 0.24__py3-none-any.whl

commonmeta/__init__.py

@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+
+# commonmeta-py
+
+"""
+commonmeta-py library
+~~~~~~~~~~~~~~~~~~~~~
+
+commonmeta-py is a Python library to convert scholarly metadata
+"""
+
+__title__ = "commonmeta-py"
+__version__ = "0.24"
+__author__ = "Martin Fenner"
+__license__ = "MIT"
+
+# ruff: noqa: F401
+from .metadata import Metadata, MetadataList
+from .readers import (
+    cff_reader,
+    codemeta_reader,
+    crossref_reader,
+    crossref_xml_reader,
+    datacite_reader,
+    datacite_xml_reader,
+    inveniordm_reader,
+    json_feed_reader,
+    kbase_reader,
+    ris_reader,
+    schema_org_reader,
+)
+from .writers import (
+    bibtex_writer,
+    citation_writer,
+    commonmeta_writer,
+    csl_writer,
+    datacite_writer,
+    ris_writer,
+    schema_org_writer,
+)
+from .utils import (
+    dict_to_spdx,
+    from_csl,
+    from_schema_org,
+    get_language,
+    normalize_cc_url,
+    normalize_id,
+    normalize_ids,
+    normalize_orcid,
+    normalize_url,
+    normalize_ror,
+    pages_as_string,
+    to_csl,
+    validate_orcid,
+    validate_url,
+    get_language,
+    encode_doi,
+    name_to_fos,
+    from_json_feed,
+)
+from .author_utils import (
+    authors_as_string,
+    cleanup_author,
+    get_affiliations,
+    get_authors,
+    get_one_author,
+    is_personal_name,
+)
+from .base_utils import (
+    wrap,
+    unwrap,
+    compact,
+    presence,
+    parse_attributes,
+    sanitize,
+)
+from .date_utils import (
+    get_date_from_crossref_parts,
+    get_date_from_date_parts,
+    get_date_from_unix_timestamp,
+    get_date_parts,
+    get_iso8601_date,
+    strip_milliseconds,
+)
+from .doi_utils import (
+    crossref_api_url,
+    crossref_xml_api_url,
+    doi_from_url,
+    doi_as_url,
+    doi_resolver,
+    datacite_api_url,
+    get_doi_ra,
+    normalize_doi,
+    validate_doi,
+    validate_prefix,
+)

commonmeta/api_utils.py

@@ -0,0 +1,77 @@
+"""API Utils module for commonmeta-py"""
+
+from typing import Optional
+from datetime import datetime as date
+import httpx
+from furl import furl
+import jwt
+
+from .doi_utils import validate_doi, doi_as_url
+from .readers.json_feed_reader import get_json_feed_item_uuid
+
+
+def generate_ghost_token(key: str) -> str:
+    """Generate a short-lived JWT for the Ghost Admin API.
+    From https://ghost.org/docs/admin-api/#token-authentication"""
+
+    # Split the key into ID and SECRET
+    _id, secret = key.split(":")
+
+    # Prepare header and payload
+    iat = int(date.now().timestamp())
+
+    header = {"alg": "HS256", "typ": "JWT", "kid": _id}
+    payload = {"iat": iat, "exp": iat + 5 * 60, "aud": "/admin/"}
+
+    # Create and return the token (including decoding secret)
+    return jwt.encode(payload, bytes.fromhex(secret), algorithm="HS256", headers=header)
+
+
+def update_ghost_post_via_api(
+    _id: str, api_key: Optional[str] = None, api_url: Optional[str] = None
+) -> dict[str, str]:
+    """Update Ghost post via API"""
+    # get post doi and url from Rogue Scholar API
+    # post url is needed to find post via Ghost API
+    post = get_json_feed_item_uuid(_id)
+    if post.get("error", None):
+        return post
+    doi = validate_doi(post.get("doi", None))
+    doi = doi_as_url(doi)
+    url = post.get("url", None)
+    if not doi or not url:
+        return {"error": "DOI or URL not found"}
+
+    # get post_id and updated_at from ghost api
+    token = generate_ghost_token(api_key)
+    headers = {
+        "Authorization": f"Ghost {token}",
+        "Content-Type": "application/json",
+        "Accept-Version": "v5",
+    }
+    f = furl(url)
+    slug = f.path.segments[-1]
+    ghost_url = f"{api_url}/ghost/api/admin/posts/slug/{slug}/"
+    response = httpx.get(ghost_url, headers=headers, timeout=10)
+    if response.status_code != 200:
+        return {"error": "Error fetching post"}
+    ghost_post = response.json().get("posts")[0]
+    guid = ghost_post.get("id")
+    updated_at = ghost_post.get("updated_at")
+    if not guid or not updated_at:
+        return {"error": "guid or updated_at not found"}
+
+    # update post canonical_url using doi. This requires sending
+    # the updated_at timestamp to avoid conflicts, and must use guid
+    # rather than url for put requests
+    ghost_url = f"{api_url}/ghost/api/admin/posts/{guid}/"
+
+    json = {"posts": [{"canonical_url": doi, "updated_at": updated_at}]}
+    response = httpx.put(
+        ghost_url,
+        headers=headers,
+        json=json,
+    )
+    if response.status_code != 200:
+        return {"error": "Error updating post"}
+    return {"message": f"DOI {doi} added", "guid": guid, "updated_at": updated_at}

commonmeta/author_utils.py

@@ -0,0 +1,260 @@
+"""Author utils module for commonmeta-py"""
+import re
+from typing import List
+from nameparser import HumanName
+from pydash import py_
+from furl import furl
+
+from .utils import (
+    normalize_orcid,
+    normalize_id,
+    normalize_ror,
+    normalize_isni,
+    format_name_identifier,
+    validate_ror,
+    validate_orcid,
+)
+from .base_utils import parse_attributes, wrap, presence, compact
+
+from .constants import (
+    COMMONMETA_CONTRIBUTOR_ROLES,
+)
+
+
+def get_one_author(author, **kwargs):
+    """parse one author string into commonmeta format"""
+    # if author is a string
+    if isinstance(author, str):
+        author = {"creatorName": author}
+
+    # malformed XML
+    if isinstance(author.get("creatorName", None), list):
+        return None
+
+    name = (
+        parse_attributes(author.get("creatorName", None))
+        or parse_attributes(author.get("contributorName", None))
+        or parse_attributes(author.get("name", None))
+    )
+    given_name = parse_attributes(author.get("givenName", None)) or parse_attributes(
+        author.get("given", None)
+    )
+    family_name = parse_attributes(author.get("familyName", None)) or parse_attributes(
+        author.get("family", None)
+    )
+
+    name = cleanup_author(name)
+
+    # make sure we have a name
+    if not name and not given_name and not family_name:
+        return None
+
+    # parse contributor roles, checking for roles supported by commonmeta
+    contributor_roles = wrap(
+        parse_attributes(author.get("contributorType", None))
+    ) or wrap(parse_attributes(author.get("contributor_roles", None)))
+    contributor_roles = [
+        i for i in contributor_roles if i in COMMONMETA_CONTRIBUTOR_ROLES
+    ] or ["Author"]
+
+    # parse author type, i.e. "Person", "Organization" or not specified
+    _type = parse_attributes(
+        author.get("creatorName", None), content="type", first=True
+    ) or parse_attributes(
+        author.get("contributorName", None), content="type", first=True
+    )
+    print(author)
+    # also handle Crossref, JSON Feed, or DataCite metadata
+    _id = (
+        author.get("id", None)
+        or author.get("ORCID", None)
+        or author.get("url", None)
+        or next(
+            (
+                format_name_identifier(i)
+                for i in wrap(author.get("nameIdentifiers", None or author.get("identifiers", None)))
+            ),
+            None,
+        )
+    )
+    _id = normalize_orcid(_id) or normalize_ror(_id) or normalize_isni(_id) or _id
+
+    # DataCite metadata
+    if isinstance(_type, str) and _type.endswith("al"):
+        _type = _type[:-3]
+
+    if not _type and isinstance(_id, str) and validate_ror(_id) is not None:
+        _type = "Organization"
+    elif not _type and isinstance(_id, str) and validate_orcid(_id) is not None:
+        _type = "Person"
+    elif not _type and (given_name or family_name):
+        _type = "Person"
+    elif not _type and name and kwargs.get("via", None) == "crossref":
+        _type = "Organization"
+    elif not _type and is_personal_name(name):
+        _type = "Person"
+    elif not _type and name:
+        _type = "Organization"
+
+    # split name for type Person into given/family name if not already provided
+    if _type == "Person" and name and not given_name and not family_name:
+        names = HumanName(name)
+
+        if names:
+            given_name = (
+                " ".join([names.first, names.middle]).strip() if names.first else None
+            )
+            family_name = names.last if names.last else None
+        else:
+            given_name = None
+            family_name = None
+
+    # support various keys for affiliations
+    affiliations = author.get("affiliation", None) or author.get("affiliations", None)
+
+    # return author in commonmeta format, using name vs. given/family name
+    # depending on type
+    return compact(
+        {
+            "id": _id,
+            "type": _type,
+            "contributorRoles": contributor_roles,
+            "name": name if _type == "Organization" else None,
+            "givenName": given_name if _type == "Person" else None,
+            "familyName": family_name if _type == "Person" else None,
+            "affiliations": presence(get_affiliations(wrap(affiliations))),
+        }
+    )
+
+
+def is_personal_name(name):
+    """is_personal_name"""
+    # personal names are not allowed to contain semicolons
+    if ";" in name:
+        return False
+
+    # check if a name has only one word, e.g. "FamousOrganization", not including commas
+    if len(name.split(" ")) == 1 and "," not in name:
+        return False
+
+    # check if name contains words known to be used in organization names
+    if any(
+        word in name
+        for word in [
+            "University",
+            "College",
+            "Institute",
+            "School",
+            "Center",
+            "Department",
+            "Laboratory",
+            "Library",
+            "Museum",
+            "Foundation",
+            "Society",
+            "Association",
+            "Company",
+            "Corporation",
+            "Collaboration",
+            "Consortium",
+            "Incorporated",
+            "Inc.",
+            "Institut",
+            "Research",
+            "Science",
+            "Team",
+            "Ministry",
+            "Government",
+        ]
+    ):
+        return False
+
+    # check for suffixes, e.g. "John Smith, MD"
+    if name.split(", ")[-1] in ["MD", "PhD", "BS"]:
+        return True
+
+    # check of name can be parsed into given/family name
+    names = HumanName(name)
+    if names and (names.first or names.last):
+        return True
+
+    return False
+
+
+def cleanup_author(author):
+    """clean up author string"""
+    if author is None:
+        return None
+
+    if author.startswith(","):
+        return None
+
+    # detect pattern "Smith J.", but not "Smith, John K."
+    if "," not in author:
+        author = re.sub(r"/([A-Z]\.)?(-?[A-Z]\.)/", ", \1\2", author)
+
+    # remove spaces around hyphens
+    author = author.replace(" - ", "-")
+
+    # remove non-standard space characters
+    author = re.sub("/[ \t\r\n\v\f]/", " ", author)
+    return author
+
+
+def get_authors(authors, **kwargs):
+    """transform array of author dicts into commonmeta format"""
+    return py_.uniq(py_.compact([get_one_author(i, **kwargs) for i in authors]))
+
+
+def authors_as_string(authors: List[dict]) -> str:
+    """convert authors list to string, e.g. for bibtex"""
+
+    def format_author(author):
+        if author.get("familyName", None) and author.get("givenName", None):
+            return f"{author['familyName']}, {author['givenName']}"
+        elif author.get("familyName", None):
+            return author["familyName"]
+        return author.get("name", None)
+
+    return " and ".join([format_author(i) for i in wrap(authors) if i is not None])
+
+
+def get_affiliations(affiliations: List[dict]) -> List[dict]:
+    """parse array of affiliation strings into commonmeta format"""
+
+    def format_element(i):
+        """format single affiliation element"""
+        affiliation_identifier = None
+        if isinstance(i, str):
+            name = i
+            scheme_uri = None
+        else:
+            if i.get("affiliationIdentifier", None) is not None:
+                affiliation_identifier = i["affiliationIdentifier"]
+                if i.get("schemeURI", None) is not None:
+                    scheme_uri = (
+                        i["schemeURI"]
+                        if i["schemeURI"].endswith("/")
+                        else "{affiliation['schemeURI']}/"
+                    )
+                affiliation_identifier = (
+                    normalize_id(scheme_uri + affiliation_identifier)
+                    if (
+                        not affiliation_identifier.startswith("https://")
+                        and scheme_uri is not None
+                    )
+                    else normalize_id(affiliation_identifier)
+                )
+            elif i.get("id", None) is not None:
+                f = furl(i.get("id"))
+                if f.scheme in ["http", "https"]:
+                    affiliation_identifier = i.get("id")
+            name = i.get("name", None) or i.get("#text", None)
+        return compact(
+            {
+                "id": affiliation_identifier,
+                "name": name,
+            }
+        )
+
+    return py_.uniq(py_.compact([format_element(i) for i in affiliations]))

commonmeta/base_utils.py

@@ -0,0 +1,121 @@
+"""Base utilities for commonmeta-py"""
+import html
+from os import path
+import re
+import xmltodict
+from typing import Optional, Union
+import nh3
+
+
+def wrap(item) -> list:
+    """Turn None, dict, or list into list"""
+    if item is None:
+        return []
+    if isinstance(item, list):
+        return item
+    return [item]
+
+
+def unwrap(lst: list) -> Optional[Union[dict, list]]:
+    """Turn list into dict or None, depending on list size"""
+    if len(lst) == 0:
+        return None
+    if len(lst) == 1:
+        return lst[0]
+    return lst
+
+
+def presence(
+    item: Optional[Union[dict, list, str]],
+) -> Optional[Union[dict, list, str]]:
+    """Turn empty list, dict or str into None"""
+    return None if item is None or len(item) == 0 or item == [{}] else item
+
+
+def compact(dict_or_list: Union[dict, list]) -> Optional[Union[dict, list]]:
+    """Remove None from dict or list"""
+    if isinstance(dict_or_list, dict):
+        return {k: v for k, v in dict_or_list.items() if v is not None}
+    if isinstance(dict_or_list, list):
+        lst = [compact(i) for i in dict_or_list]
+        return lst if len(lst) > 0 else None
+
+    return None
+
+
+def parse_attributes(
+    element: Union[str, dict, list], **kwargs
+) -> Optional[Union[str, list]]:
+    """extract attributes from a string, dict or list"""
+
+    def parse_item(item):
+        if isinstance(item, dict):
+            return item.get(html.unescape(content), None)
+        return html.unescape(item)
+
+    content = kwargs.get("content", "#text")
+    if isinstance(element, str) and kwargs.get("content", None) is None:
+        return html.unescape(element)
+    if isinstance(element, dict):
+        return element.get(html.unescape(content), None)
+    if isinstance(element, list):
+        arr = [parse_item(i) for i in element if i]
+        arr = arr[0] if len(arr) > 0 and kwargs.get("first") else unwrap(arr)
+        return arr
+
+
+def parse_xml(string: Optional[str], **kwargs) -> Optional[Union[dict, list]]:
+    """Parse XML into dict. Set default options, and options for Crossref XML"""
+    if string is None:
+        return None
+    if path.exists(string):
+        with open(string, encoding="utf-8") as file:
+            string = file.read()
+
+    if kwargs.get("dialect", None) == "crossref":
+        # remove namespaces from xml
+        namespaces = {
+            "http://www.crossref.org/schema/5.3.1": None,
+            "http://www.crossref.org/qrschema/3.0": None,
+            "http://www.crossref.org/xschema/1.0": None,
+            "http://www.crossref.org/xschema/1.1": None,
+            "http://www.crossref.org/AccessIndicators.xsd": None,
+            "http://www.crossref.org/relations.xsd": None,
+            "http://www.crossref.org/fundref.xsd": None,
+            "http://www.ncbi.nlm.nih.gov/JATS1": None,
+        }
+
+        kwargs["process_namespaces"] = True
+        kwargs["namespaces"] = namespaces
+        kwargs["force_list"] = {
+            "person_name",
+            "organization",
+            "titles",
+            "item",
+            "citation",
+            "program",
+        }
+
+    kwargs["attr_prefix"] = ""
+    kwargs["dict_constructor"] = dict
+    kwargs.pop("dialect", None)
+    return xmltodict.parse(string, **kwargs)
+
+
+def sanitize(text: str, **kwargs) -> str:
+    """Sanitize text"""
+    # default whitelisted HTML tags
+    tags = kwargs.get("tags", None) or {
+        "b",
+        "br",
+        "code",
+        "em",
+        "i",
+        "sub",
+        "sup",
+        "strong",
+    }
+    attributes = kwargs.get("attributes", None)
+    string = nh3.clean(text, tags=tags, attributes=attributes, link_rel=None)
+    # remove excessive internal whitespace
+    return " ".join(re.split(r"\s+", string, flags=re.UNICODE))