commonmeta-py 0.23__py3-none-any.whl → 0.24__py3-none-any.whl

commonmeta/readers/datacite_xml_reader.py

@@ -0,0 +1,359 @@
+"""datacite_xml reader for Commonmeta"""
+
+from collections import defaultdict
+import httpx
+from pydash import py_
+
+from ..base_utils import compact, wrap, presence, sanitize, parse_attributes
+from ..author_utils import get_authors
+from ..date_utils import strip_milliseconds, normalize_date_dict
+from ..doi_utils import doi_from_url, doi_as_url, datacite_api_url, normalize_doi
+from ..utils import normalize_url, normalize_cc_url, dict_to_spdx
+from ..constants import DC_TO_CM_TRANSLATIONS, Commonmeta
+
+
+def get_datacite_xml(pid: str, **kwargs) -> dict:
+    """get_datacite_xml"""
+    doi = doi_from_url(pid)
+    if doi is None:
+        return {"state": "not_found"}
+    url = datacite_api_url(doi)
+    response = httpx.get(url, timeout=10, **kwargs)
+    if response.status_code != 200:
+        return {"state": "not_found"}
+    return py_.get(response.json(), "data.attributes", {}) | {"via": "datacite_xml"}
+
+
+def read_datacite_xml(data: dict, **kwargs) -> Commonmeta:
+    """read_datacite_xml"""
+    if data is None:
+        return {"state": "not_found"}
+
+    read_options = kwargs or {}
+
+    meta = data.get("resource", {})
+
+    doi = parse_attributes(meta.get("identifier", None))
+    _id = doi_as_url(doi) if doi else None
+
+    resource__typegeneral = py_.get(meta, "resourceType.resourceTypeGeneral")
+    _type = DC_TO_CM_TRANSLATIONS.get(resource__typegeneral, "Other")
+    additional_type = py_.get(meta, "resourceType.#text")
+
+    identifiers = wrap(py_.get(meta, "alternateIdentifiers.alternateIdentifier"))
+    identifiers = get_xml_identifiers(identifiers)
+
+    def format_title(title):
+        """format_title"""
+        if isinstance(title, str):
+            return {"title": title}
+        if isinstance(title, dict):
+            return {
+                "title": title.get("#text", None),
+                "titleType": title.get("titleType", None),
+                "lang": title.get("xml:lang", None),
+            }
+        return None
+
+    titles = [format_title(i) for i in wrap(py_.get(meta, "titles.title"))]
+
+    contributors = get_authors(wrap(py_.get(meta, "creators.creator")))
+    contrib = get_authors(wrap(meta.get("contributors", None)))
+    if contrib:
+        contributors = contributors + contrib
+    publisher = {"name": py_.get(meta, "publisher")}
+    date = get_dates(
+        wrap(py_.get(meta, "dates.date")), meta.get("publicationYear", None)
+    )
+
+    def format_description(description):
+        """format_description"""
+        if isinstance(description, str):
+            return {"description": description, "type": "Abstract"}
+        if isinstance(description, dict):
+            return compact(
+                {
+                    "description": sanitize(description.get("#text", None)),
+                    "type": description.get("descriptionType", "Abstract"),
+                    "language": description.get("xml:lang", None),
+                }
+            )
+        return None
+
+    descriptions = [
+        format_description(i) for i in wrap(py_.get(meta, "descriptions.description"))
+    ]
+
+    def format_subject(subject):
+        """format_subject"""
+        if isinstance(subject, str):
+            return {"subject": subject, "subjectScheme": "None"}
+        if isinstance(subject, dict):
+            return compact(
+                {
+                    "subject": subject.get("#text", None),
+                    "subjectScheme": subject.get("subjectScheme", None),
+                    "language": subject.get("xml:lang", None),
+                }
+            )
+        return None
+
+    subjects = [format_subject(i) for i in wrap(py_.get(meta, "subjects.subject")) if i]
+
+    def format_geo_location(geo_location):
+        """format_geo_location"""
+        if isinstance(geo_location, str):
+            return {"geoLocationPlace": geo_location}
+        if isinstance(geo_location, dict):
+            return compact(
+                {
+                    "geoLocationPoint": compact(
+                        {
+                            "pointLatitude": compact(
+                                geo_location.get("geoLocationPoint.pointLatitude", None)
+                            ),
+                            "pointLongitude": compact(
+                                geo_location.get(
+                                    "geoLocationPoint.pointLongitude", None
+                                )
+                            ),
+                        }
+                    ),
+                    "geoLocationBox": compact(
+                        {
+                            "westBoundLongitude": compact(
+                                geo_location.get(
+                                    "geoLocationBox.westBoundLongitude", None
+                                )
+                            ),
+                            "eastBoundLongitude": compact(
+                                geo_location.get(
+                                    "geoLocationBox.eastBoundLongitude", None
+                                )
+                            ),
+                            "southBoundLatitude": compact(
+                                geo_location.get(
+                                    "geoLocationBox.southBoundLatitude", None
+                                )
+                            ),
+                            "northBoundLatitude": compact(
+                                geo_location.get(
+                                    "geoLocationBox.northBoundLatitude", None
+                                )
+                            ),
+                        }
+                    ),
+                    "geoLocationPolygon": {
+                        "polygonPoint": compact(
+                            {
+                                "pointLatitude": geo_location.get(
+                                    "geoLocationPolygon.polygonPoint.pointLatitude",
+                                    None,
+                                ),
+                                "pointLongitude": geo_location.get(
+                                    "geoLocationPolygon.polygonPoint.pointLongitude",
+                                    None,
+                                ),
+                            }
+                        )
+                    },
+                    "geoLocationPlace": geo_location.get("geoLocationPlace", None),
+                }
+            )
+        return None
+
+    geo_locations = []  # [format_geo_location(i) for i in wrap(py_.get(meta, "geoLocations.geoLocation")) if i]
+
+    def map_rights(rights):
+        """map_rights"""
+        return compact(
+            {
+                "rights": rights.get("#text", None),
+                "url": rights.get("rightsURI", None),
+                "lang": rights.get("xml:lang", None),
+            }
+        )
+
+    license_ = wrap(py_.get(meta, "rightsList.rights"))
+    if len(license_) > 0:
+        license_ = normalize_cc_url(license_[0].get("rightsURI", None))
+        license_ = dict_to_spdx({"url": license_}) if license_ else None
+
+    references = get_xml_references(
+        wrap(py_.get(meta, "relatedIdentifiers.relatedIdentifier"))
+    )
+    relations = get_xml_relations(
+        wrap(py_.get(meta, "relatedIdentifiers.relatedIdentifier"))
+    )
+
+    def map_funding_reference(funding_reference):
+        """map_funding_reference"""
+        return {
+            "funderName": funding_reference.get("funderName", None),
+            "funderIdentifier": funding_reference.get("funderIdentifier", None),
+            "funderIdentifierType": funding_reference.get("funderIdentifierType", None),
+            "awardNumber": funding_reference.get("awardNumber", None),
+            "awardTitle": funding_reference.get("awardTitle", None),
+        }
+
+    funding_references = []  # [map_funding_reference(i) for i in wrap(py_.get(meta, "fundingReferences.fundingReference"))]
+
+    files = meta.get("contentUrl", None)
+    state = "findable" if _id or read_options else "not_found"
+
+    return {
+        # required properties
+        "id": _id,
+        "type": _type,
+        "doi": doi_from_url(_id),
+        "url": normalize_url(meta.get("url", None)),
+        "contributors": presence(contributors),
+        "titles": compact(titles),
+        "publisher": publisher,
+        "date": date,
+        # recommended and optional properties
+        "additionalType": presence(additional_type),
+        "subjects": presence(subjects),
+        "language": meta.get("language", None),
+        "identifiers": identifiers,
+        "version": meta.get("version", None),
+        "license": presence(license_),
+        "descriptions": presence(descriptions),
+        "geoLocations": presence(geo_locations),
+        "fundingReferences": presence(funding_references),
+        "references": presence(references),
+        "relations": presence(relations),
+        # other properties
+        "date_created": strip_milliseconds(meta.get("created", None)),
+        "date_registered": strip_milliseconds(meta.get("registered", None)),
+        "date_published": strip_milliseconds(meta.get("published", None)),
+        "date_updated": strip_milliseconds(meta.get("updated", None)),
+        "files": presence(files),
+        "container": presence(meta.get("container", None)),
+        "provider": "DataCite",
+        "state": state,
+        "schema_version": meta.get("xmlns", None),
+    } | read_options
+
+
+def get_xml_identifiers(identifiers: list) -> list:
+    """get_identifiers"""
+
+    def is_identifier(identifier):
+        """supported identifier types"""
+        return identifier.get("alternateIdentifierType", None) in [
+            "ARK",
+            "arXiv",
+            "Bibcode",
+            "DOI",
+            "Handle",
+            "ISBN",
+            "ISSN",
+            "PMID",
+            "PMCID",
+            "PURL",
+            "URL",
+            "URN",
+            "Other",
+        ]
+
+    def format_identifier(identifier):
+        """format_identifier"""
+
+        if is_identifier(identifier):
+            type_ = identifier.get("alternateIdentifierType")
+        else:
+            type_ = "Other"
+
+        return compact(
+            {
+                "identifier": identifier.get("#text", None),
+                "identifierType": type_,
+            }
+        )
+
+    return [format_identifier(i) for i in identifiers]
+
+
+def get_xml_references(references: list) -> list:
+    """get_xml_references"""
+
+    def is_reference(reference):
+        """is_reference"""
+        return reference.get("relationType", None) in [
+            "Cites",
+            "References",
+        ] and reference.get("relatedIdentifierType", None) in ["DOI", "URL"]
+
+    def map_reference(reference):
+        """map_reference"""
+        identifier = reference.get("relatedIdentifier", None)
+        identifier_type = reference.get("relatedIdentifierType", None)
+        if identifier and identifier_type == "DOI":
+            reference["doi"] = normalize_doi(identifier)
+        elif identifier and identifier_type == "URL":
+            reference["url"] = normalize_url(identifier)
+        reference = py_.omit(
+            reference,
+            [
+                "relationType",
+                "relatedIdentifier",
+                "relatedIdentifierType",
+                "resourceTypeGeneral",
+                "schemeType",
+                "schemeUri",
+                "relatedMetadataScheme",
+            ],
+        )
+        return reference
+
+    return [map_reference(i) for i in references if is_reference(i)]
+
+
+def get_xml_relations(relations: list) -> list:
+    """get_xml_relations"""
+
+    def is_relation(relation):
+        """is_relation"""
+        return relation.get("relationType", None) in [
+            "IsNewVersionOf",
+            "IsPreviousVersionOf",
+            "IsVersionOf",
+            "HasVersion",
+            "IsPartOf",
+            "HasPart",
+            "IsVariantFormOf",
+            "IsOriginalFormOf",
+            "IsIdenticalTo",
+            "IsTranslationOf",
+            "IsReviewedBy",
+            "Reviews",
+            "IsPreprintOf",
+            "HasPreprint",
+            "IsSupplementTo",
+        ]
+
+    def map_relation(relation):
+        """map_relation"""
+        identifier = relation.get("relatedIdentifier", None)
+        identifier_type = relation.get("relatedIdentifierType", None)
+        if identifier and identifier_type == "DOI":
+            relation["doi"] = normalize_doi(identifier)
+        elif identifier and identifier_type == "URL":
+            relation["url"] = normalize_url(identifier)
+        return {
+            "id": identifier,
+            "type": identifier_type,
+        }
+
+    return [map_relation(i) for i in relations if is_relation(i)]
+
+
+def get_dates(dates: list, publication_year) -> dict:
+    """convert date list to dict, rename and/or remove some keys"""
+    date: dict = defaultdict(list)
+    for sub in dates:
+        date[sub.get("dateType", None)] = sub.get("#text", None)
+    if date.get("Issued", None) is None and publication_year is not None:
+        date["Issued"] = str(publication_year)
+    return normalize_date_dict(date)

commonmeta/readers/inveniordm_reader.py

@@ -0,0 +1,218 @@
+"""InvenioRDM reader for Commonmeta"""
+import httpx
+from pydash import py_
+from furl import furl
+
+from ..utils import (
+    normalize_url,
+    normalize_doi,
+    dict_to_spdx,
+    name_to_fos,
+    from_inveniordm,
+    get_language,
+)
+from ..base_utils import compact, wrap, presence, sanitize
+from ..author_utils import get_authors
+from ..date_utils import strip_milliseconds
+from ..doi_utils import doi_as_url, doi_from_url
+from ..constants import (
+    INVENIORDM_TO_CM_TRANSLATIONS,
+    COMMONMETA_RELATION_TYPES,
+    Commonmeta,
+)
+
+
+def get_inveniordm(pid: str, **kwargs) -> dict:
+    """get_inveniordm"""
+    if pid is None:
+        return {"state": "not_found"}
+    url = normalize_url(pid)
+    response = httpx.get(url, timeout=10, follow_redirects=True, **kwargs)
+    if response.status_code != 200:
+        return {"state": "not_found"}
+    return response.json()
+
+
+def read_inveniordm(data: dict, **kwargs) -> Commonmeta:
+    """read_inveniordm"""
+    print(data)
+    meta = data
+    read_options = kwargs or {}
+
+    url = normalize_url(py_.get(meta, "links.self_html"))
+    _id = doi_as_url(meta.get("doi", None)) or url
+    resource_type = py_.get(meta, "metadata.resource_type.type") or py_.get(meta, "metadata.resource_type.id")
+    resource_type = resource_type.split("-")[0]
+    _type = INVENIORDM_TO_CM_TRANSLATIONS.get(resource_type, "Other")
+    
+    contributors = py_.get(meta, "metadata.creators")
+    print(contributors)
+    
+    contributors = get_authors(
+        from_inveniordm(wrap(contributors)),
+    )
+    publisher = {"name": meta.get("publisher", None) or py_.get(meta, "metadata.publisher") or "Zenodo"}
+
+    title = py_.get(meta, "metadata.title")
+    titles = [{"title": sanitize(title)}] if title else None
+    additional_titles = py_.get(meta, "metadata.additional_titles")
+    # if additional_titles:
+    #     titles += [{"title": sanitize("bla")} for i in wrap(additional_titles)]
+
+    date: dict = {}
+    date["published"] = py_.get(meta, ("metadata.publication_date"))
+    if date["published"]:
+        date["published"] = date["published"].split("/")[0]
+    date["updated"] = strip_milliseconds(meta.get("updated", None))
+    f = furl(url)
+    if f.host == "zenodo.org":
+        container = compact(
+            {
+                "id": "https://www.re3data.org/repository/r3d100010468",
+                "type": "DataRepository" if _type == "Dataset" else "Repository",
+                "title": "Zenodo",
+            }
+        )
+    elif f.host in ["rogue-scholar.org", "beta.rogue-scholar.org", "demo.front-matter.io"]:
+        container = compact(
+            {
+                "type": "Repository",
+                "title": "Rogue Scholar",
+            }
+        )
+    else:
+        container = None
+    license_ = py_.get(meta, "metadata.license.id")
+    if license_:
+        license_ = dict_to_spdx({"id": license_})
+
+    descriptions = format_descriptions(
+        [
+            py_.get(meta, "metadata.description"),
+            py_.get(meta, "metadata.notes"),
+        ]
+    )
+    language = py_.get(meta, "metadata.language") or py_.get(meta, "metadata.languages[0].id")
+    subjects = [name_to_fos(i) for i in wrap(py_.get(meta, "metadata.keywords"))]
+
+    references = get_references(wrap(py_.get(meta, "metadata.related_identifiers")))
+    relations = get_relations(wrap(py_.get(meta, "metadata.related_identifiers")))
+    if meta.get("conceptdoi", None):
+        relations.append(
+            {
+                "id": doi_as_url(meta.get("conceptdoi")),
+                "type": "IsVersionOf",
+            }
+        )
+    files = [get_file(i) for i in wrap(meta.get("files"))]
+
+    return {
+        # required properties
+        "id": _id,
+        "type": _type,
+        "doi": doi_from_url(_id),
+        "url": url,
+        "contributors": presence(contributors),
+        "titles": titles,
+        "publisher": publisher,
+        "date": compact(date),
+        # recommended and optional properties
+        # "additional_type": additional_type,
+        "subjects": presence(subjects),
+        "language": get_language(language),
+        "version": py_.get(meta, "metadata.version"),
+        "license": presence(license_),
+        "descriptions": descriptions,
+        "geoLocations": None,
+        # "funding_references": presence(meta.get("fundingReferences", None)),
+        "references": presence(references),
+        "relations": presence(relations),
+        # other properties
+        "files": files,
+        "container": container,
+        "provider": "DataCite",
+    } | read_options
+
+
+def get_references(references: list) -> list:
+    """get_references"""
+
+    def is_reference(reference):
+        """is_reference"""
+        return reference.get("relationType", None) in ["Cites", "References"]
+
+    def map_reference(reference):
+        """map_reference"""
+        identifier = reference.get("relatedIdentifier", None)
+        identifier_type = reference.get("relatedIdentifierType", None)
+        if identifier and identifier_type == "DOI":
+            reference["id"] = normalize_doi(identifier)
+        elif identifier and identifier_type == "URL":
+            reference["id"] = normalize_url(identifier)
+        reference = py_.omit(
+            reference,
+            [
+                "relationType",
+                "relatedIdentifier",
+                "relatedIdentifierType",
+                "resourceTypeGeneral",
+                "schemeType",
+                "schemeUri",
+                "relatedMetadataScheme",
+            ],
+        )
+        return reference
+
+    return [map_reference(i) for i in references if is_reference(i)]
+
+
+def get_file(file: dict) -> str:
+    """get_file"""
+    _type = file.get("type", None)
+    return compact(
+        {
+            "bucket": file.get("bucket", None),
+            "key": file.get("key", None),
+            "checksum": file.get("checksum", None),
+            "url": py_.get(file, "links.self"),
+            "size": file.get("size", None),
+            "mimeType": "application/" + _type if _type else None,
+        }
+    )
+
+
+def get_relations(relations: list) -> list:
+    """get_relations"""
+
+    def map_relation(relation: dict) -> dict:
+        """map_relation"""
+        identifier = relation.get("identifier", None)
+        scheme = relation.get("scheme", None)
+        relation_type = relation.get("relation", None) or relation.get("relation_type", None)
+        if scheme == "doi":
+            identifier = doi_as_url(identifier)
+        else:
+            identifier = normalize_url(identifier)
+        return {
+            "id": identifier,
+            "type": py_.capitalize(relation_type, False) if relation_type else None,
+        }
+
+    identifiers = [map_relation(i) for i in relations]
+    return [
+        i
+        for i in identifiers
+        if py_.upper_first(i["type"]) in COMMONMETA_RELATION_TYPES
+    ]
+
+
+def format_descriptions(descriptions: list) -> list:
+    """format_descriptions"""
+    return [
+        {
+            "description": sanitize(i),
+            "type": "Abstract" if index == 0 else "Other",
+        }
+        for index, i in enumerate(descriptions)
+        if i
+    ]