commonmeta-py 0.107__tar.gz → 0.108__tar.gz

{commonmeta_py-0.107 → commonmeta_py-0.108}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: commonmeta-py
-Version: 0.107
+Version: 0.108
 Summary: Library for conversions to/from the Commonmeta scholarly metadata format
 Project-URL: Homepage, https://python.commonmeta.org
 Project-URL: Repository, https://github.com/front-matter/commonmeta-py
@@ -33,6 +33,7 @@ Requires-Dist: pydash<9,>=6
 Requires-Dist: pyjwt<3,>=2.8.0
 Requires-Dist: python-dateutil<3,>=2.8.2
 Requires-Dist: pyyaml>=5.4
+Requires-Dist: requests-toolbelt>=1.0.0
 Requires-Dist: requests>=2.31.0
 Requires-Dist: requests>=2.32.3
 Requires-Dist: simplejson~=3.18
@@ -40,6 +41,7 @@ Requires-Dist: types-beautifulsoup4<5,>=4.11
 Requires-Dist: types-dateparser~=1.1
 Requires-Dist: types-pyyaml>=5.4
 Requires-Dist: types-xmltodict<0.20,>=0.13
+Requires-Dist: xmlschema>=4.0.1
 Requires-Dist: xmltodict<0.20,>=0.12
 Description-Content-Type: text/markdown
 
@@ -85,7 +87,7 @@ Commometa-py reads and/or writes these metadata formats:
 | [BibTex](http://en.wikipedia.org/wiki/BibTeX)                                                    | bibtex        | application/x-bibtex                   | later | yes     |
 | [RIS](http://en.wikipedia.org/wiki/RIS_(file_format))                                            | ris           | application/x-research-info-systems    | yes   | yes     |
 | [InvenioRDM](https://inveniordm.docs.cern.ch/reference/metadata/)                                | inveniordm    | application/vnd.inveniordm.v1+json     | yes   | yes     |
-| [JSON Feed](https://www.jsonfeed.org/)                                                           | json_feed_item     | application/feed+json    | yes | later     |
+| [JSON Feed](https://www.jsonfeed.org/)                                                           | jsonfeed     | application/feed+json    | yes | later     |
 | [OpenAlex](https://www.openalex.org/)                                                           | openalex     |    | yes | no     |
 
 _commonmeta_: the Commonmeta format is the native format for the library and used internally.

{commonmeta_py-0.107 → commonmeta_py-0.108}/README.md

@@ -40,7 +40,7 @@ Commometa-py reads and/or writes these metadata formats:
 | [BibTex](http://en.wikipedia.org/wiki/BibTeX)                                                    | bibtex        | application/x-bibtex                   | later | yes     |
 | [RIS](http://en.wikipedia.org/wiki/RIS_(file_format))                                            | ris           | application/x-research-info-systems    | yes   | yes     |
 | [InvenioRDM](https://inveniordm.docs.cern.ch/reference/metadata/)                                | inveniordm    | application/vnd.inveniordm.v1+json     | yes   | yes     |
-| [JSON Feed](https://www.jsonfeed.org/)                                                           | json_feed_item     | application/feed+json    | yes | later     |
+| [JSON Feed](https://www.jsonfeed.org/)                                                           | jsonfeed     | application/feed+json    | yes | later     |
 | [OpenAlex](https://www.openalex.org/)                                                           | openalex     |    | yes | no     |
 
 _commonmeta_: the Commonmeta format is the native format for the library and used internally.

{commonmeta_py-0.107 → commonmeta_py-0.108}/commonmeta/__init__.py

@@ -10,7 +10,7 @@ commonmeta-py is a Python library to convert scholarly metadata
 """
 
 __title__ = "commonmeta-py"
-__version__ = "0.107"
+__version__ = "0.108"
 __author__ = "Martin Fenner"
 __license__ = "MIT"
 
@@ -31,18 +31,6 @@ from .base_utils import (
     unwrap,
     wrap,
 )
-from .crossref_utils import (
-    CrossrefBadRequestError,
-    CrossrefError,
-    CrossrefForbiddenError,
-    CrossrefNoContentError,
-    CrossrefNotFoundError,
-    CrossrefRequestError,
-    CrossrefServerError,
-    CrossrefUnauthorizedError,
-    HttpError,
-    generate_crossref_xml,
-)
 from .date_utils import (
     get_date_from_crossref_parts,
     get_date_from_date_parts,
@@ -66,6 +54,14 @@ from .doi_utils import (
     validate_doi,
     validate_prefix,
 )
+from .file_utils import (
+    download_file,
+    read_file,
+    read_gz_file,
+    read_zip_file,
+    uncompress_content,
+    unzip_content,
+)
 from .metadata import Metadata, MetadataList
 from .readers import (
     cff_reader,
@@ -75,7 +71,7 @@ from .readers import (
     datacite_reader,
     datacite_xml_reader,
     inveniordm_reader,
-    json_feed_reader,
+    jsonfeed_reader,
     kbase_reader,
     openalex_reader,
     ris_reader,
@@ -87,7 +83,7 @@ from .utils import (
     extract_url,
     extract_urls,
     from_csl,
-    from_json_feed,
+    from_jsonfeed,
     from_schema_org,
     get_language,
     issn_as_url,
@@ -109,6 +105,7 @@ from .writers import (
     bibtex_writer,
     citation_writer,
     commonmeta_writer,
+    crossref_xml_writer,
     csl_writer,
     datacite_writer,
     ris_writer,

{commonmeta_py-0.107 → commonmeta_py-0.108}/commonmeta/api_utils.py

@@ -7,8 +7,9 @@ import jwt
 import requests
 from furl import furl
 
+from commonmeta.readers.jsonfeed_reader import get_jsonfeed_uuid
+
 from .doi_utils import doi_as_url, validate_doi
-from .readers.json_feed_reader import get_json_feed_item_uuid
 
 
 def generate_ghost_token(key: str) -> str:
@@ -34,7 +35,7 @@ def update_ghost_post_via_api(
     """Update Ghost post via API"""
     # get post doi and url from Rogue Scholar API
     # post url is needed to find post via Ghost API
-    post = get_json_feed_item_uuid(_id)
+    post = get_jsonfeed_uuid(_id)
     if post.get("error", None):
         return post
     doi = validate_doi(post.get("doi", None))

commonmeta_py-0.108/commonmeta/base_utils.py

@@ -0,0 +1,307 @@
+"""Base utilities for commonmeta-py"""
+
+import html
+import re
+import uuid
+from datetime import datetime
+from os import path
+from typing import Optional, Union
+
+import nh3
+import pydash as py_
+import xmltodict
+
+
+def wrap(item) -> list:
+    """Turn None, dict, or list into list"""
+    if item is None:
+        return []
+    if isinstance(item, list):
+        return item
+    return [item]
+
+
+def unwrap(lst: list) -> Optional[Union[dict, list]]:
+    """Turn list into dict or None, depending on list size"""
+    if len(lst) == 0:
+        return None
+    if len(lst) == 1:
+        return lst[0]
+    return lst
+
+
+def presence(
+    item: Optional[Union[dict, list, str]],
+) -> Optional[Union[dict, list, str]]:
+    """Turn empty list, dict or str into None"""
+    return None if item is None or len(item) == 0 or item == [{}] else item
+
+
+def compact(dict_or_list: Union[dict, list]) -> Optional[Union[dict, list]]:
+    """Remove None from dict or list"""
+    if isinstance(dict_or_list, dict):
+        return {k: v for k, v in dict_or_list.items() if v is not None}
+    if isinstance(dict_or_list, list):
+        lst = [compact(i) for i in dict_or_list]
+        return lst if len(lst) > 0 else None
+
+    return None
+
+
+def parse_attributes(
+    element: Union[str, dict, list], **kwargs
+) -> Optional[Union[str, list]]:
+    """extract attributes from a string, dict or list"""
+
+    def parse_item(item):
+        if isinstance(item, dict):
+            return item.get(html.unescape(content), None)
+        return html.unescape(item)
+
+    content = kwargs.get("content", "#text")
+    if isinstance(element, str) and kwargs.get("content", None) is None:
+        return html.unescape(element)
+    if isinstance(element, dict):
+        return element.get(html.unescape(content), None)
+    if isinstance(element, list):
+        arr = [parse_item(i) for i in element if i]
+        arr = arr[0] if len(arr) > 0 and kwargs.get("first") else unwrap(arr)
+        return arr
+
+
+def parse_xml(string: Optional[str], **kwargs) -> Optional[Union[dict, list]]:
+    """Parse XML into dict using xmltodict. Set default options, and options for Crossref XML"""
+    if string is None or string == "{}":
+        return None
+    if path.exists(string):
+        with open(string, encoding="utf-8") as file:
+            string = file.read()
+
+    if kwargs.get("dialect", None) == "crossref":
+        # remove namespaces from xml
+        namespaces = {
+            "http://www.crossref.org/schema/5.4.0": None,
+            "http://www.crossref.org/qrschema/3.0": None,
+            "http://www.crossref.org/xschema/1.0": None,
+            "http://www.crossref.org/xschema/1.1": None,
+            "http://www.crossref.org/AccessIndicators.xsd": None,
+            "http://www.crossref.org/relations.xsd": None,
+            "http://www.crossref.org/fundref.xsd": None,
+            "http://www.ncbi.nlm.nih.gov/JATS1": None,
+        }
+
+        kwargs["process_namespaces"] = True
+        kwargs["namespaces"] = namespaces
+        kwargs["force_list"] = {
+            "person_name",
+            "organization",
+            "titles",
+            "abstract",
+            "item",
+            "citation",
+            "program",
+            "related_item",
+        }
+
+    kwargs["attr_prefix"] = ""
+    kwargs["dict_constructor"] = dict
+    kwargs.pop("dialect", None)
+    return xmltodict.parse(string, **kwargs)
+
+
+def unparse_xml(input: Optional[dict], **kwargs) -> str:
+    """Unparse (dump) dict into XML using xmltodict. Set default options, and options for Crossref XML"""
+    if input is None:
+        return None
+    if kwargs.get("dialect", None) == "crossref":
+        # Add additional logic for crossref dialect
+        # add body and root element as wrapping elements
+        type = next(iter(input))
+        attributes = input.get(type)
+        input.pop(type)
+
+        if type == "book":
+            book_metadata = py_.get(input, "book_metadata") or {}
+            input.pop("book_metadata")
+            book_metadata = {**book_metadata, **input}
+            input = {"book": {**attributes, "book_metadata": book_metadata}}
+        elif type == "database":
+            database_metadata = py_.get(input, "database_metadata") or {}
+            input.pop("database_metadata")
+            val = input.pop("publisher_item")
+            institution = input.pop("institution", None)
+            database_metadata = {**{"titles": val}, **database_metadata}
+            database_metadata["institution"] = institution or {}
+            component = input.pop("component", None)
+            input = {
+                "database": {
+                    **attributes,
+                    "database_metadata": database_metadata,
+                    "component_list": {"component": component | input},
+                }
+            }
+        elif type == "journal":
+            journal_metadata = py_.get(input, "journal_metadata") or {}
+            journal_issue = py_.get(input, "journal_issue") or {}
+            journal_article = py_.get(input, "journal_article") or {}
+            input.pop("journal_metadata")
+            input.pop("journal_issue")
+            input.pop("journal_article")
+            input = {
+                "journal": {
+                    "journal_metadata": journal_metadata,
+                    "journal_issue": journal_issue,
+                    "journal_article": journal_article | input,
+                }
+            }
+        elif type == "proceedings_article":
+            proceedings_metadata = py_.get(input, "proceedings_metadata") or {}
+            input.pop("proceedings_metadata")
+            input = {
+                "proceedings": {
+                    **attributes,
+                    "proceedings_metadata": proceedings_metadata,
+                    "conference_paper": input,
+                }
+            }
+        elif type == "sa_component":
+            component = py_.get(input, "component") or {}
+            input.pop("component")
+            input = {
+                "sa_component": {
+                    **attributes,
+                    "component_list": {"component": component | input},
+                }
+            }
+        else:
+            input = {type: attributes | input}
+
+        doi_batch = {
+            "@xmlns": "http://www.crossref.org/schema/5.4.0",
+            "@version": "5.4.0",
+            "head": get_crossref_xml_head(input),
+            "body": input,
+        }
+        input = {"doi_batch": doi_batch}
+    kwargs["pretty"] = True
+    kwargs["indent"] = "  "
+    kwargs.pop("dialect", None)
+    return xmltodict.unparse(input, **kwargs)
+
+
+def unparse_xml_list(input: Optional[list], **kwargs) -> str:
+    """Unparse (dump) list into XML using xmltodict. Set default options, and options for Crossref XML"""
+    if input is None:
+        return None
+    if kwargs.get("dialect", None) == "crossref":
+        # Add additional logic for crossref dialect
+        # add body and root element as wrapping elements
+
+        # Group items by type with minimal grouping
+        items_by_type = {}
+
+        for item in wrap(input):
+            type = next(iter(item))
+            attributes = item.get(type)
+            item.pop(type)
+
+            # handle nested book_metadata and journal structure as in unparse_xml
+            if type == "book":
+                book_metadata = py_.get(item, "book_metadata") or {}
+                item.pop("book_metadata")
+                book_metadata = {**book_metadata, **item}
+                item = {"book": {**attributes, "book_metadata": book_metadata}}
+            elif type == "database":
+                database_metadata = py_.get(item, "database_metadata") or {}
+                item.pop("database_metadata")
+                database_metadata = {**database_metadata, **item}
+                item = {
+                    "database": {**attributes, "database_metadata": database_metadata}
+                }
+            elif type == "journal":
+                journal_metadata = py_.get(item, "journal_metadata") or {}
+                journal_issue = py_.get(item, "journal_issue") or {}
+                journal_article = py_.get(item, "journal_article") or {}
+                item.pop("journal_metadata")
+                item.pop("journal_issue")
+                item.pop("journal_article")
+                item = {
+                    "journal": {
+                        "journal_metadata": journal_metadata,
+                        "journal_issue": journal_issue,
+                        "journal_article": journal_article | item,
+                    }
+                }
+            elif type == "sa_component":
+                component = py_.get(input, "component") or {}
+                item.pop("component")
+                item = {
+                    "sa_component": {
+                        **attributes,
+                        "component_list": {"component": component | item},
+                    }
+                }
+            else:
+                item = {type: attributes | item}
+
+            # Add item to appropriate type bucket
+            if type not in items_by_type:
+                items_by_type[type] = []
+            items_by_type[type].append(item[type])
+
+        # Create the final structure with body containing all grouped items
+        body_content = {}
+        for type_key, items in items_by_type.items():
+            if len(items) == 1:
+                body_content[type_key] = items[0]  # Use single item without array
+            else:
+                body_content[type_key] = items  # Use array when multiple items
+        head = kwargs["head"] or {}
+        doi_batch = {
+            "@xmlns": "http://www.crossref.org/schema/5.4.0",
+            "@xmlns:ai": "http://www.crossref.org/AccessIndicators.xsd",
+            "@xmlns:rel": "http://www.crossref.org/relations.xsd",
+            "@xmlns:fr": "http://www.crossref.org/fundref.xsd",
+            "@version": "5.4.0",
+            "head": get_crossref_xml_head(head),
+            "body": body_content,
+        }
+        output = {"doi_batch": doi_batch}
+
+    kwargs["pretty"] = True
+    kwargs["indent"] = "  "
+    kwargs.pop("dialect", None)
+    kwargs.pop("head", None)
+    return xmltodict.unparse(output, **kwargs)
+
+
+def sanitize(text: str, **kwargs) -> str:
+    """Sanitize text"""
+    # default whitelisted HTML tags
+    tags = kwargs.get("tags", None) or {
+        "b",
+        "br",
+        "code",
+        "em",
+        "i",
+        "sub",
+        "sup",
+        "strong",
+    }
+    attributes = kwargs.get("attributes", None)
+    string = nh3.clean(text, tags=tags, attributes=attributes, link_rel=None)
+    # remove excessive internal whitespace
+    return " ".join(re.split(r"\s+", string, flags=re.UNICODE))
+
+
+def get_crossref_xml_head(metadata: dict) -> dict:
+    """Get head element for Crossref XML"""
+    return {
+        "doi_batch_id": str(uuid.uuid4()),
+        "timestamp": datetime.now().strftime("%Y%m%d%H%M%S"),
+        "depositor": {
+            "depositor_name": metadata.get("depositor", None) or "test",
+            "email_address": metadata.get("email", None) or "info@example.org",
+        },
+        "registrant": metadata.get("registrant", None) or "test",
+    }

{commonmeta_py-0.107 → commonmeta_py-0.108}/commonmeta/cli.py

@@ -2,16 +2,12 @@ import time
 
 import click
 import orjson as json
-import pydash as py_
 
 from commonmeta import Metadata, MetadataList  # __version__
 from commonmeta.api_utils import update_ghost_post_via_api
 from commonmeta.doi_utils import decode_doi, encode_doi, validate_prefix
 from commonmeta.readers.crossref_reader import get_random_crossref_id
 from commonmeta.readers.datacite_reader import get_random_datacite_id
-from commonmeta.readers.json_feed_reader import (
-    get_json_feed_item_uuid,
-)
 from commonmeta.readers.openalex_reader import get_random_openalex_id
 
 
@@ -46,6 +42,49 @@ def convert(
     email,
     registrant,
     show_errors,
+):
+    metadata = Metadata(input, via=via, doi=doi, prefix=prefix)
+    if show_errors and not metadata.is_valid:
+        raise click.ClickException(str(metadata.errors))
+
+    click.echo(
+        metadata.write(
+            to=to,
+            style=style,
+            locale=locale,
+            depositor=depositor,
+            email=email,
+            registrant=registrant,
+        )
+    )
+    if show_errors and metadata.write_errors:
+        raise click.ClickException(str(metadata.write_errors))
+
+
+@cli.command()
+@click.argument("input", type=str, required=True)
+@click.option("--via", "-f", type=str, default=None)
+@click.option("--to", "-t", type=str, default="commonmeta")
+@click.option("--style", "-s", type=str, default="apa")
+@click.option("--locale", "-l", type=str, default="en-US")
+@click.option("--doi", type=str)
+@click.option("--prefix", type=str)
+@click.option("--depositor", type=str)
+@click.option("--email", type=str)
+@click.option("--registrant", type=str)
+@click.option("--show-errors/--no-errors", type=bool, show_default=True, default=False)
+def put(
+    input,
+    via,
+    to,
+    style,
+    locale,
+    doi,
+    prefix,
+    depositor,
+    email,
+    registrant,
+    show_errors,
 ):
     metadata = Metadata(input, via=via, doi=doi, prefix=prefix)
     if show_errors and not metadata.is_valid:
@@ -75,8 +114,7 @@ def convert(
 @click.option("--depositor", type=str)
 @click.option("--email", type=str)
 @click.option("--registrant", type=str)
-@click.option("--filename", type=str)
-@click.option("--jsonlines/--no-jsonlines", type=bool, show_default=True, default=False)
+@click.option("--file", type=str)
 @click.option("--show-errors/--no-errors", type=bool, show_default=True, default=False)
 @click.option("--show-timer/--no-timer", type=bool, show_default=True, default=False)
 def list(
@@ -89,8 +127,69 @@ def list(
     depositor,
     email,
     registrant,
-    filename,
-    jsonlines,
+    file,
+    show_errors,
+    show_timer,
+):
+    start = time.time()
+    metadata_list = MetadataList(
+        string,
+        via=via,
+        file=file,
+        depositor=depositor,
+        email=email,
+        registrant=registrant,
+        prefix=prefix,
+    )
+    end = time.time()
+    runtime = end - start
+    if show_errors and not metadata_list.is_valid:
+        raise click.ClickException(str(metadata_list.errors))
+    if file:
+        metadata_list.write(to=to, style=style, locale=locale)
+    else:
+        click.echo(metadata_list.write(to=to, style=style, locale=locale))
+
+    if show_errors and len(metadata_list.write_errors) > 0:
+        raise click.ClickException(str(metadata_list.write_errors))
+    if show_timer:
+        click.echo(f"Runtime: {runtime:.2f} seconds")
+
+
+@cli.command()
+@click.argument("string", type=str, required=True)
+@click.option("--via", "-f", type=str)
+@click.option("--to", "-t", type=str, default="commonmeta")
+@click.option("--style", "-s", type=str, default="apa")
+@click.option("--locale", "-l", type=str, default="en-US")
+@click.option("--prefix", type=str)
+@click.option("--depositor", type=str)
+@click.option("--email", type=str)
+@click.option("--registrant", type=str)
+@click.option("--login_id", type=str)
+@click.option("--login_passwd", type=str)
+@click.option("--host", type=str)
+@click.option("--token", type=str)
+@click.option("--legacy-key", type=str)
+@click.option("--file", type=str)
+@click.option("--show-errors/--no-errors", type=bool, show_default=True, default=False)
+@click.option("--show-timer/--no-timer", type=bool, show_default=True, default=False)
+def push(
+    string,
+    via,
+    to,
+    style,
+    locale,
+    prefix,
+    depositor,
+    email,
+    registrant,
+    login_id,
+    login_passwd,
+    host,
+    token,
+    legacy_key,
+    file,
     show_errors,
     show_timer,
 ):
@@ -98,18 +197,22 @@ def list(
     metadata_list = MetadataList(
         string,
         via=via,
+        file=file,
         depositor=depositor,
         email=email,
         registrant=registrant,
+        login_id=login_id,
+        login_passwd=login_passwd,
+        host=host,
+        token=token,
         prefix=prefix,
-        filename=filename,
-        jsonlines=jsonlines,
     )
     end = time.time()
     runtime = end - start
     if show_errors and not metadata_list.is_valid:
         raise click.ClickException(str(metadata_list.errors))
-    click.echo(metadata_list.write(to=to, style=style, locale=locale))
+
+    click.echo(metadata_list.push(to=to, style=style, locale=locale))
     if show_errors and len(metadata_list.write_errors) > 0:
         raise click.ClickException(str(metadata_list.write_errors))
     if show_timer:
@@ -167,29 +270,6 @@ def decode(doi):
     click.echo(output)
 
 
-@cli.command()
-@click.argument("id", type=str, required=True)
-def encode_by_id(id):
-    post = get_json_feed_item_uuid(id)
-    prefix = py_.get(post, "blog.prefix")
-    if validate_prefix(prefix) is None:
-        return None
-    output = encode_doi(prefix)
-    click.echo(output)
-
-
-@cli.command()
-@click.argument("filter", type=str, required=True, default="unregistered")
-@click.option("--id", type=str)
-def json_feed(filter, id=None):
-    if filter == "blog_slug" and id is not None:
-        post = get_json_feed_item_uuid(id)
-        output = py_.get(post, "blog.slug", "no slug found")
-    else:
-        output = "no filter specified"
-    click.echo(output)
-
-
 @cli.command()
 @click.argument("id", type=str, required=True)
 @click.option("--api-key", "-k", type=str, required=True)

{commonmeta_py-0.107 → commonmeta_py-0.108}/commonmeta/constants.py

@@ -190,6 +190,7 @@ CM_TO_CR_TRANSLATIONS = {
     "JournalIssue": "JournalIssue",
     "JournalVolume": "JournalVolume",
     "Journal": "Journal",
+    "PeerReview": "PeerReview",
     "ProceedingsArticle": "ProceedingsArticle",
     "ProceedingsSeries": "ProceedingsSeries",
     "Proceedings": "Proceedings",
@@ -698,3 +699,22 @@ ROR_TO_CROSSREF_FUNDER_ID_TRANSLATIONS = {
     "https://ror.org/00yjd3n13": "https://doi.org/10.13039/501100001711",
     "https://ror.org/04wfr2810": "https://doi.org/10.13039/501100003043",
 }
+
+COMMUNITY_TRANSLATIONS = {
+    "ai": "artificialintelligence",
+    "llms": "artificialintelligence",
+    "book%20review": "bookreview",
+    "bjps%20review%20of%20books": "bookreview",
+    "books": "bookreview",
+    "nachrichten": "news",
+    "opencitations": "researchassessment",
+    "papers": "researchblogging",
+    "urheberrecht": "copyright",
+    "workshop": "events",
+    "veranstaltungen": "events",
+    "veranstaltungshinweise": "events",
+    "asapbio": "preprints",
+    "biorxiv": "preprints",
+    "runiverse": "r",
+    "bericht": "report",
+}