commonmeta-py 0.106__py3-none-any.whl → 0.108__py3-none-any.whl

commonmeta/__init__.py

@@ -10,7 +10,7 @@ commonmeta-py is a Python library to convert scholarly metadata
 """
 
 __title__ = "commonmeta-py"
-__version__ = "0.106"
+__version__ = "0.108"
 __author__ = "Martin Fenner"
 __license__ = "MIT"
 
@@ -54,6 +54,14 @@ from .doi_utils import (
     validate_doi,
     validate_prefix,
 )
+from .file_utils import (
+    download_file,
+    read_file,
+    read_gz_file,
+    read_zip_file,
+    uncompress_content,
+    unzip_content,
+)
 from .metadata import Metadata, MetadataList
 from .readers import (
     cff_reader,
@@ -63,7 +71,7 @@ from .readers import (
     datacite_reader,
     datacite_xml_reader,
     inveniordm_reader,
-    json_feed_reader,
+    jsonfeed_reader,
     kbase_reader,
     openalex_reader,
     ris_reader,
@@ -75,7 +83,7 @@ from .utils import (
     extract_url,
     extract_urls,
     from_csl,
-    from_json_feed,
+    from_jsonfeed,
     from_schema_org,
     get_language,
     issn_as_url,
@@ -97,6 +105,7 @@ from .writers import (
     bibtex_writer,
     citation_writer,
     commonmeta_writer,
+    crossref_xml_writer,
     csl_writer,
     datacite_writer,
     ris_writer,

commonmeta/api_utils.py

@@ -7,8 +7,9 @@ import jwt
 import requests
 from furl import furl
 
+from commonmeta.readers.jsonfeed_reader import get_jsonfeed_uuid
+
 from .doi_utils import doi_as_url, validate_doi
-from .readers.json_feed_reader import get_json_feed_item_uuid
 
 
 def generate_ghost_token(key: str) -> str:
@@ -34,7 +35,7 @@ def update_ghost_post_via_api(
     """Update Ghost post via API"""
     # get post doi and url from Rogue Scholar API
     # post url is needed to find post via Ghost API
-    post = get_json_feed_item_uuid(_id)
+    post = get_jsonfeed_uuid(_id)
     if post.get("error", None):
         return post
     doi = validate_doi(post.get("doi", None))

commonmeta/base_utils.py

@@ -2,10 +2,13 @@
 
 import html
 import re
+import uuid
+from datetime import datetime
 from os import path
 from typing import Optional, Union
 
 import nh3
+import pydash as py_
 import xmltodict
 
 
@@ -67,8 +70,8 @@ def parse_attributes(
 
 
 def parse_xml(string: Optional[str], **kwargs) -> Optional[Union[dict, list]]:
-    """Parse XML into dict. Set default options, and options for Crossref XML"""
-    if string is None:
+    """Parse XML into dict using xmltodict. Set default options, and options for Crossref XML"""
+    if string is None or string == "{}":
         return None
     if path.exists(string):
         with open(string, encoding="utf-8") as file:
@@ -77,7 +80,7 @@ def parse_xml(string: Optional[str], **kwargs) -> Optional[Union[dict, list]]:
     if kwargs.get("dialect", None) == "crossref":
         # remove namespaces from xml
         namespaces = {
-            "http://www.crossref.org/schema/5.3.1": None,
+            "http://www.crossref.org/schema/5.4.0": None,
             "http://www.crossref.org/qrschema/3.0": None,
             "http://www.crossref.org/xschema/1.0": None,
             "http://www.crossref.org/xschema/1.1": None,
@@ -93,6 +96,7 @@ def parse_xml(string: Optional[str], **kwargs) -> Optional[Union[dict, list]]:
             "person_name",
             "organization",
             "titles",
+            "abstract",
             "item",
             "citation",
             "program",
@@ -105,6 +109,172 @@ def parse_xml(string: Optional[str], **kwargs) -> Optional[Union[dict, list]]:
     return xmltodict.parse(string, **kwargs)
 
 
+def unparse_xml(input: Optional[dict], **kwargs) -> str:
+    """Unparse (dump) dict into XML using xmltodict. Set default options, and options for Crossref XML"""
+    if input is None:
+        return None
+    if kwargs.get("dialect", None) == "crossref":
+        # Add additional logic for crossref dialect
+        # add body and root element as wrapping elements
+        type = next(iter(input))
+        attributes = input.get(type)
+        input.pop(type)
+
+        if type == "book":
+            book_metadata = py_.get(input, "book_metadata") or {}
+            input.pop("book_metadata")
+            book_metadata = {**book_metadata, **input}
+            input = {"book": {**attributes, "book_metadata": book_metadata}}
+        elif type == "database":
+            database_metadata = py_.get(input, "database_metadata") or {}
+            input.pop("database_metadata")
+            val = input.pop("publisher_item")
+            institution = input.pop("institution", None)
+            database_metadata = {**{"titles": val}, **database_metadata}
+            database_metadata["institution"] = institution or {}
+            component = input.pop("component", None)
+            input = {
+                "database": {
+                    **attributes,
+                    "database_metadata": database_metadata,
+                    "component_list": {"component": component | input},
+                }
+            }
+        elif type == "journal":
+            journal_metadata = py_.get(input, "journal_metadata") or {}
+            journal_issue = py_.get(input, "journal_issue") or {}
+            journal_article = py_.get(input, "journal_article") or {}
+            input.pop("journal_metadata")
+            input.pop("journal_issue")
+            input.pop("journal_article")
+            input = {
+                "journal": {
+                    "journal_metadata": journal_metadata,
+                    "journal_issue": journal_issue,
+                    "journal_article": journal_article | input,
+                }
+            }
+        elif type == "proceedings_article":
+            proceedings_metadata = py_.get(input, "proceedings_metadata") or {}
+            input.pop("proceedings_metadata")
+            input = {
+                "proceedings": {
+                    **attributes,
+                    "proceedings_metadata": proceedings_metadata,
+                    "conference_paper": input,
+                }
+            }
+        elif type == "sa_component":
+            component = py_.get(input, "component") or {}
+            input.pop("component")
+            input = {
+                "sa_component": {
+                    **attributes,
+                    "component_list": {"component": component | input},
+                }
+            }
+        else:
+            input = {type: attributes | input}
+
+        doi_batch = {
+            "@xmlns": "http://www.crossref.org/schema/5.4.0",
+            "@version": "5.4.0",
+            "head": get_crossref_xml_head(input),
+            "body": input,
+        }
+        input = {"doi_batch": doi_batch}
+    kwargs["pretty"] = True
+    kwargs["indent"] = "  "
+    kwargs.pop("dialect", None)
+    return xmltodict.unparse(input, **kwargs)
+
+
+def unparse_xml_list(input: Optional[list], **kwargs) -> str:
+    """Unparse (dump) list into XML using xmltodict. Set default options, and options for Crossref XML"""
+    if input is None:
+        return None
+    if kwargs.get("dialect", None) == "crossref":
+        # Add additional logic for crossref dialect
+        # add body and root element as wrapping elements
+
+        # Group items by type with minimal grouping
+        items_by_type = {}
+
+        for item in wrap(input):
+            type = next(iter(item))
+            attributes = item.get(type)
+            item.pop(type)
+
+            # handle nested book_metadata and journal structure as in unparse_xml
+            if type == "book":
+                book_metadata = py_.get(item, "book_metadata") or {}
+                item.pop("book_metadata")
+                book_metadata = {**book_metadata, **item}
+                item = {"book": {**attributes, "book_metadata": book_metadata}}
+            elif type == "database":
+                database_metadata = py_.get(item, "database_metadata") or {}
+                item.pop("database_metadata")
+                database_metadata = {**database_metadata, **item}
+                item = {
+                    "database": {**attributes, "database_metadata": database_metadata}
+                }
+            elif type == "journal":
+                journal_metadata = py_.get(item, "journal_metadata") or {}
+                journal_issue = py_.get(item, "journal_issue") or {}
+                journal_article = py_.get(item, "journal_article") or {}
+                item.pop("journal_metadata")
+                item.pop("journal_issue")
+                item.pop("journal_article")
+                item = {
+                    "journal": {
+                        "journal_metadata": journal_metadata,
+                        "journal_issue": journal_issue,
+                        "journal_article": journal_article | item,
+                    }
+                }
+            elif type == "sa_component":
+                component = py_.get(input, "component") or {}
+                item.pop("component")
+                item = {
+                    "sa_component": {
+                        **attributes,
+                        "component_list": {"component": component | item},
+                    }
+                }
+            else:
+                item = {type: attributes | item}
+
+            # Add item to appropriate type bucket
+            if type not in items_by_type:
+                items_by_type[type] = []
+            items_by_type[type].append(item[type])
+
+        # Create the final structure with body containing all grouped items
+        body_content = {}
+        for type_key, items in items_by_type.items():
+            if len(items) == 1:
+                body_content[type_key] = items[0]  # Use single item without array
+            else:
+                body_content[type_key] = items  # Use array when multiple items
+        head = kwargs["head"] or {}
+        doi_batch = {
+            "@xmlns": "http://www.crossref.org/schema/5.4.0",
+            "@xmlns:ai": "http://www.crossref.org/AccessIndicators.xsd",
+            "@xmlns:rel": "http://www.crossref.org/relations.xsd",
+            "@xmlns:fr": "http://www.crossref.org/fundref.xsd",
+            "@version": "5.4.0",
+            "head": get_crossref_xml_head(head),
+            "body": body_content,
+        }
+        output = {"doi_batch": doi_batch}
+
+    kwargs["pretty"] = True
+    kwargs["indent"] = "  "
+    kwargs.pop("dialect", None)
+    kwargs.pop("head", None)
+    return xmltodict.unparse(output, **kwargs)
+
+
 def sanitize(text: str, **kwargs) -> str:
     """Sanitize text"""
     # default whitelisted HTML tags
@@ -122,3 +292,16 @@ def sanitize(text: str, **kwargs) -> str:
     string = nh3.clean(text, tags=tags, attributes=attributes, link_rel=None)
     # remove excessive internal whitespace
     return " ".join(re.split(r"\s+", string, flags=re.UNICODE))
+
+
+def get_crossref_xml_head(metadata: dict) -> dict:
+    """Get head element for Crossref XML"""
+    return {
+        "doi_batch_id": str(uuid.uuid4()),
+        "timestamp": datetime.now().strftime("%Y%m%d%H%M%S"),
+        "depositor": {
+            "depositor_name": metadata.get("depositor", None) or "test",
+            "email_address": metadata.get("email", None) or "info@example.org",
+        },
+        "registrant": metadata.get("registrant", None) or "test",
+    }

commonmeta/cli.py

@@ -2,16 +2,12 @@ import time
 
 import click
 import orjson as json
-import pydash as py_
 
 from commonmeta import Metadata, MetadataList  # __version__
 from commonmeta.api_utils import update_ghost_post_via_api
 from commonmeta.doi_utils import decode_doi, encode_doi, validate_prefix
 from commonmeta.readers.crossref_reader import get_random_crossref_id
 from commonmeta.readers.datacite_reader import get_random_datacite_id
-from commonmeta.readers.json_feed_reader import (
-    get_json_feed_item_uuid,
-)
 from commonmeta.readers.openalex_reader import get_random_openalex_id
 
 
@@ -46,6 +42,49 @@ def convert(
     email,
     registrant,
     show_errors,
+):
+    metadata = Metadata(input, via=via, doi=doi, prefix=prefix)
+    if show_errors and not metadata.is_valid:
+        raise click.ClickException(str(metadata.errors))
+
+    click.echo(
+        metadata.write(
+            to=to,
+            style=style,
+            locale=locale,
+            depositor=depositor,
+            email=email,
+            registrant=registrant,
+        )
+    )
+    if show_errors and metadata.write_errors:
+        raise click.ClickException(str(metadata.write_errors))
+
+
+@cli.command()
+@click.argument("input", type=str, required=True)
+@click.option("--via", "-f", type=str, default=None)
+@click.option("--to", "-t", type=str, default="commonmeta")
+@click.option("--style", "-s", type=str, default="apa")
+@click.option("--locale", "-l", type=str, default="en-US")
+@click.option("--doi", type=str)
+@click.option("--prefix", type=str)
+@click.option("--depositor", type=str)
+@click.option("--email", type=str)
+@click.option("--registrant", type=str)
+@click.option("--show-errors/--no-errors", type=bool, show_default=True, default=False)
+def put(
+    input,
+    via,
+    to,
+    style,
+    locale,
+    doi,
+    prefix,
+    depositor,
+    email,
+    registrant,
+    show_errors,
 ):
     metadata = Metadata(input, via=via, doi=doi, prefix=prefix)
     if show_errors and not metadata.is_valid:
@@ -75,8 +114,7 @@ def convert(
 @click.option("--depositor", type=str)
 @click.option("--email", type=str)
 @click.option("--registrant", type=str)
-@click.option("--filename", type=str)
-@click.option("--jsonlines/--no-jsonlines", type=bool, show_default=True, default=False)
+@click.option("--file", type=str)
 @click.option("--show-errors/--no-errors", type=bool, show_default=True, default=False)
 @click.option("--show-timer/--no-timer", type=bool, show_default=True, default=False)
 def list(
@@ -89,8 +127,69 @@ def list(
     depositor,
     email,
     registrant,
-    filename,
-    jsonlines,
+    file,
+    show_errors,
+    show_timer,
+):
+    start = time.time()
+    metadata_list = MetadataList(
+        string,
+        via=via,
+        file=file,
+        depositor=depositor,
+        email=email,
+        registrant=registrant,
+        prefix=prefix,
+    )
+    end = time.time()
+    runtime = end - start
+    if show_errors and not metadata_list.is_valid:
+        raise click.ClickException(str(metadata_list.errors))
+    if file:
+        metadata_list.write(to=to, style=style, locale=locale)
+    else:
+        click.echo(metadata_list.write(to=to, style=style, locale=locale))
+
+    if show_errors and len(metadata_list.write_errors) > 0:
+        raise click.ClickException(str(metadata_list.write_errors))
+    if show_timer:
+        click.echo(f"Runtime: {runtime:.2f} seconds")
+
+
+@cli.command()
+@click.argument("string", type=str, required=True)
+@click.option("--via", "-f", type=str)
+@click.option("--to", "-t", type=str, default="commonmeta")
+@click.option("--style", "-s", type=str, default="apa")
+@click.option("--locale", "-l", type=str, default="en-US")
+@click.option("--prefix", type=str)
+@click.option("--depositor", type=str)
+@click.option("--email", type=str)
+@click.option("--registrant", type=str)
+@click.option("--login_id", type=str)
+@click.option("--login_passwd", type=str)
+@click.option("--host", type=str)
+@click.option("--token", type=str)
+@click.option("--legacy-key", type=str)
+@click.option("--file", type=str)
+@click.option("--show-errors/--no-errors", type=bool, show_default=True, default=False)
+@click.option("--show-timer/--no-timer", type=bool, show_default=True, default=False)
+def push(
+    string,
+    via,
+    to,
+    style,
+    locale,
+    prefix,
+    depositor,
+    email,
+    registrant,
+    login_id,
+    login_passwd,
+    host,
+    token,
+    legacy_key,
+    file,
     show_errors,
     show_timer,
 ):
@@ -98,18 +197,22 @@ def list(
     metadata_list = MetadataList(
         string,
         via=via,
+        file=file,
         depositor=depositor,
         email=email,
         registrant=registrant,
+        login_id=login_id,
+        login_passwd=login_passwd,
+        host=host,
+        token=token,
         prefix=prefix,
-        filename=filename,
-        jsonlines=jsonlines,
     )
     end = time.time()
     runtime = end - start
     if show_errors and not metadata_list.is_valid:
         raise click.ClickException(str(metadata_list.errors))
-    click.echo(metadata_list.write(to=to, style=style, locale=locale))
+
+    click.echo(metadata_list.push(to=to, style=style, locale=locale))
     if show_errors and len(metadata_list.write_errors) > 0:
         raise click.ClickException(str(metadata_list.write_errors))
     if show_timer:
@@ -167,29 +270,6 @@ def decode(doi):
     click.echo(output)
 
 
-@cli.command()
-@click.argument("id", type=str, required=True)
-def encode_by_id(id):
-    post = get_json_feed_item_uuid(id)
-    prefix = py_.get(post, "blog.prefix")
-    if validate_prefix(prefix) is None:
-        return None
-    output = encode_doi(prefix)
-    click.echo(output)
-
-
-@cli.command()
-@click.argument("filter", type=str, required=True, default="unregistered")
-@click.option("--id", type=str)
-def json_feed(filter, id=None):
-    if filter == "blog_slug" and id is not None:
-        post = get_json_feed_item_uuid(id)
-        output = py_.get(post, "blog.slug", "no slug found")
-    else:
-        output = "no filter specified"
-    click.echo(output)
-
-
 @cli.command()
 @click.argument("id", type=str, required=True)
 @click.option("--api-key", "-k", type=str, required=True)

commonmeta/constants.py

@@ -190,6 +190,7 @@ CM_TO_CR_TRANSLATIONS = {
     "JournalIssue": "JournalIssue",
     "JournalVolume": "JournalVolume",
     "Journal": "Journal",
+    "PeerReview": "PeerReview",
     "ProceedingsArticle": "ProceedingsArticle",
     "ProceedingsSeries": "ProceedingsSeries",
     "Proceedings": "Proceedings",
@@ -698,3 +699,22 @@ ROR_TO_CROSSREF_FUNDER_ID_TRANSLATIONS = {
     "https://ror.org/00yjd3n13": "https://doi.org/10.13039/501100001711",
     "https://ror.org/04wfr2810": "https://doi.org/10.13039/501100003043",
 }
+
+COMMUNITY_TRANSLATIONS = {
+    "ai": "artificialintelligence",
+    "llms": "artificialintelligence",
+    "book%20review": "bookreview",
+    "bjps%20review%20of%20books": "bookreview",
+    "books": "bookreview",
+    "nachrichten": "news",
+    "opencitations": "researchassessment",
+    "papers": "researchblogging",
+    "urheberrecht": "copyright",
+    "workshop": "events",
+    "veranstaltungen": "events",
+    "veranstaltungshinweise": "events",
+    "asapbio": "preprints",
+    "biorxiv": "preprints",
+    "runiverse": "r",
+    "bericht": "report",
+}

commonmeta/file_utils.py

@@ -0,0 +1,112 @@
+"""File utils module for commonmeta-py"""
+
+import gzip
+import io
+import zipfile
+from pathlib import Path
+from typing import Optional, Union
+
+import requests
+
+
+def read_file(filename: str) -> bytes:
+    with open(filename, "rb") as f:
+        return f.read()
+
+
+def uncompress_content(input: bytes) -> bytes:
+    with gzip.GzipFile(fileobj=io.BytesIO(input)) as gz:
+        return gz.read()
+
+
+def unzip_content(input: bytes, filename: Optional[str] = None) -> bytes:
+    output = b""
+    with zipfile.ZipFile(io.BytesIO(input)) as zf:
+        for info in zf.infolist():
+            if filename and info.filename != filename:
+                continue
+            with zf.open(info) as file:
+                output += file.read()
+    return output
+
+
+def read_gz_file(filename: str) -> bytes:
+    input_bytes = read_file(filename)
+    return uncompress_content(input_bytes)
+
+
+def read_zip_file(filename: str, name: Optional[str] = None) -> bytes:
+    input_bytes = read_file(filename)
+    return unzip_content(input_bytes, name)
+
+
+def download_file(url: str) -> bytes:
+    resp = requests.get(url, stream=True)
+    resp.raise_for_status()
+    return resp.content
+    # # Progress bar
+    # total = int(resp.headers.get("content-length", 0))
+
+    # buf = io.BytesIO()
+    # with tqdm(total=total, unit="B", unit_scale=True, desc="downloading") as bar:
+    #     for chunk in resp.iter_content(chunk_size=8192):
+    #         if chunk:
+    #             buf.write(chunk)
+    #             bar.update(len(chunk))
+    # return buf.getvalue()
+
+
+def write_file(filename: str, output: bytes) -> None:
+    with open(filename, "xb") as f:
+        f.write(output)
+
+
+def write_gz_file(filename: str, output: bytes) -> None:
+    with gzip.open(filename, "xb") as gzfile:
+        gzfile.write(output)
+
+
+def write_zip_file(filename: str, output: bytes) -> None:
+    path = Path(filename)
+    with zipfile.ZipFile(filename, "w", zipfile.ZIP_DEFLATED) as zipf:
+        zipf.writestr(path.name, output)
+
+
+def get_extension(filename: str) -> tuple[str, str, Optional[str]]:
+    """Extract extension and compression from filename"""
+    extension = Path(filename).suffix
+    if extension == ".gz":
+        compress = ".gz"
+        filename = filename[:-3]
+        extension = Path(filename).suffix
+    elif extension == ".zip":
+        compress = ".zip"
+        filename = filename[:-4]
+        extension = Path(filename).suffix
+    elif extension == "":
+        compress = None
+        filename = filename + ".json"
+        extension = ".json"
+    else:
+        compress = None
+    return filename, extension, compress
+
+
+def write_output(filename: str, input: Union[bytes, str], ext: list[str]) -> None:
+    """Write output to file with supported extension"""
+
+    # Convert string to bytes if necessary
+    if isinstance(input, str):
+        input = input.encode("utf-8")
+
+    filename, extension, compress = get_extension(filename)
+    if extension not in ext:
+        raise ValueError(
+            f"File format not supported. Please provide a filename with {ext} extension."
+        )
+    if compress == ".gz":
+        write_gz_file(filename + compress, input)
+    elif compress == ".zip":
+        write_zip_file(filename + compress, input)
+    else:
+        write_file(filename, input)