somef_core 0.0.1__py3-none-any.whl

somef_core/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+__version__ = "0.0.1"

somef_core/__main__.py

@@ -0,0 +1,293 @@
+# -*- coding: utf-8 -*-
+
+import click
+from click_option_group import optgroup, RequiredMutuallyExclusiveOptionGroup, RequiredAnyOptionGroup
+
+import logging
+from . import configuration
+from .utils import constants
+from . import __version__
+
+
+class URLParamType(click.types.StringParamType):
+    name = "url"
+
+
+@click.group(context_settings={'help_option_names': ['-h', '--help']})
+@click.version_option(__version__)
+def cli():
+    click.echo("SOftware Metadata Extraction Framework (SOMEF) Command Line Interface")
+
+
+@cli.group(help="Configure repository credentials and classifiers file path", invoke_without_command=True)
+@click.option('-a', '--auto', help="Automatically configure SOMEF", is_flag=True, default=False)
+@click.option('-b', '--base_uri', type=URLParamType(), help="Base URI for somef transformations",
+              default=constants.CONF_DEFAULT_BASE_URI)
+@click.pass_context
+def configure(ctx, auto, base_uri):
+    if ctx.invoked_subcommand is not None:
+        return  
+    if auto:
+        click.echo(
+            "Configuring SOMEF automatically. To assign credentials edit the configuration file or run "
+            "the interactive mode")
+        configuration.configure()
+    elif base_uri is not constants.CONF_DEFAULT_BASE_URI:
+        configuration.update_base_uri(base_uri)
+    else:
+        # authorization = click.prompt("Authorization", default="")
+        github_authorization = click.prompt("GitHub Authorization token (leave blank to skip)", default="")
+        gitlab_authorization = click.prompt(
+            "GitLab Authorization token (leave blank to skip; works for gitlab.com and self-hosted instances)",
+            default="",
+        )
+        codeberg_authorization = click.prompt("Codeberg Authorization token (leave blank to skip)", default="")
+        
+        bitbucket_authorization = click.prompt("Bitbucket Authorization token (leave blank to skip)", default="")
+        bitbucket_email = click.prompt("Bitbucket Atlassian account email (for API token auth)", default="")
+
+        # description = click.prompt(... model file ...)
+        # invocation = click.prompt(...)
+        # installation = click.prompt(...)
+        # citation = click.prompt(...)
+        base_uri = click.prompt("Base URI for RDF generation", default=base_uri)
+        download_limit = click.prompt("Download size limit in MB", 
+                               default=constants.SIZE_DOWNLOAD_LIMIT_MB, type=int)
+        # configuration.configure()
+        configuration.configure(github_authorization, gitlab_authorization, codeberg_authorization, bitbucket_authorization, bitbucket_email, description=None, invocation=None, installation=None, citation=None, base_uri=base_uri, download_limit_mb=download_limit)
+        
+    click.secho(f"Success", fg="green")
+
+@configure.command(
+    help="Test the configured API tokens")
+def test():
+    """Checks the configured tokens against each provider."""
+    results = configuration.test_configuration_tokens() 
+    for label, res in results.items():
+        if res["ok"] is None:
+            click.secho(f"{label}: {res['message']}", fg="yellow")
+        elif res["ok"]:
+            click.secho(f"{label}: {res['message']}", fg="green")
+        else:
+            click.secho(f"{label}: {res['message']}", fg="red")
+    if any(not res['ok'] for res in results.values()):
+        raise SystemExit(1)
+    
+@cli.command(help="Running SOMEF Command Line Interface")
+@click.option(
+    "--threshold",
+    "-t",
+    type=float,
+    help="Threshold to classify the text",
+    required=True,
+    default=0.8,
+)
+@click.option(
+    "--ignore_classifiers",
+    "-ic",
+    is_flag=True,
+    default=False,
+    help="Flag to ignore running the classifiers (by default False)"
+)
+@click.option(
+    "--ignore_github_metadata",
+    "-igm",
+    is_flag=True,
+    default=False,
+    help="Flag to ignore Github Metadata (by default False)"
+)
+@click.option(
+    "--readme_only",
+    "-ro",
+    is_flag=True,
+    default=False,
+    help="Flag to retrieve only the README.md file from the Github/Gitlab Repository URL. If such file does not exist, "
+         "no metadata will be retrieved (by default False)"
+)
+@optgroup.group('Input', cls=RequiredMutuallyExclusiveOptionGroup)
+@optgroup.option(
+    "--repo_url",
+    "-r",
+    type=URLParamType(),
+    help="Github/Gitlab Repository URL",
+)
+@optgroup.option(
+    "--doc_src",
+    "-d",
+    type=click.Path(exists=True),
+    help="Path to the README file source"
+)
+@optgroup.option(
+    "--local_repo",
+    "-l",
+    type=click.Path(exists=True),
+    help="Path to local repository"
+)
+@optgroup.option(
+    "--in_file",
+    "-i",
+    type=click.Path(exists=True),
+    help=""""A file of newline separated links to GitHub/Gitlab repositories to process in bulk. Each repository will be 
+         stored in a different file called $out_$url.json where $out is the name selected as out file and $url is the
+         url of the target repository (url encoded)"""
+)
+@optgroup.group('Output', cls=RequiredAnyOptionGroup)
+@optgroup.option(
+    "--output",
+    "-o",
+    type=click.Path(),
+    help="Path to the output file. If supplied, the output will be in JSON",
+)
+@optgroup.option(
+    "--codemeta_out",
+    "-c",
+    type=click.Path(),
+    help="Path to an output codemeta file"
+)
+@optgroup.option(
+    "--google_codemeta_out",
+    "-gc",
+    type=click.Path(),
+    help="Path to an output Google-compliant codemeta file"
+)
+@optgroup.option(
+    "--graph_out",
+    "-g",
+    type=click.Path(),
+    help="""Path to the output Knowledge Graph export file. If supplied, the output will be a Knowledge Graph,
+            in the format given in the --format option chosen (turtle, json-ld)"""
+)
+@click.option(
+    "--graph_format",
+    "-f",
+    type=click.Choice(["turtle", "json-ld"]),
+    default="turtle",
+    help="""If the --graph_out option is given, this is the format that the graph will be stored in"""
+)
+@click.option(
+    "--pretty",
+    "-p",
+    is_flag=True,
+    default=False,
+    help="""Pretty print the JSON output file so that it is easy to compare to another JSON output file."""
+)
+@click.option(
+    "--missing",
+    "-m",
+    is_flag=True,
+    default=False,
+    help="""The JSON will include a field missing_categories to report with the missing metadata fields SOMEF was not 
+    able to find. """
+)
+@click.option(
+    "--keep_tmp",
+    "-kt",
+    type=click.Path(),
+    help="""SOMEF will NOT delete the temporary folder where files are stored for analysis. Files will be stored at the
+    desired path"""
+)
+@click.option(
+    "--ignore_test_folder",
+    "-itf",
+    is_flag=True,
+    default=True,
+    help="""SOMEF will ignore the contents of all files within folders named test (True by default)"""
+)
+@click.option(
+    "--requirements_all",
+    "-all",
+    is_flag=True,
+    default=False,
+    help="Export all detected requirements, including text and libraries (default)."
+)
+@click.option(
+    "--requirements_v",
+    "-v",
+    is_flag=True,
+    default=False,
+    help="Export only requirements from structured sources (pom.xml, requirements.txt, etc.)"
+)
+@click.option(
+    "--reconcile_authors",
+    "-ra",
+    is_flag=True,
+    default=False,
+    help="""SOMEF will extract additional information from certain files like CODEOWNERS, etc."""
+)
+@click.option(
+    "--branch",
+    "-b",
+    type=str,
+    default=None,
+    help="Branch of the repository to analyze. Overrides the default branch."
+)
+@click.option(
+    "--tag",
+    type=str,
+    default=None,
+    help="Tag of the repository to analyze. Incompatible with --branch"
+)
+@click.option(
+    "--commit",
+    type=str,
+    default=None,
+    help="Commit SHA of the repository to analyze. Incompatible with --branch and --tag"
+)
+@click.option(
+    "--enrich",
+    "-e",
+    is_flag=True,
+    default=False,
+    help="Enrich metadata with external APIs (OpenAlex, OpenAIRE, Zenodo)"
+  )
+@click.option(
+    "--github-token",
+    type=str,
+    default=None,
+    help="GitHub personal access token (if invalid, stored config is used instead)"
+)
+@click.option(
+    "--gitlab-token",
+    "-gt",
+    type=str,
+    default=None,
+    help="GitLab personal access token (if invalid, stored config is used instead)"
+)
+@click.option(
+    "--codeberg-token",
+    type=str,
+    default=None,
+    help="Codeberg personal access token (if invalid, stored config is used instead)"
+)
+@click.option(
+    "--bitbucket-token",
+    type=str,
+    default=None,
+    help="Bitbucket API token (if invalid, stored config is used instead)"
+)
+@click.option(
+    "--bitbucket-email",
+    type=str,
+    default=None,
+    help="Bitbucket Atlassian account email (required with --bitbucket-token)"
+)
+@click.option(
+    "--download-limit",
+    type=int,
+    default=None,
+    help="Download size limit in MB (overrides config file value)"
+)
+
+def describe(requirements_v, requirements_all, **kwargs):
+    # import so missing packages get installed when appropriate
+    if requirements_v:
+        kwargs["requirements_mode"] = "v"
+    elif requirements_all:
+        kwargs["requirements_mode"] = "all"
+    else:
+        kwargs["requirements_mode"] = "all" 
+
+    from . import somef_cli
+    somef_cli.run_cli(**kwargs)
+    click.secho(f"Success", fg="green")
+    

somef_core/configuration.py

@@ -0,0 +1,177 @@
+import os
+from pathlib import Path
+import json
+import sys
+import logging
+import base64
+import requests
+import nltk
+from .utils import constants
+
+path = Path(__file__).parent.absolute()
+# default_description = os.path.join(str(path), "models", "description.p")
+# default_invocation = os.path.join(str(path), "models", "invocation.p")
+# default_installation = os.path.join(str(path), "models", "installation.p")
+# default_citation = os.path.join(str(path), "models", "citation.p")
+
+
+def get_configuration_file():
+    """
+    Function that retrieves the configuration file
+    Returns
+    -------
+    The configuration object in JSON format
+    """
+    credentials_file = Path(
+        os.getenv("SOMEF_CONFIGURATION_FILE", '~/.somef_core/config.json')
+    ).expanduser()
+    if credentials_file.exists():
+        with credentials_file.open("r") as fh:
+            file_paths = json.load(fh)
+        if constants.CONF_SIMILARITY_THRESHOLD not in file_paths:
+            file_paths[constants.CONF_SIMILARITY_THRESHOLD] = constants.CONF_DEFAULT_SIMILARITY_THRESHOLD
+        if constants.CONF_DOWNLOAD_LIMIT_MB not in file_paths:
+            file_paths[constants.CONF_DOWNLOAD_LIMIT_MB] = constants.SIZE_DOWNLOAD_LIMIT_MB
+    else:
+        sys.exit("Error: Please provide a config.json file or run somef configure.")
+    return file_paths
+
+
+def update_base_uri(base_uri):
+    credentials_file = Path(
+        os.getenv("SOMEF_CONFIGURATION_FILE", constants.__DEFAULT_SOMEF_CONFIGURATION_FILE__)
+    ).expanduser()
+    os.makedirs(str(credentials_file.parent), exist_ok=True)
+
+    if credentials_file.exists():
+        with credentials_file.open("r") as fh:
+            data = json.load(fh)
+            data[constants.CONF_BASE_URI] = base_uri
+
+        with credentials_file.open("w") as fh:
+            credentials_file.parent.chmod(0o700)
+            credentials_file.chmod(0o600)
+            json.dump(data, fh)
+
+
+def configure(
+        # authorization="",
+        github_authorization="",
+        gitlab_authorization="",
+        codeberg_authorization="",
+        bitbucket_authorization="",
+        bitbucket_email="",
+        description=None,
+        invocation=None,
+        installation=None,
+        citation=None,
+        base_uri=constants.CONF_DEFAULT_BASE_URI,
+        similarity_threshold=constants.CONF_DEFAULT_SIMILARITY_THRESHOLD,
+        download_limit_mb=constants.SIZE_DOWNLOAD_LIMIT_MB):
+    
+    """ Function to configure the main program"""
+
+    nltk.download('wordnet')
+    nltk.download('omw-1.4')
+    nltk.download('punkt')
+    nltk.download('punkt_tab')
+    nltk.download('stopwords')
+
+    credentials_file = Path(
+        os.getenv("SOMEF_CONFIGURATION_FILE", constants.__DEFAULT_SOMEF_CONFIGURATION_FILE__)
+    ).expanduser()
+    os.makedirs(str(credentials_file.parent), exist_ok=True)
+
+    # if credentials_file.exists():
+    #     with credentials_file.open("r") as fh:
+    #         data = json.load(fh)
+    # else:
+    data = {
+        # constants.CONF_AUTHORIZATION: "token " + authorization,
+        constants.CONF_BASE_URI: base_uri,
+        constants.CONF_SIMILARITY_THRESHOLD: similarity_threshold,
+        constants.CONF_DOWNLOAD_LIMIT_MB: download_limit_mb
+    }
+
+    # if data[constants.CONF_AUTHORIZATION] == "token ":
+    #     del data[constants.CONF_AUTHORIZATION]
+
+    if github_authorization:
+        data[constants.CONF_GITHUB_AUTHORIZATION] = "token " + github_authorization
+
+    if gitlab_authorization:
+        token = gitlab_authorization
+        if not token.lower().startswith("bearer "):
+            token = "Bearer " + token
+        data[constants.CONF_GITLAB_AUTHORIZATION] = token
+
+    if codeberg_authorization:
+        token = codeberg_authorization
+        if not token.lower().startswith("token "):
+            token = "token " + token
+        data[constants.CONF_CODEBERG_AUTHORIZATION] = token
+
+
+    if bitbucket_authorization:
+        token = bitbucket_authorization
+        email = bitbucket_email  
+        if not token.lower().startswith("basic "):
+            raw = f"{email}:{token}"
+            token = "Basic " + base64.b64encode(raw.encode()).decode()
+        data[constants.CONF_BITBUCKET_AUTHORIZATION] = token
+
+    with credentials_file.open("w") as fh:
+        credentials_file.parent.chmod(0o700)
+        credentials_file.chmod(0o600)
+        json.dump(data, fh)
+        logging.info("Configuration file saved at "+os.path.dirname(credentials_file))
+
+
+def test_configuration_tokens():
+    """
+    Checks the API tokens stored in the configuration file against each
+    provider without running SOMEF.
+
+    Returns
+    -------
+    dict
+        Mapping of provider name to {"ok": bool, "message": str}. Providers
+        without a configured token are omitted.
+    """
+
+    file_paths = get_configuration_file()
+    providers = [
+        ("GitHub",    constants.CONF_GITHUB_AUTHORIZATION,    "https://api.github.com/user"),
+        ("GitLab",    constants.CONF_GITLAB_AUTHORIZATION,    "https://gitlab.com/api/v4/user"),
+        ("Codeberg",  constants.CONF_CODEBERG_AUTHORIZATION,  "https://codeberg.org/api/v1/user"),
+        ("Bitbucket", constants.CONF_BITBUCKET_AUTHORIZATION, "https://api.bitbucket.org/2.0/user"),
+    ]
+    results = {}
+    for label, key, url in providers:
+        if key not in file_paths:
+            results[label] = {"ok": None, "message": "token not configured"}
+            continue
+
+        stored = file_paths[key]
+        if label == "Bitbucket" and not stored.lower().startswith("basic "):
+            results[label] = {
+                "ok": False,
+                "message": "Bitbucket token has an incorrect format (expected a 'Basic ' prefix). "
+                           "Run 'somef configure' to set it correctly.",
+            }
+            continue
+
+        try:
+            resp = requests.get(url, headers={constants.PROP_AUTHORIZATION: file_paths[key]}, timeout=10)
+        except requests.RequestException as e:
+            results[label] = {"ok": False, "message": f"Could not reach the api: {e}"}
+            continue
+        if resp.status_code == 200:
+            results[label] = {"ok": True, "message": "token valid"}
+        elif resp.status_code == 401:
+            results[label] = {"ok": False, "message": "token invalid (401)"}
+        elif resp.status_code == 403:
+            results[label] = {"ok": True, "message": "token valid but with limited permissions (403)"}
+        else:
+            results[label] = {"ok": False, "message": f"unexpected response ({resp.status_code})"}
+    return results

somef_core/export/google_codemeta_export.py

@@ -0,0 +1,171 @@
+import json
+import copy
+from . import json_export
+from ..utils import constants
+
+def save_google_codemeta_output(repo_data, output_path, pretty=False, requirements_mode="all"):
+    """
+    Generate a Google-compliant Codemeta JSON-LD file from repo_data.
+    """
+
+    # Generate base codemeta using SOMEF. It is basically a codemeta file with a few changes.
+    # This way we have the core in the codemeta export and changes about content go just in the first method.
+
+    json_export.save_codemeta_output(
+        repo_data,
+        output_path,
+        pretty=pretty,
+        requirements_mode=requirements_mode
+    )
+
+    # Load the generated codemeta
+    with open(output_path, "r") as f:
+        codemeta = json.load(f)
+
+    # Apply google-compliant transformations
+    codemeta = make_google_compliant(codemeta)
+
+    # Overwrite the same file (no tmp file)
+    with open(output_path, "w") as f:
+        if pretty:
+            json.dump(codemeta, f, indent=2, sort_keys=True)
+        else:
+            json.dump(codemeta, f)
+
+
+# SCHEMA_ORG_PROPERTIES = { 
+#     "@type", 
+#     "name", 
+#     "description", 
+#     "author", 
+#     "keywords",
+#     "license",
+#     "url",
+#     "identifier",
+#     "programmingLanguage",
+#     "releaseNotes",
+#     "releaseDate"
+#     }
+
+def make_google_compliant(codemeta):
+    codemeta = copy.deepcopy(codemeta)
+
+    # context is different from codemeta
+    codemeta["@context"] = { 
+        "@vocab": "https://schema.org/", 
+        "codemeta": "https://w3id.org/codemeta/3.0/" 
+        }
+
+    # Some categories must be in a @list even if only one value is present. Required order
+    for prop in ["author", "contributor", "editor"]:
+        if prop in codemeta:
+            codemeta[prop] = wrap_list(codemeta[prop])
+
+    # referencePublication.author is also a list
+    if "referencePublication" in codemeta:
+        for pub in codemeta["referencePublication"]:
+            if "author" in pub:
+                pub["author"] = wrap_list(pub["author"])
+
+    if isinstance(codemeta.get("softwareRequirements"), list): 
+        codemeta["softwareRequirements"] = [ 
+            r for r in codemeta["softwareRequirements"] if isinstance(r, dict) 
+            ]
+        
+    if isinstance(codemeta.get("developmentStatus"), str): 
+        codemeta["developmentStatus"] = codemeta["developmentStatus"].capitalize()
+
+    codemeta = prefix_all_codemeta_properties(codemeta)
+    for key, value in codemeta.items(): 
+        codemeta[key] = normalize_value(value, key)
+
+    # No mappings. No normalizations. No cleaning.
+    # The PR only requires @context and @list wrapping.
+
+    return codemeta
+
+
+# ------------------------------------------------------------
+# UTILITIES
+# ------------------------------------------------------------
+
+def wrap_list(value):
+    """
+    Always wrap lists in @list, even if only one element.
+    """
+    if isinstance(value, list):
+        return {"@list": value}
+    return value
+
+def prefix_all_codemeta_properties(codemeta): 
+    """ Add codemeta: prefix to all properties that are NOT in Schema.org. """ 
+    new = {}
+
+    for key, value in codemeta.items(): 
+        if key in constants.SCHEMA_ORG_PROPERTIES or key == "@context": 
+            new[key] = value 
+        else: 
+            new[f"codemeta:{key}"] = value 
+    
+    return new
+
+def normalize_value(value, key=None):
+    """
+    Minimal normalization:
+    - Only normalize keywords (CSV -> list)
+    - Only filter softwareRequirements (remove strings)
+    - Do NOT touch any other property
+    """
+
+    # keywords: convert CSV to list
+    if key == "keywords" and isinstance(value, str):
+        parts = [p.strip() for p in value.split(",") if p.strip()]
+        return parts
+
+    # softwareRequirements: keep only dicts
+    if key == "softwareRequirements" and isinstance(value, list):
+        return [v for v in value if isinstance(v, dict)]
+
+    return value
+
+
+# def apply_schemaorg_mappings(c):
+#     """
+#     Apply mappings from Codemeta to Schema.org for Google compliance.
+#     """
+
+#     # referencePublication  -> citation
+#     if "referencePublication" in c:
+#         c["citation"] = c.pop("referencePublication")
+
+#     # buildInstructions ->  installUrl (if URL) or softwareHelp
+#     if "buildInstructions" in c:
+#         bi = c.pop("buildInstructions")
+#         if isinstance(bi, list) and len(bi) == 1 and is_url(bi[0]):
+#             c["installUrl"] = bi[0]
+#         else:
+#             c.setdefault("softwareHelp", [])
+#             c["softwareHelp"].append({"@type": "CreativeWork", "text": bi})
+
+#     # continuousIntegration -> softwareHelp
+#     if "continuousIntegration" in c:
+#         ci = c.pop("continuousIntegration")
+#         c.setdefault("softwareHelp", [])
+#         c["softwareHelp"].append({"@type": "CreativeWork", "url": ci})
+
+#     # readme -> softwareHelp
+#     if "readme" in c:
+#         rd = c.pop("readme")
+#         c.setdefault("softwareHelp", [])
+#         c["softwareHelp"].append({"@type": "CreativeWork", "url": rd})
+
+#     # developmentStatus normalization
+#     if "developmentStatus" in c:
+#         status = c["developmentStatus"]
+#         if isinstance(status, str):
+#             c["developmentStatus"] = status.capitalize()
+
+#     return c
+
+
+