cmem-cmemc 24.3.0rc1__tar.gz → 24.3.0rc2__tar.gz

{cmem_cmemc-24.3.0rc1 → cmem_cmemc-24.3.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cmem-cmemc
-Version: 24.3.0rc1
+Version: 24.3.0rc2
 Summary: Command line client for eccenca Corporate Memory
 Home-page: https://eccenca.com/go/cmemc
 License: Apache-2.0

{cmem_cmemc-24.3.0rc1 → cmem_cmemc-24.3.0rc2}/cmem_cmemc/commands/graph.py

@@ -4,13 +4,16 @@ import gzip
 import hashlib
 import io
 import json
+import mimetypes
 import os
+from json import JSONDecodeError
 from xml.dom import minidom  # nosec
 from xml.etree.ElementTree import (  # nosec
     Element,
     SubElement,
     tostring,
 )
+from xml.sax import SAXParseException
 
 import click
 from click import Argument
@@ -20,6 +23,9 @@ from cmem.cmempy.dp.proxy import graph as graph_api
 from cmem.cmempy.dp.proxy.graph import get_graph_import_tree, get_graph_imports
 from cmem.cmempy.dp.proxy.sparql import get as sparql_api
 from jinja2 import Template
+from rdflib import Graph
+from rdflib.exceptions import ParserError
+from rdflib.plugins.parsers.notation3 import BadSyntax
 from six.moves.urllib.parse import quote
 from treelib import Tree
 
@@ -103,7 +109,7 @@ def _get_graph_to_file(  # noqa: PLR0913
 
 
 def _get_export_names(
-    app: ApplicationContext, iris: list[str], template: str, file_extension: str = "ttl"
+    app: ApplicationContext, iris: list[str], template: str, file_extension: str = ".ttl"
 ) -> dict:
     """Get a dictionary of generated file names based on a template.
 
@@ -131,7 +137,7 @@ def _get_export_names(
             hash=hashlib.sha256(iri.encode("utf-8")).hexdigest(),
             iriname=convert_uri_to_filename(iri),
         )
-        _name_created = f"{Template(template).render(template_data)}.{file_extension}"
+        _name_created = f"{Template(template).render(template_data)}{file_extension}"
         _names[iri] = _name_created
     if len(_names.values()) != len(set(_names.values())):
         raise ValueError(
@@ -542,9 +548,9 @@ def _validate_export_command_input_parameters(
 )
 @click.option(
     "--mime-type",
-    default="application/n-triples",
+    default="text/turtle",
     show_default=True,
-    type=click.Choice(["application/n-triples", "text/turtle"]),
+    type=click.Choice(["application/n-triples", "text/turtle", "application/rdf+xml"]),
     help="Define the requested mime type",
 )
 @click.option(
@@ -589,8 +595,11 @@ def export_command(  # noqa: PLR0913
         app.echo_debug("output is directory")
         # pre-calculate all filenames with the template,
         # in order to output errors on naming clashes as early as possible
-        _names = _get_export_names(app, iris, template, "ttl.gz" if compress else "ttl")
-        _graph_file_names = _get_export_names(app, iris, template, "ttl.graph")
+        extension = mimetypes.guess_extension(mime_type)
+        _names = _get_export_names(
+            app, iris, template, f"{extension}.gz" if compress else f"{extension}"
+        )
+        _graph_file_names = _get_export_names(app, iris, template, f"{extension}.graph")
         # create directory
         if not Path(output_dir).exists():
             app.echo_warning("Output directory does not exist: " + "will create it.")
@@ -661,6 +670,57 @@ def validate_input_path(input_path: str) -> None:
         )
 
 
+def _get_graph_supported_formats() -> dict[str, str]:
+    return {
+        "application/rdf+xml": "xml",
+        "application/ld+json": "json-ld",
+        "text/turtle": "turtle",
+        "application/n-triples": "nt",
+    }
+
+
+def _guess_rdf_mime_type(content: str) -> str:
+    formats = _get_graph_supported_formats()
+    for mime_type, rdf_format in formats.items():
+        try:
+            g = Graph()
+            g.parse(data=content, format=rdf_format)
+        except (SAXParseException, JSONDecodeError, BadSyntax, ParserError):
+            continue
+        else:
+            return mime_type
+    raise ValueError("Unknown format")
+
+
+def _parse_triple_file(triple_file: str) -> tuple[io.BytesIO, str]:
+    """Parse the content of the triple file."""
+    buffer = io.BytesIO()
+    transport_params = {}
+
+    if Path(str(triple_file)).schema in ["http", "https"]:
+        transport_params["headers"] = {
+            "Accept": "text/turtle; q=1.0, application/x-turtle; q=0.9, text/n3;"
+            " q=0.8, application/rdf+xml; q=0.5, text/plain; q=0.1"
+        }
+
+    with ClickSmartPath.open(triple_file, transport_params=transport_params) as file_obj:
+        buffer.write(file_obj.read())
+
+    buffer.seek(0)
+    is_gzip = buffer.read(2) == b"\x1f\x8b"
+    buffer.seek(0)
+
+    if is_gzip:
+        with gzip.GzipFile(fileobj=buffer, mode="rb") as gzip_file:
+            graph_content = gzip_file.read().decode("utf-8")
+    else:
+        graph_content = buffer.read().decode("utf-8")
+
+    content_type = _guess_rdf_mime_type(graph_content)
+    buffer.seek(0)
+    return buffer, content_type
+
+
 @click.command(cls=CmemcCommand, name="import")
 @click.option(
     "--replace",
@@ -722,8 +782,12 @@ def import_command(
             graphs = read_rdf_graph_files(input_path)
         else:
             # in case a directory is the source AND IRI is given
-            graphs = [(file, iri) for file in Path(input_path).glob("*.ttl")]
-            graphs += [(file, iri) for file in Path(input_path).glob("*.ttl.gz")]
+            graphs = []
+            for _ in _get_graph_supported_formats():
+                extension = mimetypes.guess_extension(_)
+                graphs += [(file, iri) for file in Path(input_path).glob(f"*{extension}")]
+                graphs += [(file, iri) for file in Path(input_path).glob(f"*{extension}.gz")]
+
     elif Path(input_path).is_file():
         if iri is None:
             raise ValueError(
@@ -748,17 +812,10 @@ def import_command(
             continue
         # prevents re-replacing of graphs in a single run
         _replace = False if graph_iri in processed_graphs else replace
-        _buffer = io.BytesIO()
-        transport_prams = {}
-        if Path(str(triple_file)).schema in ["http", "https"]:
-            transport_prams["headers"] = {
-                "Accept": "text/turtle; q=1.0, application/x-turtle; q=0.9, text/n3;"
-                " q=0.8, application/rdf+xml; q=0.5, text/plain; q=0.1"
-            }
-        with ClickSmartPath.open(triple_file, transport_params=transport_prams) as _:
-            _buffer.write(_.read())
-        _buffer.seek(0)
-        response = graph_api.post_streamed(graph_iri, _buffer, replace=_replace)
+        _buffer, content_type = _parse_triple_file(triple_file)
+        response = graph_api.post_streamed(
+            graph_iri, _buffer, replace=_replace, content_type=content_type
+        )
         request_headers = response.request.headers
         request_headers.pop("Authorization")
         app.echo_debug(f"cmemc request headers: {request_headers}")

{cmem_cmemc-24.3.0rc1 → cmem_cmemc-24.3.0rc2}/cmem_cmemc/commands/migration.py

@@ -18,6 +18,7 @@ from cmem_cmemc.migrations.bootstrap_data import MigrateBootstrapData
 from cmem_cmemc.migrations.shapes_widget_integrations_243 import (
     ChartsOnNodeShapesToToWidgetIntegrations,
     ChartsOnPropertyShapesToWidgetIntegrations,
+    TableReportPropertyShapesToWidgetIntegrations,
     WorkflowTriggerPropertyShapesToWidgetIntegrations,
 )
 from cmem_cmemc.migrations.workspace_configurations import MigrateWorkspaceConfiguration
@@ -54,6 +55,7 @@ def get_migrations(ctx: click.Context) -> list[dict]:  # noqa: ARG001
             ChartsOnNodeShapesToToWidgetIntegrations(),
             ChartsOnPropertyShapesToWidgetIntegrations(),
             WorkflowTriggerPropertyShapesToWidgetIntegrations(),
+            TableReportPropertyShapesToWidgetIntegrations(),
         ]
     ]
     data.sort(key=lambda x: x["first_version"])

{cmem_cmemc-24.3.0rc1 → cmem_cmemc-24.3.0rc2}/cmem_cmemc/commands/python.py

@@ -181,11 +181,11 @@ def list_command(app: ApplicationContext, raw: bool, id_only: bool, available: b
             return
 
         table_published = [
-            (_.name, str(_.published)[:10], _.description) for _ in published_packages
+            (_.name, _.version, str(_.published)[:10], _.description) for _ in published_packages
         ]
         app.echo_info_table(
             table_published,
-            headers=["Name", "Published", "Description"],
+            headers=["Name", "Version", "Published", "Description"],
             sort_column=0,
             empty_table_message="No available python packages found.",
         )

{cmem_cmemc-24.3.0rc1 → cmem_cmemc-24.3.0rc2}/cmem_cmemc/completion.py

@@ -460,7 +460,7 @@ def installed_package_names(ctx: Context, param: Argument, incomplete: str) -> l
 
 def published_package_names(ctx: Context, param: Argument, incomplete: str) -> list[CompletionItem]:
     """List of plugin packages scraped from pypi.org."""
-    options = [(_.name, f"{_.description}") for _ in get_published_packages()]
+    options = [(_.name, f"{_.version}: {_.description}") for _ in get_published_packages()]
     return finalize_completion(candidates=options, incomplete=incomplete, sort_by=SORT_BY_KEY)
 
 
@@ -600,13 +600,19 @@ def sparql_files(ctx: Context, param: Argument, incomplete: str) -> list[Complet
 def triple_files(ctx: Context, param: Argument, incomplete: str) -> list[CompletionItem]:
     """Prepare a list of triple files."""
     return (
-        file_list(incomplete=incomplete, suffix=".ttl", description="RDF Turtle file")
-        + file_list(incomplete=incomplete, suffix=".nt", description="RDF NTriples file")
+        file_list(incomplete=incomplete, suffix=".ttl", description="Turtle file")
+        + file_list(incomplete=incomplete, suffix=".nt", description="NTriples file")
+        + file_list(incomplete=incomplete, suffix=".rdf", description="RDF/XML file")
+        + file_list(incomplete=incomplete, suffix=".jsonld", description="JSON-LD file")
+        + file_list(incomplete=incomplete, suffix=".ttl.gz", description="Turtle file (compressed)")
         + file_list(
-            incomplete=incomplete, suffix=".ttl.gz", description="Compressed RDF Turtle file"
+            incomplete=incomplete, suffix=".nt.gz", description="NTriples file (compressed)"
         )
         + file_list(
-            incomplete=incomplete, suffix=".nt.gz", description="Compressed RDF NTriples file"
+            incomplete=incomplete, suffix=".rdf.gz", description="RDF/XML file (compressed)"
+        )
+        + file_list(
+            incomplete=incomplete, suffix=".jsonld.gz", description="JSON-LD file (compressed)"
         )
     )
 

{cmem_cmemc-24.3.0rc1 → cmem_cmemc-24.3.0rc2}/cmem_cmemc/migrations/access_conditions_243.py

@@ -20,6 +20,7 @@ SELECT ?subject
 FROM <urn:elds-backend-access-conditions-graph>
 WHERE {
   ?subject a ?class .
+  FILTER NOT EXISTS { ?subject shui:isSystemResource true }
   FILTER (?class IN (auth:AccessCondition, shui:SparqlQuery) ) .
 }
     """
@@ -31,6 +32,7 @@ WHERE {
     ?s ?p ?o .
     ?s a ?class .
     FILTER (?class IN (auth:AccessCondition, shui:SparqlQuery) ) .
+    FILTER NOT EXISTS { ?s shui:isSystemResource true }
   }
 }
     """
@@ -59,6 +61,7 @@ SELECT DISTINCT ?acl_id
 WHERE {
   GRAPH ?acl_graph {
 	?acl_id a auth:AccessCondition .
+    FILTER NOT EXISTS { ?acl_id shui:isSystemResource true }
     ?acl_id ?property ?old_term .
     FILTER (?old_term in (
         <urn:elds-backend-all-actions>,
@@ -85,6 +88,7 @@ INSERT { GRAPH ?acl_graph { ?s ?p <{{NEW_IRI}}> . } }
 WHERE {
   GRAPH ?acl_graph {
     ?s ?p ?o .
+    FILTER NOT EXISTS { ?s shui:isSystemResource true }
     FILTER (?o = <{{OLD_IRI}}> )
   }
   FILTER (?acl_graph IN (<urn:elds-backend-access-conditions-graph>, <https://ns.eccenca.com/data/ac/>))

{cmem_cmemc-24.3.0rc1 → cmem_cmemc-24.3.0rc2}/cmem_cmemc/migrations/bootstrap_data.py

@@ -20,10 +20,8 @@ class MigrateBootstrapData(MigrationRecipe):
     def is_applicable(self) -> bool:
         """Test if the recipe can be applied."""
         status_info = get_complete_status_info()
-        return status_info["shapes"]["version"] not in (
-            status_info["explore"]["version"],
-            "UNKNOWN",
-        )
+        shapes_status = status_info["shapes"]["version"]
+        return bool(shapes_status != status_info["explore"]["version"])
 
     def apply(self) -> None:
         """Apply the recipe to the current version."""

{cmem_cmemc-24.3.0rc1 → cmem_cmemc-24.3.0rc2}/cmem_cmemc/migrations/shapes_widget_integrations_243.py

@@ -192,3 +192,83 @@ WHERE {
     def apply(self) -> None:
         """Apply the recipe to the current version."""
         self._update(self.move_query)
+
+
+class TableReportPropertyShapesToWidgetIntegrations(MigrationRecipe):
+    """24.3 Migrate Table Report Property Shapes to Widget Integrations"""
+
+    id = "table-reports-on-pshapes-24.3"
+    description = (
+        "Migrate Table Reports (which use shui:null) on Property Shapes to Widget Integrations"
+    )
+    component: components = "explore"
+    first_version = "24.3"
+    tags: ClassVar[list[str]] = ["shapes", "user"]
+    check_query = """{{DEFAULT_PREFIXES}}
+SELECT ?propertyShape
+WHERE {
+  GRAPH ?shapeCatalog {
+    ?shapeCatalog a shui:ShapeCatalog .
+    ?propertyShape a sh:PropertyShape ;
+      shui:valueQuery ?valueQuery ;
+      sh:path shui:null .
+    FILTER NOT EXISTS { ?propertyShape shui:isSystemResource true }
+  }
+}
+    """
+    move_query = """{{DEFAULT_PREFIXES}}
+DELETE {
+  GRAPH ?shapeCatalog {
+    ?propertyShape ?p ?o .
+    ?nodeShape sh:property ?propertyShape .
+  }
+}
+INSERT {
+  GRAPH ?shapeCatalog {
+    ?propertyShape a shui:WidgetIntegration ;
+      shui:WidgetIntegration_widget ?newTableReport ;
+      rdfs:label ?name ;
+      rdfs:comment ?description ;
+      sh:order ?order ;
+      shui:WidgetIntegration_group ?group .
+    ?nodeShape shui:WidgetIntegration_integrate ?propertyShape .
+
+    ?newTableReport a shui:TableReport ;
+      rdfs:label ?newTableReportLabel ;
+      shui:TableReport_hideFooter ?hideFooter ;
+      shui:TableReport_hideHeader ?hideHeader ;
+      shui:TableReport_query ?valueQuery .
+  }
+}
+WHERE {
+  GRAPH ?shapeCatalog {
+    ?shapeCatalog a shui:ShapeCatalog .
+    ?propertyShape a sh:PropertyShape ;
+      sh:path shui:null ;
+      shui:valueQuery ?valueQuery .
+    FILTER NOT EXISTS { ?propertyShape shui:isSystemResource true }
+    ?propertyShape ?p ?o .
+    OPTIONAL {?propertyShape sh:name ?name }
+    OPTIONAL {?propertyShape sh:description ?description }
+    OPTIONAL {?propertyShape sh:order ?order }
+    OPTIONAL {?propertyShape sh:group ?group }
+    OPTIONAL {?propertyShape sh:name ?name }
+    OPTIONAL {?propertyShape shui:valueQueryHideHeader ?hideHeaderValue }
+    OPTIONAL {?propertyShape shui:valueQueryHideFooter ?hideFooterValue }
+    OPTIONAL {?nodeShape sh:property ?propertyShape }
+    BIND ( IRI(CONCAT(STR(?propertyShape), "-TableReport")) AS ?newTableReport )
+    BIND ( STRLANG(CONCAT("Table Report: ", ?name), lang(?name)) AS ?newTableReportLabel )
+    BIND ( COALESCE(?hideHeaderValue, false) as ?hideHeader)
+    BIND ( COALESCE(?hideFooterValue, false) as ?hideFooter)
+  }
+}
+    """
+
+    def is_applicable(self) -> bool:
+        """Test if the recipe can be applied."""
+        table_report_property_shapes = self._select(self.check_query)
+        return len(table_report_property_shapes) > 0
+
+    def apply(self) -> None:
+        """Apply the recipe to the current version."""
+        self._update(self.move_query)

{cmem_cmemc-24.3.0rc1 → cmem_cmemc-24.3.0rc2}/cmem_cmemc/string_processor.py

@@ -24,7 +24,13 @@ class TimeAgo(StringProcessor):
 
     def process(self, text: str) -> str:
         """Process a single string content and output the processed string."""
-        stamp = datetime.fromtimestamp(int(text) / 1000, tz=timezone.utc)
+        if text is None:
+            return ""
+        try:
+            text_as_int = int(text)
+        except ValueError:
+            return text
+        stamp = datetime.fromtimestamp(text_as_int / 1000, tz=timezone.utc)
         return str(timeago.format(stamp, datetime.now(tz=timezone.utc)))
 
 

{cmem_cmemc-24.3.0rc1 → cmem_cmemc-24.3.0rc2}/cmem_cmemc/utils.py

@@ -12,7 +12,6 @@ from typing import TYPE_CHECKING
 from zipfile import BadZipFile, ZipFile
 
 import requests
-from bs4 import BeautifulSoup
 from cmem.cmempy.dp.proxy.graph import get_graphs_list
 from cmem.cmempy.queries import QueryCatalog
 from cmem.cmempy.workspace.projects.project import get_projects
@@ -288,41 +287,30 @@ class PublishedPackage:
     name: str
     description: str
     published: str
+    version: str
 
 
 def get_published_packages() -> list[PublishedPackage]:
     """Get a scraped list of plugin packages scraped from pypi.org."""
-    url_pattern = "https://pypi.org/search/?q=%22cmem-plugin-%22&page={}"
-    packages = []
+    url = "https://download.eccenca.com/cmem-plugin-index/cmem-plugin-index.json"
     package_names = []
-    page = 0
-    while True:
-        page += 1
-        url = url_pattern.format(page)
-        soup = BeautifulSoup(requests.get(url, timeout=5).content, "html.parser")
-        snippets = soup.find_all("a", class_="package-snippet")
-        if len(snippets) == 0:
-            break
-        for _ in snippets:
-            name = _.findChildren(class_="package-snippet__name")[0].getText()
-            if name == "cmem-plugin-base":
-                continue
-            if not name.startswith("cmem-plugin-"):
-                continue
-            try:
-                description = _.findChildren(class_="package-snippet__description")[0].getText()
-            except IndexError:
-                description = "no description"
-            published = _.findChildren(name="time")[0].attrs["datetime"]
-            if name not in package_names:
-                package_names.append(name)
-                packages.append(
-                    PublishedPackage(
-                        name=name,
-                        description=description,
-                        published=published,
-                    )
+    packages = []
+    for _ in requests.get(url, timeout=5).json():
+        name = _["name"]
+        if name == "cmem-plugin-base":
+            continue
+        if not name.startswith("cmem-plugin-"):
+            continue
+        if name not in package_names:
+            package_names.append(name)
+            packages.append(
+                PublishedPackage(
+                    name=name,
+                    description=_["summary"],
+                    published=_["latest_version_time"],
+                    version=_["latest_version"],
                 )
+            )
     return packages
 
 

{cmem_cmemc-24.3.0rc1 → cmem_cmemc-24.3.0rc2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cmem-cmemc"
-version = "24.3.0rc1"
+version = "24.3.0rc2"
 description = "Command line client for eccenca Corporate Memory"
 license = "Apache-2.0"
 classifiers = [