pyCEURsemantify 0.0.1__py3-none-any.whl

ceursemantify/__init__.py

@@ -0,0 +1,7 @@
+"""
+Created on 2026-10-01
+
+@author: wf
+"""
+
+__version__ = "0.0.1"

ceursemantify/ceursemantify_cmd.py

@@ -0,0 +1,174 @@
+"""
+Created on 2026-10-01
+
+@author: wf
+"""
+
+import sys
+from argparse import ArgumentParser, Namespace
+from dataclasses import dataclass
+from typing import List, Optional
+
+from basemkit.base_cmd import BaseCmd
+
+import ceursemantify
+from ceursemantify.endpoint import SptEndpoint
+from ceursemantify.generator import Generator
+from ceursemantify.site import SiteConfig, SiteReport, VolumeRanges
+
+
+@dataclass
+class Version:
+    """
+    Version information for pyCEURsemantify
+    """
+
+    name: str = "pyCEURsemantify"
+    version: str = ceursemantify.__version__
+    date: str = "2026-10-01"
+    updated: str = "2026-10-13"
+    description: str = "Semantification of the CEUR-WS publication workflow"
+    authors: str = "Wolfgang Fahl"
+    doc_url: str = "https://github.com/ceurws/semantify"
+    chat_url: str = "https://github.com/ceurws/semantify/discussions"
+    cm_url: str = "https://github.com/ceurws/semantify"
+
+
+class CeurSemantifyCmd(BaseCmd):
+    """
+    command line interface of pyCEURsemantify:
+    generate volume pages, rebuild year pages, serve the endpoint
+    """
+
+    def __init__(self):
+        """
+        constructor
+        """
+        super().__init__(Version())
+
+    def add_arguments(self, parser: ArgumentParser) -> None:
+        """
+        add the arguments to the given parser
+
+        Args:
+            parser: the argument parser
+        """
+        super().add_arguments(parser)
+        parser.add_argument(
+            "volumes",
+            nargs="*",
+            help="volume numbers and ranges to generate, e.g. 3889 3887-3912",
+        )
+        parser.add_argument(
+            "-y",
+            "--year",
+            type=int,
+            action="append",
+            dest="years",
+            help="rebuild the page of the given year from the volumes present (can be given several times)",
+        )
+        parser.add_argument(
+            "--regenerate",
+            action="store_true",
+            help="with --year: fetch and write the volumes present for the year again before rebuilding its page",
+        )
+        parser.add_argument(
+            "--embed",
+            nargs="+",
+            metavar="PATH",
+            help="embed the JSON-LD into the given volume pages of the proceedings chairs (index.html files or"
+            " volume directories) and write index.jsonld next to them",
+        )
+        parser.add_argument("--progress", action="store_true", help="show a progress bar while generating volumes")
+        parser.add_argument(
+            "--serve",
+            action="store_true",
+            help="load all JSON-LD files into memory and serve them as SPARQL endpoint",
+        )
+        parser.add_argument("--host", default="127.0.0.1", help="host of the endpoint (default: %(default)s)")
+        parser.add_argument("--port", type=int, default=9987, help="port of the endpoint (default: %(default)s)")
+        parser.add_argument(
+            "-o",
+            "--output",
+            default=".",
+            help="directory that holds the year directories (default: current directory)",
+        )
+        parser.add_argument("--config", help="site configuration yaml file (default: the packaged site.yaml)")
+
+    def show_report(self, title: str, report: SiteReport, args: Namespace) -> None:
+        """
+        show the given report: problems always, written files if verbose
+
+        Args:
+            title: what the report is about
+            report: the report
+            args: the parsed arguments
+        """
+        for problem in report.problems:
+            print(problem, file=sys.stderr)
+        if args.verbose:
+            for path in report.written:
+                print(path)
+        if not args.quiet:
+            print(f"{title}: {len(report.written)} files, {len(report.problems)} problems, {report.seconds:.1f} s")
+
+    def handle_args(self, args: Namespace) -> bool:
+        """
+        handle the parsed arguments
+
+        Args:
+            args: the parsed arguments
+
+        Returns:
+            bool: True if the arguments were handled
+        """
+        handled = super().handle_args(args)
+        if not handled:
+            config = SiteConfig.of_yaml(args.config)
+            generator = Generator.of_output(args.output, config)
+            if args.embed:
+                report = generator.embed_volumes(args.embed, progress=args.progress)
+                self.show_report(f"{len(args.embed)} volume pages", report, args)
+                handled = True
+            if args.volumes:
+                numbers = VolumeRanges.numbers(args.volumes)
+                report = generator.generate_volumes(numbers, progress=args.progress)
+                self.show_report(f"{len(numbers)} volumes", report, args)
+                handled = True
+            for year in args.years or []:
+                if args.regenerate:
+                    report = generator.regenerate_year(year, progress=args.progress)
+                else:
+                    report = generator.rebuild_year(year)
+                self.show_report(f"year {year}", report, args)
+                handled = True
+            if args.serve:
+                endpoint = SptEndpoint(generator.site)
+                load_report = endpoint.load()
+                if not args.quiet:
+                    print(
+                        f"endpoint: {load_report.files} files, {load_report.triples} triples, "
+                        f"{load_report.seconds:.1f} s, http://{args.host}:{args.port}/npq/{endpoint.name}"
+                    )
+                endpoint.serve(host=args.host, port=args.port)
+                handled = True
+        return handled
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    """
+    command line entry point
+
+    Args:
+        argv: the command line arguments
+
+    Returns:
+        int: the exit code
+    """
+    cmd = CeurSemantifyCmd()
+    exit_code = cmd.run(argv)
+    return exit_code
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ceursemantify/convert.py

@@ -0,0 +1,204 @@
+"""
+Created on 2026-10-01
+
+@author: wf
+"""
+
+import os
+import re
+from datetime import date
+from typing import Any, Optional
+from urllib.parse import urlparse
+
+
+class Convert:
+    """
+    conversions of source values to the values of the research model
+
+    Each conversion is named in resources/ceurspt_mapping.yaml and raises a
+    ValueError for a value it can not convert.
+    """
+
+    @classmethod
+    def text(cls, value: Any) -> str:
+        """
+        normalize whitespace: line breaks and runs of blanks become one blank
+
+        Args:
+            value: the source text
+
+        Returns:
+            str: the single line text
+        """
+        text = " ".join(str(value).split())
+        return text
+
+    @classmethod
+    def integer(cls, value: Any) -> int:
+        """
+        convert the given value to an integer
+
+        Args:
+            value: the source value
+
+        Returns:
+            int: the integer
+        """
+        integer = int(value)
+        return integer
+
+    @classmethod
+    def volume_number(cls, value: Any) -> int:
+        """
+        get the volume number of a volume identifier such as Vol-3889
+
+        Args:
+            value: the volume identifier
+
+        Returns:
+            int: the volume number
+        """
+        match = re.fullmatch(r"Vol[-.]?(\d+)", str(value).strip())
+        if not match:
+            raise ValueError(f"not a volume identifier: {value}")
+        volume_number = int(match.group(1))
+        return volume_number
+
+    @classmethod
+    def iso_date(cls, value: Any) -> str:
+        """
+        check that the given value is an ISO date
+
+        Args:
+            value: the source date
+
+        Returns:
+            str: the date as YYYY-MM-DD
+        """
+        iso_date = date.fromisoformat(str(value).strip()).isoformat()
+        return iso_date
+
+    @classmethod
+    def last_path_parts(cls, value: Any, marker: str) -> str:
+        """
+        get the part of the path of an url that follows the given marker
+
+        Args:
+            value: the url or an identifier without url
+            marker: the path segment the identifier follows e.g. /pid/
+
+        Returns:
+            str: the identifier
+        """
+        identifier = str(value).strip()
+        path = urlparse(identifier).path
+        if marker in path:
+            identifier = path.split(marker, 1)[1]
+        return identifier
+
+    @classmethod
+    def wikidata_id(cls, value: Any) -> str:
+        """
+        get the Q-identifier of a Wikidata entity url or identifier
+
+        Args:
+            value: the entity url or Q-identifier
+
+        Returns:
+            str: the Q-identifier
+        """
+        wikidata_id = cls.last_path_parts(value, "/entity/")
+        if not re.fullmatch(r"Q\d+", wikidata_id):
+            raise ValueError(f"not a Wikidata identifier: {value}")
+        return wikidata_id
+
+    @classmethod
+    def dblp_person_id(cls, value: Any) -> str:
+        """
+        get the dblp person identifier of a dblp person url
+
+        Args:
+            value: the url e.g. https://dblp.org/pid/32/7870
+
+        Returns:
+            str: the identifier e.g. 32/7870
+        """
+        dblp_person_id = cls.last_path_parts(value, "/pid/")
+        return dblp_person_id
+
+    @classmethod
+    def dblp_record_id(cls, value: Any) -> str:
+        """
+        get the dblp publication identifier of a dblp record url
+
+        Args:
+            value: the url e.g. https://dblp.org/rec/conf/semtab/X24
+
+        Returns:
+            str: the identifier e.g. conf/semtab/X24
+        """
+        dblp_record_id = cls.last_path_parts(value, "/rec/")
+        return dblp_record_id
+
+    @classmethod
+    def file_stem(cls, value: Any) -> str:
+        """
+        get the file name without extension of an url
+
+        Args:
+            value: the url e.g. https://ceur-ws.org/Vol-3889/paper0.pdf
+
+        Returns:
+            str: the stem e.g. paper0
+        """
+        path = urlparse(str(value)).path
+        file_stem = os.path.splitext(os.path.basename(path))[0]
+        if not file_stem:
+            raise ValueError(f"no file name in: {value}")
+        return file_stem
+
+    @classmethod
+    def page_part(cls, value: Any, index: int) -> Optional[str]:
+        """
+        get the first or last page of a page range such as 1-11
+
+        Args:
+            value: the page range or a single page
+            index: 0 for the first page, -1 for the last page
+
+        Returns:
+            Optional[str]: the page
+        """
+        parts = [part.strip() for part in re.split(r"[-–—]+", str(value)) if part.strip()]
+        if not parts:
+            raise ValueError(f"no pages in: {value}")
+        page_part = parts[index]
+        return page_part
+
+    @classmethod
+    def page_start(cls, value: Any) -> Optional[str]:
+        """
+        get the first page of a page range
+
+        Args:
+            value: the page range
+
+        Returns:
+            Optional[str]: the first page
+        """
+        page_start = cls.page_part(value, 0)
+        return page_start
+
+    @classmethod
+    def page_end(cls, value: Any) -> Optional[str]:
+        """
+        get the last page of a page range
+
+        Args:
+            value: the page range
+
+        Returns:
+            Optional[str]: the last page
+        """
+        page_end = cls.page_part(value, -1)
+        return page_end

ceursemantify/embed.py

@@ -0,0 +1,84 @@
+"""
+Created on 2026-10-13
+
+@author: wf
+"""
+
+import re
+
+from ceursemantify.pages import Fence, PageState
+
+
+class Embedder:
+    """
+    embed the JSON-LD into the volume page of the proceedings chair: the
+    state comment before the html element, the alternate link and the fence
+    in the head; the markup of the chair is otherwise untouched
+    """
+
+    alternate_link = '<link rel="alternate" type="application/ld+json" href="index.jsonld">'
+    state_pattern = re.compile(r"<!-- CEURSTATE=[^\n]*? -->\n?")
+    fence_pattern = re.compile(rf"<!--\n{Fence.fence}json-ld\n.*?\n{Fence.fence}\n-->\n?", re.DOTALL)
+    link_pattern = re.compile(re.escape(alternate_link) + r"\n?")
+    # the state comment goes after the doctype, before the html element otherwise
+    doctype_pattern = re.compile(r"<!DOCTYPE[^>]*>\n?", re.IGNORECASE)
+    html_pattern = re.compile(r"<html[\s>]", re.IGNORECASE)
+    head_pattern = re.compile(r"<head[^>]*>\n?", re.IGNORECASE)
+
+    def __init__(self, state: PageState):
+        """
+        constructor
+
+        Args:
+            state: the state the page announces
+        """
+        self.state = state
+
+    def strip(self, html_text: str) -> str:
+        """
+        remove the embedded parts from the given page
+
+        Args:
+            html_text: the html of a volume page
+
+        Returns:
+            str: the html without state comment, alternate link and fence
+        """
+        stripped = self.state_pattern.sub("", html_text)
+        stripped = self.fence_pattern.sub("", stripped)
+        stripped = self.link_pattern.sub("", stripped)
+        return stripped
+
+    def embed(self, html_text: str, jsonld_text: str, source: str) -> str:
+        """
+        embed the given JSON-LD into the given page, replacing an earlier embedding
+
+        Args:
+            html_text: the html of a volume page
+            jsonld_text: the pretty printed JSON-LD
+            source: what the JSON-LD was derived from e.g. the url of the volume
+
+        Returns:
+            str: the html with the JSON-LD embedded
+
+        Raises:
+            ValueError: if the page has no html or no head element
+        """
+        stripped = self.strip(html_text)
+        html_match = self.html_pattern.search(stripped)
+        head_match = self.head_pattern.search(stripped)
+        if html_match is None or head_match is None:
+            raise ValueError("no html or head element to embed into")
+        doctype_match = self.doctype_pattern.match(stripped)
+        state_at = doctype_match.end() if doctype_match else html_match.start()
+        state_comment = self.state.as_html(source, state="embedded")
+        inserted = f"{self.alternate_link}\n{Fence.as_html(jsonld_text)}\n"
+        embedded = (
+            stripped[:state_at]
+            + state_comment
+            + "\n"
+            + stripped[state_at : head_match.end()]
+            + inserted
+            + stripped[head_match.end() :]
+        )
+        return embedded

ceursemantify/endpoint.py

@@ -0,0 +1,72 @@
+"""
+Created on 2026-10-01
+
+@author: wf
+"""
+
+import time
+from dataclasses import dataclass
+
+from lodstorage.npq_endpoint import NpqEndpoint
+from rdflib import Graph
+
+from ceursemantify.site import Site
+
+
+@dataclass
+class LoadReport:
+    """
+    what was loaded into the endpoint
+    """
+
+    files: int = 0
+    triples: int = 0
+    seconds: float = 0.0
+
+
+class SptEndpoint:
+    """
+    single point of truth endpoint: the JSON-LD files of the site loaded
+    into memory on startup and served by the NPQ endpoint of pyLoDStorage,
+    see https://cr.bitplan.com/index.php/NPQ-FakeSparql
+    """
+
+    def __init__(self, site: Site, name: str = "ceur-ws"):
+        """
+        constructor
+
+        Args:
+            site: the directory tree with the JSON-LD files
+            name: the name the graph is served under: /npq/<name>
+        """
+        self.site = site
+        self.name = name
+        self.graph = Graph()
+
+    def load(self) -> LoadReport:
+        """
+        load all JSON-LD files of the site into the graph
+
+        Returns:
+            LoadReport: number of files and triples and the time used
+        """
+        report = LoadReport()
+        start = time.perf_counter()
+        for path in self.site.jsonld_paths():
+            self.graph.parse(path, format="json-ld")
+            report.files += 1
+        report.triples = len(self.graph)
+        report.seconds = time.perf_counter() - start
+        return report
+
+    def serve(self, host: str = "127.0.0.1", port: int = 9987) -> None:
+        """
+        serve the graph over the SPARQL protocol
+
+        Args:
+            host: the host to listen on
+            port: the port to listen on
+        """
+        endpoint = NpqEndpoint()
+        endpoint.add_graph(self.name, self.graph)
+        endpoint.run(host=host, port=port)