gamslib 0.3.1__py3-none-any.whl

gamslib/__init__.py

@@ -0,0 +1,6 @@
+"""A library with modules which a needed in other GAMS packages.
+
+The gamslib package contains modules which are needed in other GAMS packages.
+
+  * The objectcsv module provides tools to handle object and datastream metadata
+"""

gamslib/objectcsv/__init__.py

@@ -0,0 +1,57 @@
+"""Handle object and datastream metadata in csv files.
+
+When creating bags for GAMS, we provide some metadata in csv
+files (which are not part of the bag, btw).
+
+The objectcsv package provides tools to handle this metadata.
+
+  * The ObjectCSV class represents the object and datastream csv data
+    for a single object. It is created by providing the path to the
+    object directory. It is composed of two classes:
+
+    * ObjectCSVFile represents the object metadata. It hold typically
+      a single ObjectData object, but can hold multiple objects if needed.
+    * DatastreamsCSVFile represents the datastream metadata. It holds
+        typically multiple DSData objects, one for each datastream.
+  * The dublincore_csv module represents the object metadata stored in
+    the objects 'DC.xml' file. It provides useful functions for acessing
+    DC data e.g. for prefered languages etc.
+  * The create_csv module can be used to initally create the csv files for
+    all objects
+  * The manage_csv module can be used collect csv data from all objects
+    into a single file, which makes editing the data more efficient.
+    It also has a function to update the csv files in the object directories
+    based on the collected data.
+  * The xlsx module can be used to convert the csv files to xlsx files
+    and vice versa. This is useful for editing the data in a spreadsheet
+    without the hassles of importing and exporting the csv files, which
+    led to encoding problems in the past.
+
+The "public" functions and classes from the submodules are directly
+available in the objectcsv:
+
+  * ObjectCSV
+  * ObjectData
+  * DSData
+  * create_csv_files
+  * collect_csv_data
+  * update_csv_files
+  * csv_to_xlsx
+  * xlsx_to_csv
+"""
+
+from .objectcsv import ObjectCSV, ObjectData, DSData
+from .create_csv import create_csv_files
+from .manage_csv import collect_csv_data, update_csv_files
+from .xlsx import csv_to_xlsx, xlsx_to_csv
+
+__all__ = [
+    "ObjectCSV",
+    "ObjectData",
+    "DSData",
+    "create_csv_files",
+    "collect_csv_data",
+    "update_csv_files",
+    "csv_to_xlsx",
+    "xlsx_to_csv",
+]

gamslib/objectcsv/create_csv.py

@@ -0,0 +1,173 @@
+"""Create object.csv and datastreams.csv files.
+
+This module creates the object.csv and datastreams.csv files for one or many given
+object folder. It uses data from the DC.xml file and the project configuration
+to fill in the metadata. When not enough information is available, some fields
+will be left blank or filled with default values.
+"""
+
+import logging
+import mimetypes
+import re
+from pathlib import Path
+import warnings
+
+from gamslib.projectconfiguration import Configuration
+
+from .objectcsv import DSData, ObjectCSV, ObjectData
+from .dublincore import DublinCore
+from .utils import find_object_folders
+from . import defaultvalues
+
+logger = logging.getLogger()
+
+
+NAMESPACES = {
+    "dc": "http://purl.org/dc/elements/1.1/",
+}
+
+
+def get_rights(config: Configuration, dc: DublinCore) -> str:
+    """Get the rights from various sources.
+
+    Lookup in this ortder:
+
+      1. Check if set in dublin core
+      2. Check if set in the configuration
+      3. Use a default value.
+    """
+    rights = dc.get_element_as_str("rights", default="")
+    if not rights:  # empty string is a valid value
+        if config.metadata.rights:
+            rights = config.metadata.rights
+        else:
+            rights = defaultvalues.DEFAULT_RIGHTS
+    return rights
+
+
+def extract_dsid(datastream: Path | str, keep_extension=True) -> str:
+    """Extract and validate the datastream id from a datastream path.
+
+    If remove_extension is True, the file extension is removed from the PID.
+    """
+    if isinstance(datastream, str):
+        datastream = Path(datastream)
+
+    pid = datastream.name
+
+    if not keep_extension:
+        # not everything after the last dot is an extension :-(
+        mtype = mimetypes.guess_type(datastream)[0]
+        if mtype is None:
+            known_extensions = []
+        else:
+            known_extensions = mimetypes.guess_all_extensions(mtype)
+        if datastream.suffix in known_extensions:
+            pid = pid.removesuffix(datastream.suffix)
+            logger.debug("Removed extension '%s' for ID: %s", datastream.suffix, pid)
+        else:
+            parts = pid.split(".")
+            if re.match(r"^[a-zA-Z]+\w?$", parts[-1]):
+                pid = ".".join(parts[:-1])
+                logger.debug("Removed extension for ID: %s", parts[0])
+            else:
+                warnings.warn(f"'{pid[-1]}' does not look like an extension. Keeping it in PID.", 
+                        UserWarning)
+
+    if re.match(r"^[a-zA-Z0-9]+[-.%_a-zA-Z0-9]+[a-zA-Z0-9]+$", pid) is None:
+        raise ValueError(f"Invalid PID: '{pid}'")
+
+    logger.debug(
+        "Extracted PID: %s from %s (keep_extension=%s)", pid, datastream, keep_extension
+    )
+    return pid
+
+
+def collect_object_data(pid: str, config: Configuration, dc: DublinCore) -> ObjectData:
+    """Find data for the object.csv by examining dc file and configuration.
+
+    This is the place to change the resolving order for data from other sources.
+    """
+
+    title = "; ".join(dc.get_element("title", default=pid))
+    description = "; ".join(dc.get_element("description", default=""))
+
+    return ObjectData(
+        recid=pid,
+        title=title,
+        project=config.metadata.project_id,
+        description=description,
+        creator=config.metadata.creator,
+        rights=get_rights(config, dc),
+        source=defaultvalues.DEFAULT_SOURCE,
+        objectType=defaultvalues.DEFAULT_OBJECT_TYPE,
+    )
+
+
+def collect_datastream_data(
+    ds_file: Path, config: Configuration, dc: DublinCore
+) -> DSData:
+    """Collect data for a single datastream."""
+    dsid = extract_dsid(ds_file, config.general.dsid_keep_extension)
+
+    # I think it's not possible to derive a ds title or description from the DC file
+    # title = "; ".join(dc.get_element("title", default=dsid)) # ??
+    # description = "; ".join(dc.get_element("description", default="")) #??
+
+    return DSData(
+        dspath=str(ds_file.relative_to(ds_file.parents[1])),  # objectsdir
+        dsid=dsid,
+        title="",
+        description="",
+        mimetype=mimetypes.guess_type(ds_file)[0] or "",
+        creator=config.metadata.creator,
+        rights=get_rights(config, dc),
+    )
+
+
+def create_csv(
+    object_directory: Path, configuration: Configuration, force_overwrite: bool = False
+) -> ObjectCSV | None:
+    """Generate the csv file containing the preliminary metadata for a single object.
+
+    Existing csv files will not be touched unless 'force_overwrite' is True.
+    """
+    objectcsv = ObjectCSV(object_directory)
+
+    # Avoid that existing (and potentially already edited) metadata is replaced
+    if force_overwrite and not objectcsv.is_new():
+        objectcsv.clear()
+        objectcsv.obj_csv_file.unlink()
+        objectcsv.ds_csv_file.unlink()
+    if not objectcsv.is_new():
+        logger.info(
+            "CSV files for object '%s' already exist. Will not be re-created.",
+            objectcsv.object_id,
+        )
+        return None
+
+    dc = DublinCore(object_directory / "DC.xml")
+    objectcsv.add_objectdata(
+        collect_object_data(objectcsv.object_id, configuration, dc)
+    )
+    for ds_file in object_directory.glob("*"):
+        if ds_file.is_file() and ds_file.name not in ("object.csv", "datastreams.csv"):
+            objectcsv.add_datastream(
+                # collect_datastream_data(ds_file, objectcsv.object_id, configuration, dc)
+                collect_datastream_data(ds_file, configuration, dc)
+            )
+    objectcsv.write()
+    return objectcsv
+
+
+def create_csv_files(
+    root_folder: Path, config: Configuration, force_overwrite:bool=False
+) -> list[ObjectCSV]:
+    """Create the CSV files for all objects below root_folder."""
+    extended_objects: list[ObjectCSV] = []
+    for path in find_object_folders(root_folder):
+        extended_obj = create_csv(path, config, force_overwrite)
+
+        if extended_obj is not None:
+            extended_objects.append(extended_obj)
+    return extended_objects

gamslib/objectcsv/defaultvalues.py

@@ -0,0 +1,28 @@
+"""Default values for the datastream meatadata."""
+
+DEFAULT_CREATOR = "Unknown"
+DEFAULT_MIMETYPE = "application/octet-stream"
+DEFAULT_OBJECT_TYPE = "text"
+DEFAULT_RIGHTS = "Creative Commons Attribution-NonCommercial 4.0 (https://creativecommons.org/licenses/by-nc/4.0/)"
+DEFAULT_SOURCE = "local"
+
+# This is a mapping of filenames to default metadata values.
+# Add new entries here if you want to add new metadata fields.
+FILENAME_MAP = {
+    "DC.xml": {
+        "title": "Dublin Core Metadata",
+        "description": "Dublin Core Metadata in XML format for this content file.",
+    },
+    "TEI.xml": {
+        "title": "Main TEI file",
+        "description": "The central TEI File for this object"
+    },
+    "LIDO.xml": {
+        "title": "Main LIDO file",
+        "description": "The central LIDO file of this object"
+    },
+    "RDF.xml": {
+        "title": "RDF Statements",
+        "description": ""
+    }
+}

gamslib/objectcsv/dublincore.py

@@ -0,0 +1,188 @@
+"""Represents data from DC.xml and provides some methods to access it."""
+
+import logging
+from pathlib import Path
+from typing import Any
+from xml.etree import ElementTree as ET
+
+logger = logging.getLogger(__name__)
+
+DC_ELEMENTS = [
+    "contributor",
+    "coverage",
+    "creator",
+    "date",
+    "description",
+    "format",
+    "identifier",
+    "language",
+    "publisher",
+    "relation",
+    "rights",
+    "source",
+    "subject",
+    "title",
+    "type",
+]
+
+## DC_TERMS and DCMI_TYPES are not used in the current implementation,
+## but are kept here for future reference.
+# DC_TERMS = [
+#     "abstract",
+#     "accessRights",
+#     "accrualMethod",
+#     "accrualPeriodicity",
+#     "accrualPolicy",
+#     "alternative",
+#     "audience",
+#     "available",
+#     "bibliographicCitation",
+#     "conformsTo",
+#     "created",
+#     "dateAccepted",
+#     "dateCopyrighted",
+#     "dateSubmitted",
+#     "educationLevel",
+#     "extent",
+#     "hasFormat",
+#     "hasPart",
+#     "hasVersion",
+#     "instructionalMethod",
+#     "isFormatOf",
+#     "isPartOf",
+#     "isReferencedBy",
+#     "isReplacedBy",
+#     "isRequiredBy",
+#     "issued",
+#     "isVersionOf",
+#     "license",
+#     "mediator",
+#     "medium",
+#     "modified",
+#     "provenance",
+#     "references",
+#     "replaces",
+#     "requires",
+#     "rightsHolder",
+#     "spatial",
+#     "tableOfContents",
+#     "temporal",
+#     "valid",
+# ]
+
+# DCMI_TYPES = [
+#     "Collection",
+#     "Dataset",
+#     "Event",
+#     "Image",
+#     "InteractiveResource",
+#     "MovingImage",
+#     "PhysicalObject",
+#     "Service",
+#     "Software",
+#     "Sound",
+#     "StillImage",
+#     "Text",
+# ]
+
+NAMESPACES = {
+    "dc": "http://purl.org/dc/elements/1.1/",
+    "dcterms": "http://purl.org/dc/terms/",
+    "dcmitype": "http://purl.org/dc/dcmitype/",
+    "rdf": "http://www.w3.org/1999/02/22-rdf",
+    "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
+    "xml": "http://www.w3.org/XML/1998/namespace",
+    "xsd": "http://www.w3.org/2001/XMLSchema#",
+    "oai_dc": "http://www.openarchives.org/OAI/2.0/oai_dc/",
+}
+
+
+class DublinCore:
+    """Represents data from DC.xml and provides some methods to access it."""
+
+    def __init__(
+        self, path: Path, lookup_order: tuple = ("en", "de", "fr", "es", "it")
+    ):
+        self.path: Path = path
+        self.lookup_order: list[str] = list(lookup_order)
+        self._data: dict[str, Any] = {}  # [element][lang] = text
+        self._parse(path)
+
+    def _parse(self, path: Path):
+        tree = ET.parse(path)
+        root = tree.getroot()
+
+        for elem in DC_ELEMENTS:
+            for child in root.findall(f"dc:{elem}", namespaces=NAMESPACES):
+                lang = child.attrib.get(f"{{{NAMESPACES['xml']}}}lang", "unspecified")
+                element = self._data.get(elem, {})
+                values = element.get(lang, [])
+                if child.text is not None:
+                    values.append(child.text)
+                element[lang] = values
+                self._data[elem] = element
+        # TODO: Add DC_TERMS and DCMI_TYPES?
+
+    def get_element(
+        self, name: str, preferred_lang: str = "en", default: str = ""
+    ) -> list[str]:
+        """Return the value of a Dublin Core element as list of strings.
+
+        This method is able to deal with multiple values of the same element in different languages.
+        Returns always a list of strings, because in DC an element can have multiple values.
+
+        :param str name: The name of the element without namespace (e.g. "title").
+        :param str preferred_lang: The preferred language of the element (eg. "de").
+            * Default value is "en".
+            * If no entry in this language is available, the function will search
+              for entries in another language, depending on the lookup_order, set during
+              object creation. If no entry is found with a specified language, the function
+              checks for an entry with no 'xml:lang' attribute.
+              If still no value is found, the default value will be returned (als list!)
+        :param str default: The default value to return if no value is found. Be aware, that this
+            value will be returned as list element, even if it is a string.
+        :return: The value(s) of the element as list of strings.
+        """
+        if name not in DC_ELEMENTS:
+            raise ValueError(f"Element {name} is not a Dublin Core element.")
+        # element not in DC.xml
+        if name not in self._data:
+            logger.debug(
+                "Element %s not found in %s. Returning default value: [%s]",
+                name,
+                self.path,
+                default,
+            )
+            return [default]
+
+        if preferred_lang not in self._data[name]:
+            for lang in self.lookup_order + ["unspecified"]:
+                if lang in self._data[name]:
+                    preferred_lang = lang
+                    logger.debug(
+                        "Preferred language %s not found in %s. Using %s instead.",
+                        preferred_lang,
+                        self.path,
+                        lang,
+                    )
+                    break
+        return self._data[name][preferred_lang]
+
+    def get_element_as_str(
+        self, name: str, preferred_lang: str = "en", default: str = ""
+    ) -> str:
+        """Return the value(s) of a Dublin Core element as string.
+
+        This is a wrapper around get_element() which returns a single
+        string instead of a list.
+
+        It is able to distinguish between different elements
+        (eg. rights is handled differently than title).
+        """
+        values = self.get_element(name, preferred_lang, default)
+        if len(values) == 1:
+            return values[0]
+        if name == "rights" and len(values) == 2:
+            # we expect the licence name first, followed by the url
+            return f"{values[0]} ({values[1]})"
+        return "; ".join(values)

gamslib/objectcsv/manage_csv.py

@@ -0,0 +1,84 @@
+"""Function do collect and update csv files."""
+
+import logging
+from pathlib import Path
+
+from gamslib.objectcsv.objectcsv import ObjectCSV
+
+from .utils import find_object_folders
+
+logger = logging.getLogger()
+
+
+def collect_csv_data(
+    object_root_dir: Path,
+    object_csv_path: Path,
+    datastream_csv_path: Path,
+) -> ObjectCSV:
+    """Collect csv data from all object folders below object_root_dir.
+
+    This function collects all data from all object.csv and all datastream.csv files
+    below root_dir.
+    The collected data is stored in two files 'object_csv_path'  and 'datastream_csv_path'.
+
+    Returns a ObjectCSV object containing all object and datastream metadata.
+    """
+    # This is were we put all collected data
+
+    all_objects_csv = ObjectCSV(object_root_dir)
+    
+    for objectfolder in find_object_folders(object_root_dir):
+        obj_csv = ObjectCSV(objectfolder)
+
+        for objmeta in obj_csv.get_objectdata():
+            all_objects_csv.add_objectdata(objmeta)
+        for dsmeta in obj_csv.get_datastreamdata():
+            all_objects_csv.add_datastream(dsmeta)
+    all_objects_csv.sort()
+    all_objects_csv.write(
+        object_csv_path, datastream_csv_path
+    )
+    return all_objects_csv
+
+
+def update_csv_files(
+    object_root_dir: Path,
+    input_dir: Path | None = None,
+    object_csv_filename: str = "all_objects.csv",
+    ds_csv_filename: str = "all_datastreams.csv",
+) -> tuple[int, int]:
+    """Update csv metadata files with data from the combined csv data.
+
+    If collected_csv_dir is None, we assume that the directory
+    containing the combined csv data is the local working directory. This is
+    where collect_csv_data() stores the data by default.
+
+    `object_csv_filename` and `ds_csv_filename` are the names of the csv files.
+    The must only be set, if the names are different from the default names.
+
+    In other words: this function updates all object and datatstream
+    metadata with data changed in the central csv files.
+
+    Returns a a tuple of ints: number of updated objects and number of updated datastreams.
+    """
+    
+    num_of_changed_objects = 0
+    num_of_changed_datastreams = 0
+
+    if input_dir is None:
+        input_dir = Path.cwd()
+    all_objects_csv = ObjectCSV(input_dir, object_csv_filename, ds_csv_filename)
+    for objectfolder in find_object_folders(object_root_dir):
+        obj_csv = ObjectCSV(objectfolder)
+        obj_csv.clear()
+        for obj_data in all_objects_csv.get_objectdata(obj_csv.object_id):
+            obj_csv.add_objectdata(obj_data)
+            num_of_changed_objects += 1
+
+        for ds_data in all_objects_csv.get_datastreamdata(obj_csv.object_id):
+            obj_csv.add_datastream(ds_data)
+            num_of_changed_datastreams += 1
+        
+        obj_csv.write()
+
+    return num_of_changed_objects, num_of_changed_datastreams

gamslib/objectcsv/objectcsv.py

@@ -0,0 +1,315 @@
+"""Provides classes to handle object and datastream data in csv files.
+
+The central class is ObjectCSV, which represents the object and datastream data.
+
+ObjectCSV is directly accessible from the objectcsv package.
+"""
+# pylint: disable=too-many-instance-attributes
+# pylint: disable=invalid-name
+
+import csv
+from dataclasses import asdict, dataclass, fields
+from pathlib import Path
+from typing import Generator
+from . import defaultvalues
+
+
+@dataclass
+class ObjectData:
+    "Represents csv data for a single object."
+
+    recid: str
+    title: str = ""
+    project: str = ""
+    description: str = ""
+    creator: str = ""
+    rights: str = ""
+    publisher: str = ""
+    source: str = ""
+    objectType: str = ""
+    
+
+    def validate(self):
+        "Validate the object data."
+        # TODO: Needs discussion
+        if not self.recid:
+            raise ValueError("recid must not be empty")
+        if not self.title:
+            raise ValueError(f"{self.recid}: title must not be empty")
+        if not self.rights:
+            raise ValueError(f"{self.recid}: rights must not be empty")
+        if not self.source:
+            raise ValueError(f"{self.recid}: source must not be empty")
+        if not self.objectType:
+            raise ValueError(f"{self.recid}: objectType must not be empty")
+
+
+@dataclass
+class DSData:
+    "Represents csv data for a single datastream of a single object."
+
+    dspath: str
+    dsid: str = ""
+    title: str = ""
+    description: str = ""
+    mimetype: str = ""
+    creator: str = ""
+    rights: str = ""
+    lang: str = ""
+
+    def __post_init__(self):
+        "Add missing values if applicable and validate."
+        self._guess_mimetype()  
+        self._guess_missing_values()
+        
+        
+
+    @property
+    def object_id(self):
+        "Return the object id of the object the datastream is part of."
+        return Path(self.dspath).parts[0]
+
+    def validate(self):
+        "Validate the datastream data."
+        if not self.dspath.strip():
+            raise ValueError(f"{self.dsid}: dspath must not be empty")
+        if not self.dsid.strip():
+            raise ValueError(f"{self.dspath}: dsid must not be empty")
+        if not self.mimetype.strip():
+            raise ValueError(f"{self.dspath}: mimetype must not be empty")
+        if not self.rights.strip():
+            raise ValueError(f"{self.dspath}: rights must not be empty")
+
+    def _guess_mimetype(self): # pylint: disable=no-self-use
+        "Guess the mimetype if it is empty."
+        # TODO!
+        if not self.mimetype:
+            self.mimetype = defaultvalues.DEFAULT_MIMETYPE
+
+    def _guess_missing_values(self):
+        "Guess missing values."
+        filename = Path(self.dspath).name
+        if not self.title:
+            if filename in defaultvalues.FILENAME_MAP:
+                self.title = defaultvalues.FILENAME_MAP[self.dsid]["title"]
+            elif self.mimetype.startswith('image/'):
+                self.title = f"Image: {self.dsid}"
+            elif self.mimetype.startswith('audio/'):
+                self.title = f"Audio: {self.dsid}"
+            elif self.mimetype.startswith('video/'):
+                self.title = f"Video: {self.dsid}"
+        
+        if not self.description:
+            if filename in defaultvalues.FILENAME_MAP:
+                self.description = defaultvalues.FILENAME_MAP[self.dsid]["description"]
+        if not self.rights:
+            self.rights = defaultvalues.DEFAULT_RIGHTS
+        if not self.creator:
+            self.creator = defaultvalues.DEFAULT_CREATOR
+        
+              
+
+@dataclass
+class ObjectCSVFile:
+    "Represents csv data for a single object."
+
+    def __init__(self):
+        self._objectdata: list[ObjectData] = []
+
+    def add_objectdata(self, objectdata: ObjectData):
+        "Add a ObjectData objects."
+        self._objectdata.append(objectdata)
+
+    def get_data(self, pid: str | None = None) -> Generator[ObjectData, None, None]:
+        """Return the objectdata objects for a given object pid.
+
+        If pid is None, return all objectdata objects.
+        Filtering by pid is only needed if we have data from multiple objects.
+        """
+        for objdata in self._objectdata:
+            if pid is None:
+                yield objdata
+            else:
+                if objdata.recid == pid:
+                    yield objdata
+
+    @classmethod
+    def from_csv(cls, csv_file: Path) -> "ObjectCSVFile":
+        "Load the object data from a csv file."
+        obj_csv_file = ObjectCSVFile()
+        with csv_file.open(encoding="utf-8", newline="") as f:
+            reader = csv.DictReader(f)
+            for row in reader:
+                obj_csv_file.add_objectdata(ObjectData(**row))
+        return obj_csv_file
+
+    def to_csv(self, csv_file: Path) -> None:
+        "Save the object data to a csv file."
+        with csv_file.open("w", encoding="utf-8", newline="") as f:
+            writer = csv.DictWriter(
+                f, fieldnames=[field.name for field in fields(ObjectData)]
+            )
+            writer.writeheader()
+            for objdata in self._objectdata:
+                writer.writerow(asdict(objdata))
+
+    def __len__(self):
+        "Return the number of objectdata objects."
+        return len(self._objectdata)
+
+
+class DatastreamsCSVFile:
+    "Represents csv data for all datastreams of a single datastream."
+
+    def __init__(self):
+        self._datastreams: list[DSData] = []
+
+    def add_datastream(self, dsdata: DSData):
+        "Add a datastream to the datastreams."
+        self._datastreams.append(dsdata)
+
+    def get_data(self, pid: str | None = None) -> Generator[DSData, None, None]:
+        """Return the datastream objects for a given object pid.
+
+        If pid is None, return all datastream objects.
+        Filtering by pid is only needed if we have data from multiple objects.
+        """
+        for dsdata in self._datastreams:
+            if pid is None:
+                yield dsdata
+            else:  # TODO: this is not object_id!
+                if dsdata.object_id == pid:
+                    yield dsdata
+
+    @classmethod
+    def from_csv(cls, csv_file: Path) -> "DatastreamsCSVFile":
+        "Load the datastream container data from a csv file."
+        ds_csv_file = DatastreamsCSVFile()
+        with csv_file.open(encoding="utf-8", newline="") as f:
+            reader = csv.DictReader(f)
+            for row in reader:
+                ds_csv_file.add_datastream(DSData(**row))
+        return ds_csv_file
+
+    def to_csv(self, csv_file: Path):
+        "Save the datastream data to a csv file."
+        self._datastreams.sort(key=lambda x: x.dspath)
+        with csv_file.open("w", encoding="utf-8", newline="") as f:
+            writer = csv.DictWriter(
+                f, fieldnames=[field.name for field in fields(DSData)]
+            )
+            writer.writeheader()
+            for dsdata in self._datastreams:
+                writer.writerow(asdict(dsdata))
+
+    def __len__(self):
+        "Return the number of datastreams."
+        return len(self._datastreams)
+
+
+@dataclass
+class ObjectCSV:
+    """Represents the object and datastream data for a single object.
+
+    The constructor expects the Path to the object directory.
+    If the csv files are not set, we assume the default filenames:
+    object.csv and datastreams.csv.
+    """
+
+    OBJECT_CSV_FILENAME = "object.csv"
+    DATASTREAM_CSV_FILENAME = "datastreams.csv"
+
+    object_dir: Path
+    object_file: str = OBJECT_CSV_FILENAME
+    datastream_file: str = DATASTREAM_CSV_FILENAME
+
+    def __post_init__(self):
+        "Check if the object directory exists and load the object and datastream data."
+        if not self.object_dir.is_dir():
+            raise FileNotFoundError(
+                f"Object directory '{self.object_dir}' does not exist."
+            )
+
+        self.obj_csv_file = self.object_dir / self.object_file
+        self.ds_csv_file = self.object_dir / self.datastream_file
+
+        if self.obj_csv_file.is_file():
+            self.object_data = ObjectCSVFile.from_csv(self.obj_csv_file)
+        else:
+            self.object_data = ObjectCSVFile()
+
+        if self.ds_csv_file.is_file():
+            self.datastream_data = DatastreamsCSVFile.from_csv(self.ds_csv_file)
+        else:
+            self.datastream_data = DatastreamsCSVFile()
+
+    def is_new(self):
+        "Return True if at least one of the csv files exist."
+        # obj_csv = self.object_dir / self.OBJECT_CSV_FILENAME
+        # ds_csv = self.object_dir / self.DATASTREAM_CSV_FILENAME
+        return not (self.obj_csv_file.exists() or self.ds_csv_file.exists())
+
+    def add_datastream(self, dsdata: DSData):
+        "Add a datastream to the object."
+        self.datastream_data.add_datastream(dsdata)
+
+    def add_objectdata(self, objectdata: ObjectData):
+        "Add a object to the object."
+        self.object_data.add_objectdata(objectdata)
+
+    def get_objectdata(
+        self, pid: str | None = None
+    ) -> Generator[ObjectData, None, None]:
+        """Return the object data for a given object pid.
+
+        If pid is None, return all object data.
+        """
+        return self.object_data.get_data(pid)
+
+    def get_datastreamdata(
+        self, pid: str | None = None
+    ) -> Generator[DSData, None, None]:
+        """Return the datastream data for a given object pid.
+
+        If pid is None, return all datastream data.
+        """
+        return self.datastream_data.get_data(pid)
+
+    def sort(self):
+        "Sort the object and datastream data."
+        self.object_data._objectdata.sort(key=lambda x: x.recid)
+        self.datastream_data._datastreams.sort(key=lambda x: x.dspath)
+
+    def write(
+        self,
+        object_csv_path: Path | None = None,
+        datastream_csv_path: Path | None = None
+    ):
+        """Save the object and datastream data to csv files.
+
+        If no explicit output files are set, we use the default filenames and write to object_dir.
+        """
+        if object_csv_path is None:
+            object_csv_path = self.obj_csv_file
+        if datastream_csv_path is None:
+            datastream_csv_path = self.ds_csv_file
+        self.object_data.to_csv(object_csv_path)
+        self.datastream_data.to_csv(datastream_csv_path)
+
+    def count_objects(self) -> int:
+        "Return the number of object data objects."
+        return len(self.object_data)
+
+    def count_datastreams(self) -> int:
+        "Return the number of datastream data objects."
+        return len(self.datastream_data)
+
+    def clear(self):
+        "Clear the object and datastream data."
+        self.object_data = ObjectCSVFile()
+        self.datastream_data = DatastreamsCSVFile()
+
+    @property
+    def object_id(self):
+        "Return the object id."
+        return self.object_dir.name

gamslib/objectcsv/utils.py

@@ -0,0 +1,21 @@
+"""Utility functions for the objectcsv module."""
+
+import logging
+from pathlib import Path
+from typing import Generator
+import warnings
+
+logger = logging.getLogger()
+
+
+def find_object_folders(root_directory: Path) -> Generator[Path, None, None]:
+    """Find all object folders below root_directory."""
+    for directory in root_directory.rglob("*"):
+        if directory.is_dir():
+            if "DC.xml" in [f.name for f in directory.iterdir()]:
+                yield directory
+            else:
+                warnings.warn(
+                    f"Skipping '{directory}' as folder does not contain a DC.xml file.",
+                    UserWarning
+                )

gamslib/objectcsv/xlsx.py

@@ -0,0 +1,60 @@
+"""Module to convert csv to xlsx files."""
+
+import csv
+from pathlib import Path
+
+import pylightxl as xl
+
+
+def read_csv(csvfile: Path, skip_header: bool = True) -> list[list[str]]:
+    """Read a csv file and return a list of rows."""
+    with open(csvfile, "r", encoding="utf-8", newline="") as f:
+        reader = csv.reader(f)
+        if skip_header:
+            next(reader)
+        return list(reader)
+
+
+def csv_to_xlsx(object_csv: Path, ds_csv: Path, output_file: Path) -> Path:
+    """Convert csv files to xlsx files.
+
+    Convert the csv files object_csv and ds_csv to xlsx files in the output_dir.
+
+    Returns Path to output file.
+    """
+    object_data = read_csv(object_csv, skip_header=False)
+    ds_data = read_csv(ds_csv, skip_header=False)
+
+    db = xl.Database()
+    db.add_ws("Object Metadata")
+    for row_id, row in enumerate(object_data, start=1):
+        for col_id, value in enumerate(row, start=1):
+            db.ws(ws="Object Metadata").update_index(row=row_id, col=col_id, val=value)
+    db.add_ws("Datastream Metadata")
+    for row_id, row_data in enumerate(ds_data, start=1):
+        for col_id, value in enumerate(row_data, start=1):
+            db.ws(ws="Datastream Metadata").update_index(
+                row=row_id, col=col_id, val=value
+            )
+    xl.writexl(fn=output_file, db=db)
+    return output_file
+
+
+def xlsx_to_csv(xlsx_path: Path, obj_csv_path: Path, ds_csv_path: Path)-> tuple[Path, Path]:
+    """Convert a xlsx metadata file to 2 csv files: object.csv and datastreams.csv.
+    
+    Return Paths to the csv files as tuple (obj_csv_path, ds_csv_path).
+    """
+    db = xl.readxl(xlsx_path)
+
+    object_data = list(db.ws(ws="Object Metadata").rows)
+    ds_data = list(db.ws(ws="Datastream Metadata").rows)
+
+    with open(obj_csv_path, "w", encoding="utf-8", newline="") as f:
+        writer = csv.writer(f)
+        writer.writerows(object_data)
+
+    with open(ds_csv_path, "w", encoding="utf-8", newline="") as f:
+        writer = csv.writer(f)
+        writer.writerows(ds_data)
+    return obj_csv_path, ds_csv_path

gamslib/projectconfiguration/__init__.py

@@ -0,0 +1,50 @@
+"""Handle project configuration.
+
+The projectconfiguration package contains the classes and functions to
+manage the configuration of a GAMS project.
+
+It tries to find a configuration, validates the configuration and provides
+all configuration inline tables as Python objects.
+"""
+
+from pathlib import Path
+from importlib import resources as impresources
+import shutil
+import logging
+import warnings
+from .configuration import Configuration, find_project_toml
+
+__all__ = ["Configuration"]
+
+logger = logging.getLogger()
+
+
+def create_configuration(objects_dir: Path) -> Path | None:
+    """Create a project.toml template file in the objects_dir directory.
+
+    It is assumed that the objects_dir is the root directory of a GAMS project
+    and that the directory exists.
+    The template file will not be created if a project.toml file already exists.
+    Return the path to the created project.toml file or None if the file already exists.
+    """
+    toml_file = objects_dir / "project.toml"
+    if toml_file.exists():
+        warnings.warn(f"'{toml_file}' already exists. Will not be re-created.", UserWarning)
+        return None
+    toml_template_file = str(
+        impresources.files("gamslib")
+        / "projectconfiguration"
+        / "resources"
+        / "project.toml"
+    )
+    shutil.copy(toml_template_file, toml_file)
+    return objects_dir / "project.toml"
+
+def load_configuration(object_root: Path, config_file: Path | str | None = None) -> Configuration:
+    """Read the configuration file and return a configuration object."""
+    if config_file is None:
+        config_file = find_project_toml(object_root)
+    if isinstance(config_file, str):
+        config_file = Path(config_file)
+
+    return Configuration.from_toml(config_file)

gamslib/projectconfiguration/configuration.py

@@ -0,0 +1,108 @@
+"Provides a configuration class"
+
+# pylint: disable=too-many-arguments
+# pylint: disable=too-few-public-methods
+# pylint: disable=too-many-positional-arguments
+
+from pathlib import Path
+import tomllib
+
+from typing import Annotated, Literal
+
+
+from pydantic import BaseModel, ValidationError, StringConstraints
+
+
+def find_project_toml(start_dir: Path) -> Path:
+    """Find the project.toml file in the start_dir or above.
+
+    Return a path object to the project.toml file.
+    If no project.toml file is found, raise a FileNotFoundError.
+    """
+    for folder in (start_dir / "a_non_existing_folder_to_include_start_dir").parents:
+        project_toml = folder / "project.toml"
+        if project_toml.exists():
+            return project_toml
+
+    # if we read this point, no project.toml has been found in object_root or above
+    # So we check if there's a project.toml in the current working directory
+    project_toml = Path.cwd() / "project.toml"
+
+    if project_toml.exists():
+        return project_toml
+    raise FileNotFoundError("No project.toml file found in or above the start_dir.")
+
+
+class Metadata(BaseModel, validate_assignment=True):
+    """Represent the 'metadata' section of the configuration file."""
+
+    project_id: Annotated[str, StringConstraints(min_length=2)]
+    creator: Annotated[str, StringConstraints(min_length=3)]
+    publisher: Annotated[str, StringConstraints(min_length=3)]
+    rights: str = ""
+
+
+class General(BaseModel, validate_assignment=True):
+    """Represent the 'general' section of the configuration file."""
+
+    dsid_keep_extension: bool = True
+    loglevel: Literal["debug", "info", "warning", "error", "critical"] = "info"
+
+
+class Configuration(BaseModel):
+    """Represent the configuration from the project toml file.
+
+    Properties can be accessed as attributes of the object and sub object:
+        eg.: metadata.rights
+    """
+
+    toml_file: Path
+    metadata: Metadata
+    general: General
+
+    @classmethod
+    def _make_readable_message(cls, cfgfile, error_type: str, loc: tuple) -> str | None:
+        """Return a readable error message or None.
+
+        Helper function which creates a readable error messages.
+
+        Returns a readable error message or None if 'type' is not known by function.
+        """
+        # There are many more types which could be handled, but are not needed yet.
+        # See: https://docs.pydantic.dev/latest/errors/validation_errors/
+        reasons = {
+            "missing": "missing required field",
+            "string_too_short": "value is too short",
+            "bool_type": "value is not a boolean",
+            "bool_parsing": "value is not a boolean",
+            "literal_error": "value is not allowed here",
+        }
+
+        loc_str = ".".join([str(e) for e in loc])
+        reason = reasons.get(error_type)
+        if reason is None:
+            return None
+        return f"Error in project TOML file '{cfgfile}'. {reason}: '{loc_str}'"
+
+    @classmethod
+    def from_toml(cls, toml_file: Path) -> "Configuration":
+        """Create a configuration object from a toml file."""
+        try:
+            with toml_file.open("r", encoding="utf-8", newline="") as tf:
+                data = tomllib.loads(tf.read())
+                data["toml_file"] = toml_file
+            return cls(**data)
+        except FileNotFoundError as e:
+            raise FileNotFoundError(
+                f"Configuration file '{toml_file.parent}' not found."
+            ) from e
+        except tomllib.TOMLDecodeError as e:
+            raise tomllib.TOMLDecodeError(
+                f"Error in project TOML file '{toml_file}': {e}"
+            ) from e
+        except ValidationError as e:
+            msg = cls._make_readable_message(
+                toml_file, e.errors()[0]["type"], e.errors()[0]["loc"]
+            )
+            raise ValueError(msg) from e
+

gamslib/projectconfiguration/resources/project.toml

@@ -0,0 +1,23 @@
+
+# [metadata] contains general metadata for the GAMS project
+[metadata]
+# the id of the project (eg.: hsa)
+project_id = ""  
+
+# The project creator (like in Dublin Core)
+creator = "" 
+
+# You might want to keep this value ("GAMS")
+publisher = "GAMS"  
+
+# Set the default license for the project. 
+# Can be overriden in project.csv and datastream.csv for single objects/data streams
+rights = "Creative Commons Attribution-NonCommercial 4.0 (https://creativecommons.org/licenses/by-nc/4.0/)"
+
+[general]
+# Normally dsid is the full filename (with extension). Set to false to strip extension for dsid.
+dsid_keep_extension = true
+
+# debug, info, warning, error or critical
+loglevel = "info"
+

gamslib-0.3.1.dist-info/METADATA

@@ -0,0 +1,94 @@
+Metadata-Version: 2.4
+Name: gamslib
+Version: 0.3.1
+Summary: Modules and subpackages used in various GAMS5 related projects
+Project-URL: Homepage, https://github.com/DHGraz/gamslib
+Project-URL: Repository, https://github.com/DHGraz/gamslib
+Project-URL: Issues, https://github.com/DHGraz/gamslib/issues
+Project-URL: Changelog, https://github.com/DHGraz/gamslib/blob/main/CHANGELOG.md
+Author-email: Gunter Vasold <gunter.vasold@uni-graz.at>, Fabio Tosques <fabio.tosques@uni-graz.at>
+License-File: LICENSE
+Keywords: GAMS
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.11
+Requires-Dist: pydantic>=2.10.1
+Requires-Dist: pylightxl>=1.61
+Description-Content-Type: text/markdown
+
+# gamslib
+
+Gamslib is a collection of GAMS related modules and packages, which are used in multiple other packages.
+
+## Installation
+
+gamslib is available on pypi.org and can be installed via pip:
+
+```
+pip install gamslib
+```
+
+## Usage
+
+As gamslib is a library, it can only we used with other code. 
+
+The main purpose is to make code reusable in other GAMS5 projects and to have
+a unique way of doing things. If you are not working on GAMS5 reated code
+(which is very likely), this library will be useless for you.
+
+Currently these subpackages are available (more to come):
+
+## objectcsv
+
+Handle object and datastream metadata in object csv files.
+
+When creating bags for GAMS, we provide some metadata in csv
+files (which are not part of the bag, btw).
+
+The objectcsv package provides tools to handle this metadata.
+
+  * The ObjectCSV class represents the object    and datastream csv data for a
+    single object. It is created by providing the path to the  object
+    directory. 
+  * The manage_csv module can be used to collect csv data from all objects into
+    a single file, which makes editing the data more efficient. It also has a
+    function to update the csv files in the object directories based on the
+    collected data.
+  * The xlsx module can be used to convert the csv files to xlsx files and vice
+    versa. This is useful for editing the data in a spreadsheet without the
+    hassles of importing and exporting the csv files, which led to encoding
+    problems in the past.
+
+## projectconfiguration
+
+This package contains a central class `Configuration` that represents the
+project configuration. To create this object, the function
+`load_configuration(OBJECT_ROOT, [PATH_TO_TOML_FILE])` should be used.
+
+The function tries to find the project configuration file, validates its
+content, and creates the central Configuration object with all sub-objects
+(Each TOML inline table is provided as its own sub-object). These sub-objects
+are currently:
+
+  * general
+  * metadata
+
+A basic configuration file can be generated via the `create_condiguration()`
+function.
+
+
+## Contributing
+
+The Github repository is ment to be a read only mirror of the work repository
+hosted on our institutional private Github server. You can use the bug tracker
+on Github, but everything else should happen in the Zimlab Github repo.
+
+
+## License
+
+[MIT](https://choosealicense.com/licenses/mit/)

gamslib-0.3.1.dist-info/RECORD

@@ -0,0 +1,17 @@
+gamslib/__init__.py,sha256=8FpAlbzlpTpbNmRd4PRcmEDb-6UxuEAFpuLTvcPoSfI,230
+gamslib/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+gamslib/objectcsv/__init__.py,sha256=B8QoKtHmtHvxfUaDr16EzSLXeJ77LJYXRuf3oKlhA1A,2097
+gamslib/objectcsv/create_csv.py,sha256=eC6Rg-04N6St9h32z_rWTQoUqFbQxt2o0hD0Lxh6640,5936
+gamslib/objectcsv/defaultvalues.py,sha256=EEjdP7xs6IS3KdOt1o9-8U7xrnJ380LDGwHfovVfkVw,913
+gamslib/objectcsv/dublincore.py,sha256=SOSlxkC5ldWKDi62ai2-5dkIZOAQf2RR6238RAQWaZU,5984
+gamslib/objectcsv/manage_csv.py,sha256=pCuT6RGvGRBmc4H41ZpRKV8FJJ4N8pmYSkTCPUtV9e0,2856
+gamslib/objectcsv/objectcsv.py,sha256=cTWqRcs1495YqfgBMppTydvGWUwqjlD8JYSFraQG6WA,10763
+gamslib/objectcsv/utils.py,sha256=CNnVK6SlpwIQxc0jC1Uayb1SSnFuu3_0glBHLTB0u9A,667
+gamslib/objectcsv/xlsx.py,sha256=6sbtlJzMu8Lbt1di2L0hKCdIOQw1Tv224vz0B_Nfdrk,2064
+gamslib/projectconfiguration/__init__.py,sha256=dhG8PKVGNpIMMzmz2L0a1uKt8juAU_hP2EpGxSZgFIU,1751
+gamslib/projectconfiguration/configuration.py,sha256=_rDLaKfxDQ_wN2f5W0s_xplIY5EohKlxvINK24pheaE,3851
+gamslib/projectconfiguration/resources/project.toml,sha256=rKsB6jEs4gbqtAaJZeMboXbuWZ7c0lLuuD2QAR_3P88,686
+gamslib-0.3.1.dist-info/METADATA,sha256=S1zLYFX9-viVcm0RnTQu7sy42JUcttRuvS-TVH99FQA,3383
+gamslib-0.3.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+gamslib-0.3.1.dist-info/licenses/LICENSE,sha256=exKVUgS3rp03ninfpZsfMw6x7VfLkXKaN2vQfgEICCw,1085
+gamslib-0.3.1.dist-info/RECORD,,

gamslib-0.3.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

gamslib-0.3.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Fabio Tosques, Gunter Vasold
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.