ose-core 0.2.5__py3-none-any.whl

ose/database/migrations/versions/bbe766649a99_initial_migration.py

@@ -0,0 +1,54 @@
+"""Initial migration
+
+Revision ID: bbe766649a99
+Revises:
+Create Date: 2024-02-29 14:23:07.749071
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import sqlite
+from sqlalchemy.engine.reflection import Inspector
+
+# revision identifiers, used by Alembic.
+revision = 'bbe766649a99'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('release')
+    op.drop_table('users')
+    # ### end Alembic commands ###
+
+
+def upgrade():
+    conn = op.get_bind()
+    inspector = Inspector.from_engine(conn)
+    tables = inspector.get_table_names()
+
+    if "users" not in tables:
+        op.create_table('users',
+                        sa.Column('id', sa.INTEGER(), nullable=False),
+                        sa.Column('github_access_token', sa.VARCHAR(length=255), nullable=True),
+                        sa.Column('github_id', sa.INTEGER(), nullable=True),
+                        sa.Column('github_login', sa.VARCHAR(length=255), nullable=True),
+                        sa.PrimaryKeyConstraint('id'))
+
+    if "release" not in tables:
+        op.create_table('release',
+                        sa.Column('id', sa.INTEGER(), nullable=False),
+                        sa.Column('state', sa.VARCHAR(length=20), nullable=True),
+                        sa.Column('running', sa.BOOLEAN(), nullable=True),
+                        sa.Column('step', sa.INTEGER(), nullable=True),
+                        sa.Column('details', sqlite.JSON(), nullable=True),
+                        sa.Column('included_files', sqlite.JSON(), nullable=True),
+                        sa.Column('start', sa.DATETIME(), nullable=True),
+                        sa.Column('started_by', sa.VARCHAR(), nullable=True),
+                        sa.Column('end', sa.DATETIME(), nullable=True),
+                        sa.Column('release_script', sqlite.JSON(), nullable=True),
+                        sa.Column('worker_id', sa.VARCHAR(length=20), nullable=True),
+                        sa.Column('local_dir', sa.VARCHAR(), nullable=True),
+                        sa.PrimaryKeyConstraint('id'))

ose/index/BucketStorage.py

@@ -0,0 +1,42 @@
+import logging
+
+import whoosh.filedb.filestore
+from .ExtendedStorage import ExtendedStorage
+
+
+# Implementation of Google Cloud Storage for index
+class BucketStorage(ExtendedStorage, whoosh.filedb.filestore.RamStorage):
+    _logger = logging.getLogger(__name__)
+
+    def __init__(self, bucket):
+        super().__init__()
+        self.bucket = bucket
+        self.filenameslist = []
+
+    def open(self) -> None:
+        self.open_from_bucket()
+
+    def save(self) -> None:
+        self.save_to_bucket()
+
+    def save_to_bucket(self):
+        for name in self.files.keys():
+            with self.open_file(name) as source:
+                self._logger.debug("Saving file %s", name)
+                blob = self.bucket.blob(name)
+                blob.upload_from_file(source)
+        for name in self.filenameslist:
+            if name not in self.files.keys():
+                blob = self.bucket.blob(name)
+                self._logger.debug("Deleting old file %s", name)
+                self.bucket.delete_blob(blob.name)
+                self.filenameslist.remove(name)
+
+    def open_from_bucket(self):
+        self.filenameslist = []
+        for blob in self.bucket.list_blobs():
+            self._logger.debug("Opening blob", blob.name)
+            self.filenameslist.append(blob.name)
+            f = self.create_file(blob.name)
+            blob.download_to_file(f)
+            f.close()

ose/index/ExtendedStorage.py

@@ -0,0 +1,9 @@
+import whoosh.filedb.filestore
+
+
+class ExtendedStorage(whoosh.filedb.filestore.Storage):
+    def load(self) -> None:
+        pass
+
+    def save(self) -> None:
+        pass

ose/index/FileStorage.py

@@ -0,0 +1,7 @@
+import whoosh.filedb.filestore
+
+from .ExtendedStorage import ExtendedStorage
+
+
+class FileStorage(ExtendedStorage, whoosh.filedb.filestore.FileStorage):
+    pass

ose/index/create_index.py

@@ -0,0 +1,90 @@
+import io
+import logging
+from typing import List, Union, Dict
+
+import openpyxl
+from whoosh.index import FileIndex
+from whoosh.qparser import MultifieldParser
+from whoosh.writing import SegmentWriter
+
+_logger = logging.getLogger(__name__)
+
+EntityData = Dict[str, str]
+
+
+def parse_input_sheet(file: Union[str, bytes, io.BytesIO]) -> List[EntityData]:
+    """
+    Parses a given Excel file to pairs of headers and data
+
+    The first line of the Excel file is interpreted as header. All following lines are combined with this header and
+    returned.
+
+    :param file: The Excel file. Either a filename or bytes of the Excel file
+    :return: Pairs of headers and data
+    """
+    if isinstance(file, bytes):
+        file = io.BytesIO(file)
+
+    wb = openpyxl.load_workbook(file)
+
+    sheet = wb.active
+    data = sheet.rows
+
+    header = [i.value for i in next(data)]
+
+    return [{header[i]: cell.value for i, cell in enumerate(row)} for row in data]
+
+
+def re_write_entity_data_set(repo_name: str, index: FileIndex, sheet_name: str, entity_data: List[EntityData]):
+    writer = index.writer(timeout=60)  # Wait 60s for the writer lock
+    mparser = MultifieldParser(["repo", "spreadsheet"],
+                               schema=index.schema)
+    writer.delete_by_query(mparser.parse("repo:" + repo_name + " AND spreadsheet:" + sheet_name))
+    writer.commit()
+    writer = index.writer(timeout=60)  # Wait 60s for the writer lock
+    for data in entity_data:
+        add_entity_data_to_index(data, repo_name, sheet_name, writer)
+
+    writer.commit(optimize=True)
+
+
+def add_entity_data_to_index(entity_data: EntityData, repo_name: str, sheet_name: str, writer: SegmentWriter):
+    rowdata = entity_data
+
+    if 'Curation status' in entity_data and str(entity_data["Curation status"]) == "Obsolete":
+        _logger.info(f"Not adding obsolete entity '{entity_data.get('ID')}' to index")
+        return
+
+    if "ID" in rowdata:
+        class_id = rowdata["ID"]
+    else:
+        class_id = None
+    if "Label" in rowdata:
+        label = rowdata["Label"]
+    else:
+        label = None
+    if "Definition" in rowdata:
+        definition = rowdata["Definition"]
+    else:
+        definition = None
+    if "Parent" in rowdata:
+        parent = rowdata["Parent"]
+    else:
+        parent = None
+    if "To be reviewed by" in rowdata:
+        to_be_reviewed_by = rowdata["To be reviewed by"]
+    else:
+        to_be_reviewed_by = None
+
+    if class_id or label or definition or parent:
+        _logger.debug(
+            f"Adding entity data '{entity_data.get('ID')}' to index for repository '{repo_name}'"
+            f"and sheet '{sheet_name}'")
+
+        writer.add_document(repo=repo_name,
+                            spreadsheet=sheet_name,
+                            class_id=(class_id if class_id else None),
+                            label=(label if label else None),
+                            definition=(definition if definition else None),
+                            parent=(parent if parent else None),
+                            tobereviewedby=(to_be_reviewed_by if to_be_reviewed_by else None))

ose/index/schema.py

@@ -0,0 +1,14 @@
+from whoosh.fields import TEXT, KEYWORD, ID, SchemaClass
+
+
+class OntologyContentSchema(SchemaClass):
+    repo = ID(stored=True)
+    spreadsheet = ID(stored=True)
+    class_id = ID(stored=True)
+    label = TEXT(stored=True)
+    definition = TEXT(stored=True)
+    parent = KEYWORD(stored=True)
+    tobereviewedby = TEXT(stored=True)
+
+
+schema = OntologyContentSchema()

ose/model/ColumnMapping.py

@@ -0,0 +1,302 @@
+import abc
+import enum
+import re
+from dataclasses import dataclass
+from typing import Optional, Callable, Union, Any, List, Tuple
+
+import urllib3.util
+
+from .Relation import Relation, OWLPropertyType
+from .TermIdentifier import TermIdentifier
+
+
+class ColumnMappingKind(enum.Enum):
+    EXCLUDED_IMPORT_ID = 20
+    VERSION_IRI = 19
+    INVERSE_OF = 18
+    IGNORE = 17
+    RELATION_TYPE = 16
+    PREFIX = 15
+    PLAIN = 14
+    IMPORTED_ID = 13
+    ROOT_ID = 12
+    ONTOLOGY_IRI = 11
+    RANGE = 10
+    DOMAIN = 9
+    ONTOLOGY_ID = 8
+    SUB_PROPERTY_OF = 7
+    SYNONYMS = 6
+    RELATION = 5
+    DISJOINT_WITH = 4
+    EQUIVALENT_TO = 3
+    SUB_CLASS_OF = 2
+    LABEL = 1
+    ID = 0
+
+
+class ColumnMapping(abc.ABC):
+
+    @abc.abstractmethod
+    def get_name(self) -> str:
+        pass
+
+    @abc.abstractmethod
+    def get_kind(self) -> ColumnMappingKind:
+        pass
+
+    def get_value(self, value: str) -> Any:
+        return value.strip()
+
+    def valid(self, value: str) -> bool:
+        return True
+
+
+@dataclass
+class SimpleColumnMapping(ColumnMapping):
+    kind: ColumnMappingKind
+    name: str
+
+    def get_name(self) -> str:
+        return self.name
+
+    def get_kind(self) -> ColumnMappingKind:
+        return self.kind
+
+
+@dataclass
+class ChoiceColumnMapping(SimpleColumnMapping):
+    choices: List[str]
+
+    def valid(self, value: str) -> bool:
+        return value is not None and (value.strip() in self.choices)
+
+
+class PrefixColumnMapping(SimpleColumnMapping):
+    _pattern = re.compile(r'"([\w\d]+):(.*)"')
+
+    def __init__(self, name: str, separator: Optional[str] = None):
+        self.name = name
+        self.separator = separator
+        self.kind = ColumnMappingKind.PREFIX
+
+    def get_value(self, value: str) -> List[Tuple[str, str]]:
+        values = [value]
+        if self.separator is not None:
+            values = value.split(self.separator)
+
+        res: List[Tuple[str, str]] = []
+        for v in values:
+            m = self._pattern.match(v.strip())
+            res.append((m.group(1).strip(), m.group(2).strip()))
+
+        return res
+
+    def valid(self, value: str) -> bool:
+        values = [value]
+        if self.separator is not None:
+            values = value.split(self.separator)
+
+        return all(self._pattern.match(v) is not None for v in values)
+
+
+@dataclass
+class ParentMapping(SimpleColumnMapping):
+    _pattern = re.compile(r"^([^\[]+)(?:\[(\w+:\d+)\]|\((\w+:\d+)\))?$")
+
+    def get_value(self, value: str) -> TermIdentifier:
+        match = self._pattern.match(value.strip())
+        id = match.group(2)
+        label = match.group(1)
+        return TermIdentifier(id=id.strip() if id is not None else None,
+                              label=label.strip() if label is not None else None)
+
+    def valid(self, value: str) -> bool:
+        return self._pattern.match(value.strip()) is not None
+
+
+@dataclass
+class ManchesterSyntaxMapping(SimpleColumnMapping):
+    pass
+
+
+@dataclass
+class IRIMapping(SimpleColumnMapping):
+    allow_curie: bool = False
+
+    def valid(self, value: str) -> bool:
+        return urllib3.util.parse_url(value.strip()) is not None or \
+            self.allow_curie and ":" in value.strip()
+
+
+@dataclass
+class TermMapping(SimpleColumnMapping):
+    require_id: Optional[bool] = False
+    require_label: Optional[bool] = False
+    separator: Optional[str] = None
+
+    _term_pattern = re.compile(r"^([^\[]*)\s*(?:\[(.*)\])?$")
+
+    def get_value(self, value: str) -> List[TermIdentifier]:
+        idents = []
+        for val in [value.strip()] if self.separator is None else value.strip().split(self.separator):
+            if val.strip() == "":
+                continue
+
+            m = self._term_pattern.match(val.strip())
+            label = m.group(1)
+            id = m.group(2)
+            idents.append(TermIdentifier(
+                id=None if id is None else id.strip(),
+                label=None if label is None else label.strip()))
+
+        return idents
+
+    def valid(self, value: str) -> bool:
+        if self.separator is None:
+            values = [value]
+        else:
+            values = value.strip().split(self.separator)
+
+        values = [v.strip() for v in values if v.strip() != ""]
+
+        for v in values:
+            m = self._term_pattern.match(v.strip())
+            if m is None or m.group(1) is None and self.require_label or m.group(2) is None and self.require_id:
+                return False
+
+        return True
+
+
+@dataclass
+class LabelMapping(SimpleColumnMapping):
+    kind = ColumnMappingKind.LABEL
+
+    _label_pattern = re.compile(r"(.*)\s*(?:\((.*)\))?")
+
+    def __init__(self, name: str):
+        self.name = name
+
+    def get_kind(self) -> ColumnMappingKind:
+        return ColumnMappingKind.LABEL
+
+    def get_value(self, value: str) -> str:
+        return self.get_label(value)
+
+    def get_label(self, value: str) -> str:
+        match = re.match(self._label_pattern, value)
+        if match is None:
+            return value.strip()
+        else:
+            return match.group(1).strip()
+
+    def get_synonyms(self, value: str) -> List[str]:
+        match = re.match(self._label_pattern, value)
+        if match is None:
+            return []
+        else:
+            return [s.strip() for s in match.group(2).split(";")]
+
+    def valid(self, value: str) -> bool:
+        return self._label_pattern.match(value) is not None
+
+
+@dataclass
+class RelationColumnMapping(ColumnMapping):
+    relation: Relation
+    name: str
+    separator: Optional[str] = None
+
+    def get_name(self) -> str:
+        return self.name
+
+    def get_kind(self) -> ColumnMappingKind:
+        return ColumnMappingKind.RELATION
+
+    def get_relation(self) -> Relation:
+        return self.relation
+
+    def get_value(self, value: str) -> List[Tuple[TermIdentifier, Any]]:
+        values = [x.strip() for x in str(value).split(self.separator)] if self.separator is not None else [
+            str(value).strip()]
+
+        if self.relation.owl_property_type == OWLPropertyType.ObjectProperty:
+            values = [TermIdentifier(label=x) for x in values]
+
+        return [(self.relation.identifier(), x) for x in values]
+
+
+class ColumnMappingFactory(abc.ABC):
+    @abc.abstractmethod
+    def maps(self, column_name: str) -> bool:
+        pass
+
+    @abc.abstractmethod
+    def create_mapping(self, origin: str, column_name: str) -> ColumnMapping:
+        pass
+
+
+@dataclass
+class SingletonMappingFactory(ColumnMappingFactory):
+    column_names: List[str]
+    mapping: ColumnMapping
+
+    def maps(self, column_name: str) -> bool:
+        return column_name in self.column_names
+
+    def create_mapping(self, origin: str, column_name: str) -> ColumnMapping:
+        return self.mapping
+
+
+@dataclass
+class PatternMappingFactory(ColumnMappingFactory):
+    pattern: re.Pattern
+    mapping_factory: Callable[[str, str, re.Match], ColumnMapping]
+
+    def maps(self, column_name: str) -> bool:
+        return re.match(self.pattern, column_name) is not None
+
+    def create_mapping(self, origin: str, column_name: str) -> ColumnMapping:
+        match = re.match(self.pattern, column_name)
+        return self.mapping_factory(origin, column_name, match)
+
+
+def singleton(excel_names: List[str], mapping: Callable[..., ColumnMapping], *args, **kwargs) -> ColumnMappingFactory:
+    return SingletonMappingFactory(excel_names, mapping(*args, **{"name": excel_names[0], **kwargs}))
+
+
+def simple(excel_names: List[str], kind: ColumnMappingKind, name: Optional[str] = None) -> ColumnMappingFactory:
+    return SingletonMappingFactory(excel_names, SimpleColumnMapping(kind, excel_names[0] if name is None else name))
+
+
+def relation(excel_name: List[str], relation: TermIdentifier, name: Optional[str] = None,
+             separator: Optional[str] = None,
+             property_type: OWLPropertyType = OWLPropertyType.AnnotationProperty) -> ColumnMappingFactory:
+    return SingletonMappingFactory(excel_name, RelationColumnMapping(
+        Relation(relation.id, relation.label, [], [], property_type, [], None, None, [], ("<schema>", 0)),
+        excel_name[0] if name is None else name, separator))
+
+
+def internal(excel_names: List[str], name: str, split: Optional[str] = None) -> ColumnMappingFactory:
+    return relation(excel_names, TermIdentifier(id=None, label=name), name, split, OWLPropertyType.Internal)
+
+
+def ignore(excel_name: str) -> ColumnMappingFactory:
+    def _ignore(*_args, **_kwargs):
+        return SimpleColumnMapping(ColumnMappingKind.IGNORE, excel_name)
+
+    return singleton([excel_name], _ignore)
+
+
+def relation_pattern(pattern: Union[str, re.Pattern],
+                     factory: Callable[[str, re.Match], TermIdentifier],
+                     separator: Optional[str] = None,
+                     relation_kind: OWLPropertyType = OWLPropertyType.AnnotationProperty) -> ColumnMappingFactory:
+    def f(origin: str, rel_name: str, match: re.Match) -> RelationColumnMapping:
+        identifier = factory(rel_name, match)
+        return RelationColumnMapping(
+            Relation(identifier.id, identifier.label, [], [], relation_kind, [], None, None, [], (origin, 0)),
+            f"REL {rel_name}",
+            separator
+        )
+
+    return PatternMappingFactory(pattern, f)

ose/model/Diff.py

@@ -0,0 +1,77 @@
+import dataclasses
+import typing
+
+from typing import Generic, List
+
+A = typing.TypeVar("A")
+
+
+@dataclasses.dataclass
+class Diff(Generic[A]):
+    field: str
+    a: A
+    b: A
+    change_type: str = "update"
+
+    @property
+    def old(self) -> A:
+        return self.a
+
+    @old.setter
+    def old(self, value: A):
+        self.a = value
+
+    @property
+    def new(self) -> A:
+        return self.b
+
+    @new.setter
+    def new(self, value: A):
+        self.b = value
+
+
+SDiff = Diff[str]
+
+PRIMITIVE_TYPES = (bool, str, int, float, type(None))
+
+
+def diff(a, b, __path="") -> List[Diff]:
+    if not (dataclasses.is_dataclass(a) or isinstance(a, dict)) or not \
+            (dataclasses.is_dataclass(b) or isinstance(b, dict)):
+        raise ValueError(f"Values must be dataclasses or dictionaries. a: '{type(a)}', b: '{type(b)}'")
+
+    if type(a) is not type(b):
+        raise ValueError(f"Values must be of same type but '{type(a)}' != '{type(b)}'")
+
+    results = []
+
+    if dataclasses.is_dataclass(a):
+        fields = [f.name for f in dataclasses.fields(a)]
+
+        a = dataclasses.asdict(a)
+        b = dataclasses.asdict(b)
+    else:
+        fields = a.keys()
+
+    for f in fields:
+        val_a = a[f]
+        val_b = b[f]
+
+        path = __path + ("." if __path else "") + f
+
+        if isinstance(val_a, PRIMITIVE_TYPES):
+            if val_a != val_b:
+                results.append(Diff(path, val_a, val_b))
+        elif isinstance(val_a, (list, tuple, set)):
+            adds = [v for v in val_b if v not in val_a]
+            removes = [v for v in val_a if v not in val_b]
+
+            for v in adds:
+                results.append(Diff(path, None, v, "add"))
+
+            for v in removes:
+                results.append(Diff(path, v, None, "remove"))
+        else:
+            results.extend(diff(val_a, val_b, path))
+
+    return results