dapytains 0.0.1a0__py3-none-any.whl

dapitains/app/app.py

@@ -0,0 +1,285 @@
+from typing import Dict, Any, Optional
+
+try:
+    import uritemplate
+    from flask import Flask, request, Response
+    from flask_sqlalchemy import SQLAlchemy
+    import click
+except ImportError:
+    print("This part of the package can only be imported with the web requirements.")
+    raise
+
+import json
+import lxml.etree as ET
+from dapitains.tei.document import Document
+from dapitains.errors import InvalidRangeOrder
+from dapitains.app.database import db, Collection, Navigation
+from dapitains.app.navigation import get_nav, get_member_by_path
+
+
+def inject_json(collection: Collection, templates) -> Dict:
+    if collection.resource:
+        inj = {
+            "collection": templates["collection"].partial({"id": collection.identifier}).uri,
+            "document": templates["document"].partial({"resource": collection.identifier}).uri,
+        }
+        if collection.citeStructure:
+            inj["navigation"] = templates["navigation"].partial({"resource": collection.identifier}).uri
+    else:
+        inj = {"collection": templates["collection"].partial({"id": collection.identifier}).uri}
+
+    return {
+        **inj,
+        "totalParents": collection.total_parents,
+        "totalChildren": collection.total_children
+    }
+
+
+def msg_4xx(string, code=404) -> Response:
+    return Response(json.dumps({"message": string}), status=code, mimetype="application/json")
+
+
+def collection_view(
+        identifier: Optional[str],
+        nav: str,
+        templates: Dict[str, uritemplate.URITemplate]
+) -> Response:
+    """ Builds a collection view, regardless of how the parameters are received
+
+    :param identifier:
+    :param nav:
+    :param templates:
+    """
+    if not identifier:
+        coll: Collection = db.session.query(Collection).filter(~Collection.parents.any()).first()
+    else:
+        coll = Collection.query.where(Collection.identifier==identifier).first()
+    if coll is None:
+        return msg_4xx("Unknown collection")
+    out = coll.json()
+
+    if nav == 'children':
+        members = db.session.query(Collection).filter(
+            Collection.parents.any(id=coll.id)
+        ).all()
+    elif nav == 'parents':
+        members = db.session.query(Collection).filter(
+            Collection.children.any(id=coll.id)
+        ).all()
+    else:
+        return msg_4xx(f"nav parameter has a wrong value {nav}", code=400)
+
+    return Response(json.dumps({
+        "@context": "https://distributed-text-services.github.io/specifications/context/1-alpha1.json",
+        "dtsVersion": "1-alpha",
+        **out,
+        "member": [
+            member.json(inject=inject_json(member, templates=templates))
+            for member in members
+        ],
+        **inject_json(coll, templates=templates)
+    }, ), mimetype="application/ld+json", status=200)
+
+
+def document_view(resource, ref, start, end, tree) -> Response:
+    if not resource:
+        return msg_4xx("Resource parameter was not provided")
+
+    collection: Collection = Collection.query.where(Collection.identifier == resource).first()
+    if not collection:
+        return msg_4xx(f"Unknown resource `{resource}`")
+
+    nav: Navigation = Navigation.query.where(Navigation.collection_id == collection.id).first()
+    if nav is None:
+        return msg_4xx(f"The resource `{resource}` does not support navigation")
+
+    tree = tree or collection.default_tree
+
+    # Check for forbidden combinations
+    if ref or start or end:
+        if tree not in nav.references:
+            return msg_4xx(f"Unknown tree {tree} for resource `{resource}`")
+        elif ref and (start or end):
+            return msg_4xx(f"You cannot provide a ref parameter as well as start or end", code=400)
+        elif not ref and ((start and not end) or (end and not start)):
+            return msg_4xx(f"Range is missing one of its parameters (start or end)", code=400)
+
+    paths = nav.paths[tree]
+    if start and end and (start not in paths or end not in paths):
+        return msg_4xx(f"Unknown reference {start} or {end} in the requested tree.", code=404)
+    if ref and ref not in paths:
+        return msg_4xx(f"Unknown reference {ref} in the requested tree.", code=404)
+
+    if not ref and not start:
+        with open(collection.filepath) as f:
+            content = f.read()
+        return Response(content, mimetype="application/xml")
+
+    doc = Document(collection.filepath)
+    return Response(
+        ET.tostring(doc.get_passage(
+            ref_or_start=ref or start,
+            end=end,
+            tree=tree
+        ), encoding=str),
+        mimetype="application/xml"
+    )
+
+
+def navigation_view(resource, ref, start, end, tree, down, templates: Dict[str, uritemplate.URITemplate]) -> Response:
+    if not resource:
+        return msg_4xx("Resource parameter was not provided")
+
+    collection: Collection = Collection.query.where(Collection.identifier == resource).first()
+    if not collection:
+        return msg_4xx(f"Unknown resource `{resource}`")
+
+    nav: Navigation = Navigation.query.where(Navigation.collection_id == collection.id).first()
+    if nav is None:
+        return msg_4xx(f"The resource `{resource}` does not support navigation")
+
+    tree = tree or collection.default_tree
+
+    # Check for forbidden combinations
+    if ref or start or end:
+        if tree not in nav.references:
+            return msg_4xx(f"Unknown tree {tree} for resource `{resource}`")
+        elif ref and (start or end):
+            return msg_4xx(f"You cannot provide a ref parameter as well as start or end", code=400)
+        elif not ref and ((start and not end) or (end and not start)):
+            return msg_4xx(f"Range is missing one of its parameters (start or end)", code=400)
+
+    # Start the response
+    out = {
+        "@context": "https://distributed-text-services.github.io/specifications/context/1-alpha1.json",
+        "dtsVersion": "1-alpha",
+        "@type": "Navigation",
+        "@id": templates["navigation"].expand({
+            "ref": ref, "down": down, "start": start, "end": end, "tree": tree
+        }),
+        "resource": collection.json(inject={k: v.uri for k, v in templates.items()}),
+    }
+
+    refs = nav.references[tree]
+    paths = nav.paths[tree]
+
+    # Three first rows of the specs for combination of down/ref/start/end
+    if down is None:
+        if ref:
+            out["ref"] = {"@type": "CitableUnit", **get_member_by_path(refs, paths[ref])}
+        elif start and end:
+            out["start"] = {"@type": "CitableUnit", **get_member_by_path(refs, paths[start])}
+            out["end"] = {"@type": "CitableUnit", **get_member_by_path(refs, paths[end])}
+        else:
+            return msg_4xx(f"The down query parameter is required when requesting without ref or start/end", code=400)
+        return Response(json.dumps(out), mimetype="application/json", status=200)
+    elif down == 0 and start and end:
+        return msg_4xx(f"The down query parameter cannot be `0` while using start/end", code=400)
+    elif down == 0 and not ref:
+        return msg_4xx(f"The down query parameter cannot be `0` without using the `ref` parameter", code=400)
+
+    try:
+        members, start, end = get_nav(refs=refs, paths=paths, start_or_ref=start or ref, end=end, down=down)
+    except InvalidRangeOrder:
+        return msg_4xx("End reference comes before start in the document order. Interchange start and end.", code=400)
+    except Exception:
+        raise
+
+    out["member"] = members
+    if end:
+        out["start"] = start
+        out["end"] = end
+    elif start:
+        out["ref"] = start
+
+    return Response(json.dumps(out), mimetype="application/ld+json", status=200)
+
+
+def create_app(
+        app: Flask,
+        base_uri: str,
+        use_query: bool = False
+) -> (Flask, SQLAlchemy):
+    """
+
+    Initialisation of the DB is up to you
+    """
+    navigation_template = uritemplate.URITemplate(base_uri+"/navigation/{?resource}{&ref,start,end,tree,down}")
+    collection_template = uritemplate.URITemplate(base_uri+"/collection/{?id}{&nav}")
+    document_template = uritemplate.URITemplate(base_uri+"/document/{?resource}{&ref,start,end,tree}")
+
+    @app.route("/")
+    def index_route():
+        return Response(
+            json.dumps({
+                "@context": "https://distributed-text-services.github.io/specifications/context/1-alpha1.json",
+                "dtsVersion": "1-alpha",
+                "@id": f"{request.url_root}{request.path}",
+                "@type": "EntryPoint",
+                "collection": collection_template.uri,
+                "navigation": navigation_template.uri,
+                "document": document_template.uri
+            }),
+            mimetype="application/ld+json"
+        )
+
+    @app.route("/collection/")
+    def collection_route():
+        resource = request.args.get("id")
+        nav = request.args.get("nav", "children")
+
+        return collection_view(resource, nav, templates={
+            "navigation": navigation_template,
+            "collection": collection_template,
+            "document": document_template,
+        })
+
+    @app.route("/navigation/")
+    def navigation_route():
+        resource = request.args.get("resource")
+        ref = request.args.get("ref")
+        start = request.args.get("start")
+        end = request.args.get("end")
+        tree = request.args.get("tree")
+        down = request.args.get("down", type=int, default=None)
+
+        return navigation_view(resource, ref, start, end, tree, down, templates={
+            "navigation": navigation_template.partial({"resource": resource}),
+            "collection": collection_template.partial({"id": resource}),
+            "document": document_template.partial({"resource": resource}),
+        })
+
+    @app.route("/document/")
+    def document_route():
+        resource = request.args.get("resource")
+        ref = request.args.get("ref")
+        start = request.args.get("start")
+        end = request.args.get("end")
+        tree = request.args.get("tree")
+        return document_view(resource, ref, start, end, tree)
+
+    return app, db
+
+
+if __name__ == "__main__":
+    import os
+    from dapitains.app.ingest import store_catalog
+    from dapitains.metadata.xml_parser import parse
+
+    app = Flask(__name__)
+    _, db = create_app(app, base_uri="http://localhost:5000")
+
+    basedir = os.path.abspath(os.path.dirname(__file__))
+    db_path = os.path.join(basedir, 'app.db')
+    app.config['SQLALCHEMY_DATABASE_URI'] = f'sqlite:///{db_path}'
+    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+
+    db.init_app(app)
+    with app.app_context():
+        db.drop_all()
+        db.create_all()
+
+        catalog, _ = parse(f"{basedir}/../../tests/catalog/example-collection.xml")
+        store_catalog(catalog)
+
+    app.run()

dapitains/app/database.py

@@ -0,0 +1,152 @@
+from collections import defaultdict
+
+try:
+    from flask_sqlalchemy import SQLAlchemy
+    from sqlalchemy.ext.mutable import MutableDict, Mutable
+    from sqlalchemy.types import TypeDecorator, TEXT
+    from sqlalchemy import func
+    import click
+except ImportError:
+    print("This part of the package can only be imported with the web requirements.")
+    raise
+
+from typing import Optional, Dict, Any
+import dapitains.metadata.classes as abstracts
+import json
+
+
+class CustomKeyJSONDecoder(json.JSONDecoder):
+    def __init__(self, *args, **kwargs):
+        super().__init__(object_hook=self.object_hook, *args, **kwargs)
+
+    def object_hook(self, obj):
+        # Only convert 'None' string keys back to None
+        return {None if k == 'null' else k: v for k, v in obj.items()}
+
+db = SQLAlchemy()
+
+parent_child_association = db.Table('parent_child_association',
+    db.Column('parent_id', db.Integer, db.ForeignKey('collections.id'), primary_key=True),
+    db.Column('child_id', db.Integer, db.ForeignKey('collections.id'), primary_key=True)
+)
+
+
+class JSONEncoded(TypeDecorator):
+    """Enables JSON storage by encoding and decoding on the fly."""
+    impl = TEXT
+
+    def process_bind_param(self, value, dialect):
+        if value is None:
+            return None
+        else:
+            return json.dumps(value)
+
+    def process_result_value(self, value, dialect):
+        if value is None:
+            return None
+        return json.loads(value, cls=CustomKeyJSONDecoder)
+
+
+class Collection(db.Model):
+    __tablename__ = 'collections'
+
+    id = db.Column(db.Integer, primary_key=True, autoincrement=True, nullable=False)
+    identifier = db.Column(db.String, nullable=False, unique=True)
+    title = db.Column(db.String, nullable=False)
+    description = db.Column(db.String, nullable=True)
+    resource = db.Column(db.Boolean, default=False)
+    filepath = db.Column(db.String, nullable=True)
+    dublin_core = db.Column(JSONEncoded, nullable=True)
+    extensions = db.Column(JSONEncoded, nullable=True)
+    citeStructure = db.Column(JSONEncoded, nullable=True)
+    default_tree = db.Column(db.String, nullable=True)
+
+    # One-to-one relationship with Navigation
+    navigation = db.relationship('Navigation', uselist=False, backref='collection', lazy=True)
+
+    parents = db.relationship(
+        'Collection',
+        secondary=parent_child_association,
+        primaryjoin=id == parent_child_association.c.child_id,
+        secondaryjoin=id == parent_child_association.c.parent_id,
+        backref='children'
+    )
+
+    @property
+    def total_children(self):
+        return db.session.query(func.count(parent_child_association.c.child_id)).filter(
+            parent_child_association.c.parent_id == self.id
+        ).scalar()
+
+    @property
+    def total_parents(self):
+        return db.session.query(func.count(parent_child_association.c.parent_id)).filter(
+            parent_child_association.c.child_id == self.id
+        ).scalar()
+
+    def json(self, inject: Optional[Dict[str, Any]] = None):
+        data = {
+            "@type": "Resource" if self.resource else "Collection",
+            "@id": self.identifier,
+            "title": self.title,
+            **(inject or {})
+        }
+        if self.description:
+            data["description"] = self.description
+        if self.resource:
+            data["citationTrees"] = []
+        if self.citeStructure:
+            data["citationTrees"] = [self.citeStructure[self.default_tree]]
+            if len(self.citeStructure) >= 1:
+                data["citationTrees"][0]["identifier"] = self.default_tree
+                for key in self.citeStructure:
+                    if key != self.default_tree:
+                        data["citationTrees"].append(self.citeStructure[key])
+                        self.citeStructure[key]["identifier"] = key
+            for tree in data["citationTrees"]:
+                tree["@type"] = "CitationTree"
+        if self.dublin_core:  # ToDo: Fix the way it's presented to adapt to dts view
+            data["dublinCore"] = self.dublin_core
+        if self.extensions:
+            data["extensions"] = self.extensions
+
+        return data
+
+    @classmethod
+    def from_class(cls, obj: abstracts.Collection) -> "Collection":
+        dublin_core = defaultdict(list)
+        for dublin in obj.dublin_core:
+            if dublin.language:
+                dublin_core[dublin.term].append({"lang": dublin.language, "value": dublin.value})
+            else:
+                dublin_core[dublin.term].append(dublin.value)
+
+        extensions = defaultdict(list)
+        for exte in obj.extensions:
+            if exte.language:
+                extensions[exte.term].append({"lang": exte.language, "value": exte.value})
+            else:
+                extensions[exte.term].append(exte.value)
+
+        obj = cls(
+            identifier=obj.identifier,
+            title=obj.title,
+            description=obj.description,
+            resource=obj.resource,
+            filepath=obj.filepath,
+            # We are dumping because it's not read or accessible
+            dublin_core=dublin_core,  #[dub.json() for dub in obj.dublin_core],
+            extensions=extensions,  # [ext.json() for ext in obj.extension]
+        )
+        return obj
+
+
+class Navigation(db.Model):
+    __tablename__ = 'navigations'
+
+    id = db.Column(db.Integer, primary_key=True, autoincrement=True, nullable=False)
+    collection_id = db.Column(db.Integer, db.ForeignKey('collections.id'), nullable=False, unique=True)
+
+    # JSON fields stored as TEXT
+    paths = db.Column(JSONEncoded, nullable=False, default={})
+    references = db.Column(JSONEncoded, nullable=False, default={})

dapitains/app/ingest.py

@@ -0,0 +1,46 @@
+from typing import Dict, Optional
+from dapitains.app.database import Collection, Navigation, db, parent_child_association
+from dapitains.app.navigation import generate_paths
+from dapitains.metadata.xml_parser import Catalog
+from dapitains.tei.document import Document
+import tqdm
+
+
+def store_single(catalog: Catalog, keys: Optional[Dict[str, int]]):
+    keys = keys or {}
+    for identifier, collection in tqdm.tqdm(catalog.objects.items(), desc="Parsing all collections"):
+        coll_db = Collection.from_class(collection)
+        db.session.add(coll_db)
+        db.session.flush()
+        keys[coll_db.identifier] = coll_db.id
+        if collection.resource:
+            doc = Document(collection.filepath)
+            if doc.citeStructure:
+                references = {
+                    tree: [ref.json() for ref in obj.find_refs(doc.xml, structure=obj.structure)]
+                    for tree, obj in doc.citeStructure.items()
+                }
+                paths = {key: generate_paths(tree) for key, tree in references.items()}
+                nav = Navigation(collection_id=coll_db.id, paths=paths, references=references)
+                db.session.add(nav)
+                coll_db.citeStructure = {
+                    key: value.structure.json()
+                    for key, value in doc.citeStructure.items()
+                }
+                coll_db.default_tree = doc.default_tree
+                db.session.add(coll_db)
+        db.session.commit()
+
+    for parent, child in catalog.relationships:
+        insert_statement = parent_child_association.insert().values(
+            parent_id=keys[parent],
+            child_id=keys[child]
+        )
+        db.session.execute(insert_statement)
+    db.session.commit()
+
+
+def store_catalog(*catalogs):
+    keys = {}
+    for catalog in catalogs:
+        store_single(catalog, keys)

dapitains/app/navigation.py

@@ -0,0 +1,135 @@
+from typing import List, Dict, Any, Optional, Tuple
+from dapitains.errors import InvalidRangeOrder
+
+
+def get_member_by_path(data: List[Dict[str, Any]], path: List[int]) -> Optional[Dict[str, Any]]:
+    """
+    Retrieve the member at the specified path in the nested data structure.
+
+    :param data: The nested data structure (list of dictionaries).
+    :param path: A list of indices that represent the path to the desired member.
+    :return: The member at the specified path, or None if the path is invalid.
+    """
+    current_level = data
+
+    path_copy = [] + path
+    while path_copy:
+        index = path_copy.pop(0)
+        try:
+            current_level = current_level[index]
+            if 'members' in current_level and path_copy:
+                current_level = current_level['members']
+        except (IndexError, KeyError):
+            return None
+
+    return current_level
+
+
+def strip_members(obj: Dict[str, Any], add_type: bool = False) -> Dict[str, Any]:
+    obj = {k: v for k, v in obj.items() if k != "members"}
+    if add_type:
+        obj["@type"] = "CitableUnit"
+    return obj
+
+
+def generate_paths(data: List[Dict[str, Any]], path: Optional[List[int]] = None) -> Dict[str, List[int]]:
+    """
+    Generate a dictionary mapping each 'ref' in a nested data structure to its path.
+
+    The path is represented as a list of indices that show how to access each 'ref'
+    in the nested structure.
+
+    :param data: The nested data structure (list of dictionaries). Each dictionary
+                 can have a 'ref' and/or 'members' key.
+    :param path: A list of indices representing the current path in the nested data
+                 structure. Used internally for recursion. Defaults to None for the
+                 initial call.
+    :return: A dictionary where each key is a 'ref' and each value is a list of indices
+             representing the path to that 'ref' in the nested structure.
+    """
+    if path is None:
+        path = []
+
+    paths = {}
+
+    def recurse(items, current_path):
+        for index, item in enumerate(items):
+            ref = item.get('identifier')
+            if ref:
+                # Record the path for the current reference
+                paths[ref] = current_path + [index]
+
+            members = item.get('members')
+            if members:
+                # Recurse into the 'members' list
+                recurse(members, current_path + [index])
+
+    recurse(data, [])
+    return paths
+
+
+def get_nav(
+        refs: List[Dict[str, Any]],
+        paths: Dict[str, List[int]],
+        start_or_ref: Optional[str] = None,
+        end: Optional[str] = None,
+        down: Optional[int] = 1
+) -> Tuple[List[Dict[str, Any]], Optional[Dict[str, Any]], Optional[Dict[str, Any]]]:
+    """ Given a references set and a path set, provide the CitableUnit from start to end at down level.
+
+    """
+
+    paths_index = list(paths.keys())
+    start_index, end_index = None, len(paths_index)
+
+    if end:
+        # For end, as end is inclusive, we check for the last partial match
+        #   (ie, if Mark is [1], we want everything starting
+        #   by [1].)
+        end_index = paths_index.index(end)
+        len_end = len(paths[end])
+        for idx, reference in enumerate(paths_index[end_index+1:]):
+            if paths[reference][:len_end] == paths[end]:
+                end_index = end_index+idx
+            else:
+                break
+
+    if start_or_ref:
+        start_index = paths_index.index(start_or_ref)
+        if not end:
+            if down == 0:
+                end_index = len(paths_index)
+            else:
+                for index, reference in enumerate(paths_index[start_index+1:]):
+                    if len(paths[start_or_ref]) == len(paths[reference]):
+                        end_index = index + start_index
+        if start_index > end_index:
+            raise InvalidRangeOrder
+
+    paths = dict(list(paths.items())[start_index:end_index+1])
+
+    current_level = []
+    start_path, end_path = None, None
+    if start_or_ref:
+        start_path = paths[start_or_ref]
+        current_level.append(len(start_path))
+    if end:
+        end_path = paths[end]
+        current_level.append(len(end_path))
+
+    current_level = max(current_level) if current_level else 0
+
+    if down == 0:
+        paths = {key: value for key, value in paths.items() if len(value) == current_level}
+    elif down == -1:
+        paths = {key: value for key, value in paths.items() if current_level <= len(value)}
+    else:
+        paths = {key: value for key, value in paths.items() if current_level <= len(value) <= down + current_level}
+
+    return (
+        [
+            strip_members(get_member_by_path(refs, path), add_type=True) for path in paths.values()
+        ],
+        strip_members(get_member_by_path(refs, start_path), add_type=True) if start_path else None,
+        strip_members(get_member_by_path(refs, end_path), add_type=True) if end_path else None
+    )

dapitains/constants.py

@@ -0,0 +1,36 @@
+import logging
+import os
+
+try:
+    saxon_version = os.getenv("pysaxon", "HE")
+    saxon_license = os.getenv("pysaxon_license", "")
+    logging.info(f"Using SaxonLib {saxon_version}")
+    if saxon_version == "HE":
+        import saxonche as saxonlib
+        PROCESSOR = saxonlib.PySaxonProcessor()
+    elif saxon_version == "PE":
+        import saxoncpe as saxonlib
+        PROCESSOR = saxonlib.PySaxonProcessor(license=saxon_license)
+    elif saxon_version == "PE":
+        import saxoncee as saxonlib
+        PROCESSOR = saxonlib.PySaxonProcessor(license=saxon_license)
+except ImportError:
+    print("Unable to import the required PySaxonC version, resorting to PySaxonC-HE")
+    import saxonche as saxonlib
+    PROCESSOR = saxonlib.PySaxonProcessor()
+
+PROCESSOR.set_configuration_property("indent-space", "0")
+
+
+
+
+def get_xpath_proc(elem: saxonlib.PyXdmNode) -> saxonlib.PyXPathProcessor:
+    """ Builds an XPath processor around a given element, with the default TEI namespace
+
+    :param elem: An XML node, root or not
+    :return: XPathProccesor
+    """
+    xpath = PROCESSOR.new_xpath_processor()
+    xpath.declare_namespace("", "http://www.tei-c.org/ns/1.0")
+    xpath.set_context(xdm_item=elem)
+    return xpath

dapitains/errors.py

@@ -0,0 +1,5 @@
+class UnknownTreeName(Exception):
+    """This exception is raised when a requested tree is unknown """
+
+class InvalidRangeOrder(Exception):
+    """Error raised when a range is in the wrong order (start > end) """