procrustus-indexer 2.0.0__tar.gz

procrustus_indexer-2.0.0/PKG-INFO

@@ -0,0 +1,77 @@
+Metadata-Version: 2.3
+Name: procrustus-indexer
+Version: 2.0.0
+Summary: 
+Author: Rob Zeeman
+Author-email: rob.zeeman@di.huc.knaw.nl
+Requires-Python: >=3.13
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: elasticsearch (>=8.17.2,<9.0.0)
+Requires-Dist: jmespath (>=1.0.1,<2.0.0)
+Requires-Dist: pytest (>=8.3.5,<9.0.0)
+Description-Content-Type: text/markdown
+
+# Procrustus Indexer
+Python app for indexing HuC datasets.
+
+This is a first experimental version, for indexing files based on a TOML configuration.
+For now, input files need to be json.
+
+Modules will be added for:
+* XML
+* Relational databases (MySQL, Postgres)
+* Linked data (SPARQL, RDF)
+* Tabular data (CSV, MS-Excel)
+
+## Installation
+
+In order to install this library, you can use pip:
+
+```
+pip install procrustus-indexer
+```
+
+## Usage
+In order to use the indexer you need to configure it first. Use the `build_indexer`
+function to create an indexer. It requires the configuration TOML file, the name of
+the index to use and an Elasticsearch client.
+
+For this example, assume we have the following file structure:
+
+```
+json-files/
+    a.json
+    b.json
+config.toml
+```
+
+First we create the indexer:
+
+```python
+from elasticsearch import Elasticsearch
+from procrustus_indexer import build_indexer
+
+indexer = build_indexer('config.toml', 'index-name', Elasticsearch())
+```
+
+### Create index and mapping:
+We can use the indexer to generate a mapping for the Elasticsearch index.
+
+```python
+indexer.create_mapping(overwrite=True)
+```
+The `overwrite` parameter can be used to re-create the index if it already exists.
+
+### Index json files
+Now we can import our json files in Elasticsearch.
+
+```python
+indexer.import_folder("json-files")
+```
+
+or
+
+```python
+indexer.import_files(["json-files/a.json", "json-files/b.json"])
+```

procrustus_indexer-2.0.0/README.md

@@ -0,0 +1,63 @@
+# Procrustus Indexer
+Python app for indexing HuC datasets.
+
+This is a first experimental version, for indexing files based on a TOML configuration.
+For now, input files need to be json.
+
+Modules will be added for:
+* XML
+* Relational databases (MySQL, Postgres)
+* Linked data (SPARQL, RDF)
+* Tabular data (CSV, MS-Excel)
+
+## Installation
+
+In order to install this library, you can use pip:
+
+```
+pip install procrustus-indexer
+```
+
+## Usage
+In order to use the indexer you need to configure it first. Use the `build_indexer`
+function to create an indexer. It requires the configuration TOML file, the name of
+the index to use and an Elasticsearch client.
+
+For this example, assume we have the following file structure:
+
+```
+json-files/
+    a.json
+    b.json
+config.toml
+```
+
+First we create the indexer:
+
+```python
+from elasticsearch import Elasticsearch
+from procrustus_indexer import build_indexer
+
+indexer = build_indexer('config.toml', 'index-name', Elasticsearch())
+```
+
+### Create index and mapping:
+We can use the indexer to generate a mapping for the Elasticsearch index.
+
+```python
+indexer.create_mapping(overwrite=True)
+```
+The `overwrite` parameter can be used to re-create the index if it already exists.
+
+### Index json files
+Now we can import our json files in Elasticsearch.
+
+```python
+indexer.import_folder("json-files")
+```
+
+or
+
+```python
+indexer.import_files(["json-files/a.json", "json-files/b.json"])
+```

procrustus_indexer-2.0.0/procrustus_indexer/__init__.py

@@ -0,0 +1,2 @@
+from .indexer import Indexer
+from .indexer_builder import build_indexer

procrustus_indexer-2.0.0/procrustus_indexer/indexer.py

@@ -0,0 +1,92 @@
+from elasticsearch import Elasticsearch
+from elasticsearch.helpers import bulk
+
+from procrustus_indexer.parsers import Parser
+
+
+class Indexer:
+    es: Elasticsearch = None
+    config: dict
+    index_name: str
+    parser: Parser
+
+    def __init__(self, es: Elasticsearch, config: dict, parser: Parser, index_name: str):
+        self.es = es
+        self.config = config
+        self.index_name = index_name
+        self.parser = parser
+
+
+    def create_mapping(self, overwrite: bool = False) -> dict:
+        """
+        Create the elasticsearch index mapping according to config and return resulting dict.
+        :return:
+        """
+        if overwrite:
+            self.es.indices.delete(index=self.index_name, ignore=[400, 404])
+
+        properties = {}
+        for facet_name in self.config['index']['facet'].keys():
+            facet = self.config["index"]["facet"][facet_name]
+            property_type = facet.get('type', 'text')
+            if property_type == 'text':
+                properties[facet_name] = {
+                    'type': 'text',
+                    'fields': {
+                        'keyword': {
+                            'type': 'keyword',
+                            'ignore_above': 256
+                        },
+                    }
+                }
+            elif property_type == 'keyword':
+                properties[facet_name] = {
+                    'type': 'keyword',
+                }
+            elif property_type == 'number':
+                properties[facet_name] = {
+                    'type': 'integer',
+                }
+            elif property_type == 'date':
+                properties[facet_name] = {
+                    'type': 'date',
+                }
+
+        mappings = {
+            'properties': properties
+        }
+
+        settings = {
+            'number_of_shards': 1,
+            'number_of_replicas': 0
+        }
+
+        self.es.indices.create(index=self.index_name, mappings=mappings, settings=settings)
+        return mappings
+
+
+    def import_files(self, files: list[str]):
+        """
+        Import files into an elasticsearch index based on the given config.
+        :param files: list of files to import
+        :param index: Elasticsearch index
+        :return:
+        """
+        actions = []
+        for inv in files:
+            doc = {}
+            with open(inv) as f:
+                doc = self.parser.parse_file(f)
+                actions.append({'_index': self.index_name, '_id': doc['id'], '_source': doc})
+        # add to index:
+        result = bulk(self.es, actions)
+
+
+    def import_folder(self, folder: str):
+        """
+        Import all files in a folder.
+        :param folder:
+        :return:
+        """
+        input_list = glob.glob(f'{folder}/*')
+        self.import_files(input_list)

procrustus_indexer-2.0.0/procrustus_indexer/indexer_builder.py

@@ -0,0 +1,25 @@
+import tomllib
+
+from elasticsearch import Elasticsearch
+from procrustus_indexer import Indexer
+from procrustus_indexer.parsers import JsonParser
+
+
+def build_indexer(config_file: str, es_index: str, es_client: Elasticsearch) -> Indexer:
+    """
+    Build a new Indexer based on the given config
+    :param config_file: Location of the config toml file
+    :param es_index: The name of the Elasticsearch index
+    :param es_client: The Elasticsearch client
+    :return:
+    """
+    with open(config_file, "rb") as f:
+        config = tomllib.load(f)
+
+    input_format = config["index"]["input"]["format"]
+    if input_format == "json":
+        parser = JsonParser(config)
+    else:
+        raise ValueError(f"Invalid input format '{input_format}'")
+
+    return Indexer(es_client, config, parser, es_index)

procrustus_indexer-2.0.0/procrustus_indexer/parsers/__init__.py

@@ -0,0 +1,2 @@
+from .parser import Parser
+from .json_parser import JsonParser

procrustus_indexer-2.0.0/procrustus_indexer/parsers/json_parser.py

@@ -0,0 +1,35 @@
+import json
+from typing import IO
+
+import jmespath
+
+from procrustus_indexer.parsers import Parser
+
+class JsonParser(Parser):
+
+    config: dict
+
+    def __init__(self, config: dict):
+        if config["index"]["input"]["format"] != "json":
+            raise ValueError("JsonParser only supports JSON files")
+        self.config = config
+
+
+    @staticmethod
+    def resolve_path(rec, path):
+        if path.startswith("jmes:"):
+            # for jmes: 5:
+            return jmespath.search(path[5:], rec)
+
+
+    def parse_file(self, file: IO) -> dict:
+        data = json.load(file)
+
+        path_id = self.config['index']['id']['path']
+        doc_id = self.resolve_path(data, path_id)
+        doc = {'id': doc_id}
+        for key in self.config['index']['facet'].keys():
+            facet = self.config["index"]["facet"][key]
+            path_facet = facet["path"]
+            doc[key] = self.resolve_path(data, path_facet)
+        return doc

procrustus_indexer-2.0.0/procrustus_indexer/parsers/parser.py

@@ -0,0 +1,7 @@
+from abc import ABC
+from typing import IO
+
+
+class Parser(ABC):
+    def parse_file(self, file: IO) -> dict:
+        pass

procrustus_indexer-2.0.0/pyproject.toml

@@ -0,0 +1,21 @@
+[project]
+name = "procrustus-indexer"
+version = "2.0.0"
+description = ""
+authors = [
+    {name = "Rob Zeeman",email = "rob.zeeman@di.huc.knaw.nl"},
+    {name = "Meindert Kroese",email = "meindert.kroese@di.huc.knaw.nl"},
+    {name = "Jarno Bakker",email = "jarno.bakker@di.huc.knaw.nl"}
+]
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "jmespath (>=1.0.1,<2.0.0)",
+    "elasticsearch (>=8.17.2,<9.0.0)",
+    "pytest (>=8.3.5,<9.0.0)"
+]
+
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"