esgvoc 0.4.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

esgvoc/api/search.py

@@ -76,18 +76,45 @@ def instantiate_pydantic_terms(db_terms: Iterable[UTerm | PTerm],
         list_to_populate.append(term)
 
 
+def process_expression(expression: str) -> str:
+    """
+    Allows only SQLite FST operators AND OR NOT and perform prefix search for single word expressions.
+    """
+    # 1. Remove single and double quotes.
+    result = expression.replace('"', '')
+    result = result.replace("'", '')
+
+    # 2. Escape keywords.
+    result = result.replace('NEAR', '"NEAR"')
+    result = result.replace('+', '"+"')
+    result = result.replace('-', '"-"')
+    result = result.replace(':', '":"')
+    result = result.replace('^', '"^"')
+    result = result.replace('(', '"("')
+    result = result.replace(')', '")"')
+    result = result.replace(',', '","')
+
+    # 3. Make single word request a prefix search.
+    if not result.endswith('*'):
+        tokens = result.split(sep=None)
+        if len(tokens) == 1:
+            result += '*'
+    return result
+
+
 def generate_matching_condition(cls: type[UTermFTS5] | type[UDataDescriptorFTS5] |
                                 type[PTermFTS5] | type[PCollectionFTS5],
                                 expression: str,
                                 only_id: bool) -> ColumnElement[bool]:
+    processed_expression = process_expression(expression)
     # TODO: fix this when specs will ba available in collections and Data descriptors.
     if cls is PTermFTS5 or cls is UTermFTS5:
         if only_id:
-            result = col(cls.id).match(expression)
+            result = col(cls.id).match(processed_expression)
         else:
-            result = col(cls.specs).match(expression)  # type: ignore
+            result = col(cls.specs).match(processed_expression)  # type: ignore
     else:
-        result = col(cls.id).match(expression)
+        result = col(cls.id).match(processed_expression)
     return result
 
 

esgvoc/api/universe.py

@@ -13,6 +13,7 @@ from esgvoc.api.search import (
     handle_rank_limit_offset,
     instantiate_pydantic_term,
     instantiate_pydantic_terms,
+    process_expression,
 )
 from esgvoc.core.db.models.universe import UDataDescriptor, UDataDescriptorFTS5, UTerm, UTermFTS5
 
@@ -211,12 +212,15 @@ def find_data_descriptors_in_universe(expression: str,
                                       offset: int | None = None) -> list[tuple[str, dict]]:
     """
     Find data descriptors in the universe based on a full text search defined by the given `expression`.
-    The `expression` comes from the powerful
-    `SQLite FTS extension <https://sqlite.org/fts5.html#full_text_query_syntax>`_
-    and corresponds to the expression of the `MATCH` operator.
-    It can be composed of one or multiple keywords combined with boolean
-    operators (`NOT`, `AND`, `^`, etc. default is `OR`). Keywords can define prefixes or postfixes
-    with the wildcard `*`.
+    The `expression` can be composed of one or multiple keywords.
+    The keywords can combined with boolean operators: `AND`,
+    `OR` and `NOT` (case sensitive). The keywords are separated by whitespaces,
+    if no boolean operators is provided, whitespaces are handled as if there were
+    an implicit AND operator between each pair of keywords. Note that this
+    function does not provide any priority operator (parenthesis).
+    Keywords can define prefixes when adding a `*` at the end of them.
+    If the expression is composed of only one keyword, the function
+    automatically defines it as a prefix.
     The function returns a list of data descriptor ids and contexts, sorted according to the
     bm25 ranking metric (list index `0` has the highest rank).
     If the provided `expression` does not hit any data descriptor, the function returns an empty list.
@@ -266,12 +270,15 @@ def find_terms_in_universe(expression: str,
                            selected_term_fields: Iterable[str] | None = None) -> list[DataDescriptor]:
     """
     Find terms in the universe based on a full-text search defined by the given `expression`.
-    The `expression` comes from the powerful
-    `SQLite FTS extension <https://sqlite.org/fts5.html#full_text_query_syntax>`_
-    and corresponds to the expression of the `MATCH` operator.
-    It can be composed of one or multiple keywords combined with boolean
-    operators (`NOT`, `AND`, `^`, etc. default is `OR`). Keywords can define prefixes or postfixes
-    with the wildcard `*`.
+    The `expression` can be composed of one or multiple keywords.
+    The keywords can combined with boolean operators: `AND`,
+    `OR` and `NOT` (case sensitive). The keywords are separated by whitespaces,
+    if no boolean operators is provided, whitespaces are handled as if there were
+    an implicit AND operator between each pair of keywords. Note that this
+    function does not provide any priority operator (parenthesis).
+    Keywords can define prefixes when adding a `*` at the end of them.
+    If the expression is composed of only one keyword, the function
+    automatically defines it as a prefix.
     The function returns a list of term instances sorted according to the
     bm25 ranking metric (list index `0` has the highest rank).
     If the provided `expression` does not hit any term, the function returns an empty list.
@@ -323,12 +330,15 @@ def find_terms_in_data_descriptor(expression: str, data_descriptor_id: str,
                                                                             -> list[DataDescriptor]:
     """
     Find terms in the given data descriptor based on a full-text search defined by the given `expression`.
-    The `expression` comes from the powerful
-    `SQLite FTS extension <https://sqlite.org/fts5.html#full_text_query_syntax>`_
-    and corresponds to the expression of the `MATCH` operator.
-    It can be composed of one or multiple keywords combined with boolean
-    operators (`NOT`, `AND`, `^`, etc. default is `OR`). Keywords can define prefixes or postfixes
-    with the wildcard `*`.
+    The `expression` can be composed of one or multiple keywords.
+    The keywords can combined with boolean operators: `AND`,
+    `OR` and `NOT` (case sensitive). The keywords are separated by whitespaces,
+    if no boolean operators is provided, whitespaces are handled as if there were
+    an implicit AND operator between each pair of keywords. Note that this
+    function does not provide any priority operator (parenthesis).
+    Keywords can define prefixes when adding a `*` at the end of them.
+    If the expression is composed of only one keyword, the function
+    automatically defines it as a prefix.
     The function returns a list of term instances sorted according to the
     bm25 ranking metric (list index `0` has the highest rank).
     This function performs an exact match on the `data_descriptor_id`,
@@ -370,12 +380,16 @@ def find_items_in_universe(expression: str,
                            offset: int | None = None) -> list[Item]:
     """
     Find items, at the moment terms and data descriptors, in the universe based on a full-text
-    search defined by the given `expression`. The `expression` comes from the powerful
-    `SQLite FTS extension <https://sqlite.org/fts5.html#full_text_query_syntax>`_
-    and corresponds to the expression of the `MATCH` operator.
-    It can be composed of one or multiple keywords combined with boolean
-    operators (`NOT`, `AND`, `^`, etc. default is `OR`). Keywords can define prefixes or postfixes
-    with the wildcard `*`.
+    search defined by the given `expression`.
+    The `expression` can be composed of one or multiple keywords.
+    The keywords can combined with boolean operators: `AND`,
+    `OR` and `NOT` (case sensitive). The keywords are separated by whitespaces,
+    if no boolean operators is provided, whitespaces are handled as if there were
+    an implicit AND operator between each pair of keywords. Note that this
+    function does not provide any priority operator (parenthesis).
+    Keywords can define prefixes when adding a `*` at the end of them.
+    If the expression is composed of only one keyword, the function
+    automatically defines it as a prefix.
     The function returns a list of item instances sorted according to the
     bm25 ranking metric (list index `0` has the highest rank).
     If the provided `expression` does not hit any item, the function returns an empty list.
@@ -401,23 +415,24 @@ def find_items_in_universe(expression: str,
     # TODO: execute union query when it will be possible to compute parent of terms and data descriptors.
     result = list()
     with get_universe_session() as session:
+        processed_expression = process_expression(expression)
         if only_id:
             dd_column = col(UDataDescriptorFTS5.id)
             term_column = col(UTermFTS5.id)
         else:
             dd_column = col(UDataDescriptorFTS5.id)  # TODO: use specs when implemented!
             term_column = col(UTermFTS5.specs)  # type: ignore
-        dd_where_condition = dd_column.match(expression)
+        dd_where_condition = dd_column.match(processed_expression)
         dd_statement = select(UDataDescriptorFTS5.id,
                               text("'data_descriptor' AS TYPE"),
                               text("'universe' AS TYPE"),
                               text('rank')).where(dd_where_condition)
-        term_where_condition = term_column.match(expression)
+        term_where_condition = term_column.match(processed_expression)
         term_statement = select(UTermFTS5.id,
                                 text("'term' AS TYPE"),
                                 UDataDescriptor.id,
                                 text('rank')).join(UDataDescriptor) \
                                              .where(term_where_condition)
-        result = execute_find_item_statements(session, expression, dd_statement,
+        result = execute_find_item_statements(session, processed_expression, dd_statement,
                                               term_statement, limit, offset)
         return result

esgvoc/apps/jsg/cmip6_template.json

@@ -0,0 +1,74 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://stac-extensions.github.io/cmip6/v1.0.0/schema.json#",
+  "title": "CMIP6 Extension",
+  "description": "STAC CMIP6 Extension for STAC Items and STAC Collection Summaries.",
+  "type": "object",
+  "required": [
+    "stac_extensions"
+  ],
+  "properties": {
+    "stac_extensions": {
+      "type": "array",
+      "contains": {
+        "const": "https://stac-extensions.github.io/cmip6/v1.0.0/schema.json"
+      }
+    }
+  },
+  "oneOf": [
+    {
+      "$comment": "This is the schema for STAC Items.",
+      "type": "object",
+      "required": [
+        "type",
+        "properties"
+      ],
+      "properties": {
+        "type": {
+          "const": "Feature"
+        },
+        "properties": {
+          "allOf": [
+            {
+              "$ref": "#/definitions/require_any"
+            },
+            {
+              "$ref": "#/definitions/fields"
+            }
+          ]
+        }
+      }
+    },
+    {
+      "$comment": "This is the schema for STAC Collections, or more specifically only Collection Summaries in this case. By default, only checks the existence of the properties, but not the schema of the summaries.",
+      "type": "object",
+      "required": [
+        "type",
+        "summaries"
+      ],
+      "properties": {
+        "type": {
+          "const": "Collection"
+        },
+        "summaries": {
+          "$ref": "#/definitions/require_any"
+        }
+      }
+    }
+  ],
+  "definitions": {
+    "require_any": {
+      "$comment": "Please list all fields here so that we can force the existence of one of them in other parts of the schemas."
+    },
+    "fields": {
+      "$comment": " Don't require fields here, do that above in the corresponding schema.",
+      "type": "object",
+      "properties": {
+      },
+      "patternProperties": {
+        "^(?!cmip6:)": {}
+      },
+      "additionalProperties": false
+    }
+  }
+}

esgvoc/apps/jsg/cmip6plus_template.json

@@ -0,0 +1,74 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://stac-extensions.github.io/cmip6plus/v1.0.0/schema.json#",
+  "title": "CMIP6Plus Extension",
+  "description": "STAC CMIP6Plus Extension for STAC Items and STAC Collection Summaries.",
+  "type": "object",
+  "required": [
+    "stac_extensions"
+  ],
+  "properties": {
+    "stac_extensions": {
+      "type": "array",
+      "contains": {
+        "const": "https://stac-extensions.github.io/cmip6plus/v1.0.0/schema.json"
+      }
+    }
+  },
+  "oneOf": [
+    {
+      "$comment": "This is the schema for STAC Items.",
+      "type": "object",
+      "required": [
+        "type",
+        "properties"
+      ],
+      "properties": {
+        "type": {
+          "const": "Feature"
+        },
+        "properties": {
+          "allOf": [
+            {
+              "$ref": "#/definitions/require_any"
+            },
+            {
+              "$ref": "#/definitions/fields"
+            }
+          ]
+        }
+      }
+    },
+    {
+      "$comment": "This is the schema for STAC Collections, or more specifically only Collection Summaries in this case. By default, only checks the existence of the properties, but not the schema of the summaries.",
+      "type": "object",
+      "required": [
+        "type",
+        "summaries"
+      ],
+      "properties": {
+        "type": {
+          "const": "Collection"
+        },
+        "summaries": {
+          "$ref": "#/definitions/require_any"
+        }
+      }
+    }
+  ],
+  "definitions": {
+    "require_any": {
+      "$comment": "Please list all fields here so that we can force the existence of one of them in other parts of the schemas."
+    },
+    "fields": {
+      "$comment": " Don't require fields here, do that above in the corresponding schema.",
+      "type": "object",
+      "properties": {
+      },
+      "patternProperties": {
+        "^(?!cmip6plus:)": {}
+      },
+      "additionalProperties": false
+    }
+  }
+}

esgvoc/apps/jsg/json_schema_generator.py

@@ -0,0 +1,185 @@
+import contextlib
+import json
+from pathlib import Path
+from typing import Iterable
+
+from sqlmodel import Session
+
+from esgvoc.api import projects, search
+from esgvoc.api.project_specs import (
+    GlobalAttributeSpecBase,
+    GlobalAttributeSpecSpecific,
+    GlobalAttributeVisitor,
+)
+from esgvoc.core.constants import DRS_SPECS_JSON_KEY, PATTERN_JSON_KEY
+from esgvoc.core.db.models.project import PCollection, TermKind
+from esgvoc.core.exceptions import EsgvocNotFoundError, EsgvocNotImplementedError
+
+KEY_SEPARATOR = ':'
+JSON_SCHEMA_TEMPLATE_DIR_PATH = Path(__file__).parent
+JSON_SCHEMA_TEMPLATE_FILE_NAME_TEMPLATE = '{project_id}_template.json'
+JSON_INDENTATION = 2
+
+
+def _process_plain(collection: PCollection, selected_field: str) -> list[str]:
+    result: list[str] = list()
+    for term in collection.terms:
+        if selected_field in term.specs:
+            value = term.specs[selected_field]
+            result.append(value)
+        else:
+            raise EsgvocNotFoundError(f'missing key {selected_field} for term {term.id} in ' +
+                                      f'collection {collection.id}')
+    return result
+
+
+def _process_composite(collection: PCollection, universe_session: Session,
+                       project_session: Session) -> str:
+    result = ""
+    for term in collection.terms:
+        _, parts = projects._get_composite_term_separator_parts(term)
+        for part in parts:
+            resolved_term = projects._resolve_term(part, universe_session, project_session)
+            if resolved_term.kind == TermKind.PATTERN:
+                result += resolved_term.specs[PATTERN_JSON_KEY]
+            else:
+                raise EsgvocNotImplementedError(f'{term.kind} term is not supported yet')
+    # Patterns terms are meant to be validated individually.
+    # So their regex are defined as a whole (begins by a ^, ends by a $).
+    # As the pattern is a concatenation of plain or regex, multiple ^ and $ can exist.
+    # The later, must be removed.
+    result = result.replace('^', '').replace('$', '')
+    result = f'^{result}$'
+    return result
+
+
+def _process_pattern(collection: PCollection) -> str:
+    # The generation of the value of the field pattern for the collections with more than one term
+    # is not specified yet.
+    if len(collection.terms) == 1:
+        term = collection.terms[0]
+        return term.specs[PATTERN_JSON_KEY]
+    else:
+        msg = f"unsupported collection of term pattern with more than one term for '{collection.id}'"
+        raise EsgvocNotImplementedError(msg)
+
+
+def _generate_attribute_key(project_id: str, attribute_name) -> str:
+    return f'{project_id}{KEY_SEPARATOR}{attribute_name}'
+
+
+class JsonPropertiesVisitor(GlobalAttributeVisitor, contextlib.AbstractContextManager):
+    def __init__(self, project_id: str) -> None:
+        self.project_id = project_id
+        # Project session can't be None here.
+        self.universe_session: Session = search.get_universe_session()
+        self.project_session: Session = projects._get_project_session_with_exception(project_id)
+        self.collections: dict[str, PCollection] = dict()
+        for collection in projects._get_all_collections_in_project(self.project_session):
+            self.collections[collection.id] = collection
+
+    def __exit__(self, exception_type, exception_value, exception_traceback):
+        self.project_session.close()
+        self.universe_session.close()
+        if exception_type is not None:
+            raise exception_value
+        return True
+
+    def _generate_attribute_property(self, attribute_name: str, source_collection: str,
+                                     selected_field: str) -> tuple[str, str | list[str]]:
+        property_value: str | list[str]
+        property_key: str
+        if source_collection not in self.collections:
+            raise EsgvocNotFoundError(f"collection '{source_collection}' referenced by attribute " +
+                                      f"{attribute_name} is not found")
+        collection = self.collections[source_collection]
+        match collection.term_kind:
+            case TermKind.PLAIN:
+                property_value = _process_plain(collection=collection,
+                                                selected_field=selected_field)
+                property_key = 'enum'
+            case TermKind.COMPOSITE:
+                property_value = _process_composite(collection=collection,
+                                                    universe_session=self.universe_session,
+                                                    project_session=self.project_session)
+                property_key = 'pattern'
+            case TermKind.PATTERN:
+                property_value = _process_pattern(collection)
+                property_key = 'pattern'
+            case _:
+                msg = f"unsupported term kind '{collection.term_kind}' " + \
+                      f"for global attribute {attribute_name}"
+                raise EsgvocNotImplementedError(msg)
+        return property_key, property_value
+
+    def visit_base_attribute(self, attribute_name: str, attribute: GlobalAttributeSpecBase) \
+            -> tuple[str, dict[str, str | list[str]]]:
+        attribute_key = _generate_attribute_key(self.project_id, attribute_name)
+        attribute_properties: dict[str, str | list[str]] = dict()
+        attribute_properties['type'] = attribute.value_type.value
+        property_key, property_value = self._generate_attribute_property(attribute_name,
+                                                                         attribute.source_collection,
+                                                                         DRS_SPECS_JSON_KEY)
+        attribute_properties[property_key] = property_value
+        return attribute_key, attribute_properties
+
+    def visit_specific_attribute(self, attribute_name: str, attribute: GlobalAttributeSpecSpecific) \
+            -> tuple[str, dict[str, str | list[str]]]:
+        attribute_key = _generate_attribute_key(self.project_id, attribute_name)
+        attribute_properties: dict[str, str | list[str]] = dict()
+        attribute_properties['type'] = attribute.value_type.value
+        property_key, property_value = self._generate_attribute_property(attribute_name,
+                                                                         attribute.source_collection,
+                                                                         attribute.specific_key)
+        attribute_properties[property_key] = property_value
+        return attribute_key, attribute_properties
+
+
+def _inject_global_attributes(json_root: dict, project_id: str, attribute_names: Iterable[str]) -> None:
+    attribute_properties = list()
+    for attribute_name in attribute_names:
+        attribute_key = _generate_attribute_key(project_id, attribute_name)
+        attribute_properties.append({"required": [attribute_key]})
+    json_root['definitions']['require_any']['anyOf'] = attribute_properties
+
+
+def _inject_properties(json_root: dict, properties: list[tuple]) -> None:
+    for property in properties:
+        json_root['definitions']['fields']['properties'][property[0]] = property[1]
+
+
+def generate_json_schema(project_id: str) -> str:
+    """
+    Generate json schema for the given project.
+
+    :param project_id: The id of the given project.
+    :type project_id: str
+    :returns: The content of a json schema
+    :rtype: str
+    :raises EsgvocNotFoundError: On missing information
+    :raises EsgvocNotImplementedError: On unexpected operations
+    """
+    file_name = JSON_SCHEMA_TEMPLATE_FILE_NAME_TEMPLATE.format(project_id=project_id)
+    template_file_path = JSON_SCHEMA_TEMPLATE_DIR_PATH.joinpath(file_name)
+    if template_file_path.exists():
+        project_specs = projects.get_project(project_id)
+        if project_specs:
+            if project_specs.global_attributes_specs:
+                with open(file=template_file_path, mode='r') as file, \
+                     JsonPropertiesVisitor(project_id) as visitor:
+                    file_content = file.read()
+                    root = json.loads(file_content)
+                    properties: list[tuple[str, dict[str, str | list[str]]]] = list()
+                    for attribute_name, attribute in project_specs.global_attributes_specs.items():
+                        attribute_key, attribute_properties = attribute.accept(attribute_name, visitor)
+                        properties.append((attribute_key, attribute_properties))
+                _inject_properties(root, properties)
+                _inject_global_attributes(root, project_id, project_specs.global_attributes_specs.keys())
+                return json.dumps(root, indent=JSON_INDENTATION)
+            else:
+                raise EsgvocNotFoundError(f"global attributes for the project '{project_id}' " +
+                                          "are not provided")
+        else:
+            raise EsgvocNotFoundError(f"project '{project_id}' is not found")
+    else:
+        raise EsgvocNotFoundError(f"template for project '{project_id}' is not found")