pymetadataeditor 0.2.0__tar.gz → 0.3.1__tar.gz

{pymetadataeditor-0.2.0 → pymetadataeditor-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pymetadataeditor
-Version: 0.2.0
+Version: 0.3.1
 Summary: Metadata Editor client for python
 License: MIT
 Author: Gordon Blackadder
@@ -11,7 +11,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: markitdown (>=0.0.1a3,<0.0.2)
-Requires-Dist: metadataschemas (>=0.1.2,<0.2.0)
+Requires-Dist: metadataschemas (>=0.1.3,<0.2.0)
 Requires-Dist: openai (>=1.59.4,<2.0.0)
 Requires-Dist: pandas (>=2.2.2,<3.0.0)
 Requires-Dist: pydantic (>=2.7.4,<3.0.0)
@@ -317,9 +317,9 @@ If you are running on Windows and see errors about numpy installation errors the
 
 ### Markdown API Documentation
 
-Create documentation for the pyMetadataEditor class by running:
+Create documentation for the pyMetadataEditor class by running the following command:
 ```
-lazydocs pymetadataeditor.interface.MetadataEditor --no-watermark
+python make_docs.py
 ```
 
 ## Notes

{pymetadataeditor-0.2.0 → pymetadataeditor-0.3.1}/README.md

@@ -295,9 +295,9 @@ If you are running on Windows and see errors about numpy installation errors the
 
 ### Markdown API Documentation
 
-Create documentation for the pyMetadataEditor class by running:
+Create documentation for the pyMetadataEditor class by running the following command:
 ```
-lazydocs pymetadataeditor.interface.MetadataEditor --no-watermark
+python make_docs.py
 ```
 
 ## Notes

{pymetadataeditor-0.2.0 → pymetadataeditor-0.3.1}/pymetadataeditor/interface.py

@@ -1,6 +1,7 @@
 """The interface between pyMetadataEditor and the Metadata Editor API."""
 
 import warnings
+from io import BufferedReader
 from json import JSONDecodeError
 from pathlib import Path
 from tempfile import NamedTemporaryFile
@@ -22,6 +23,7 @@ from urllib3.exceptions import InsecureRequestWarning
 from pymetadataeditor.llm_helpers import (
     _iterated_validated_update_to_outline,
     _prepend_draft_drop_non_str,
+    call_per_field,
     get_date_as_text,
     json_to_markdown,
 )
@@ -133,7 +135,7 @@ class MetadataEditor:
 
         Args:
             api_url (str): the URL typically looks like 'https://<name_of_your_metadata_database>.org/index.php/api'
-            api_key (str): typically this is created through the web interface of the metadata system
+            api_key (str): typically this is created through the web interface of the metadata system.
             allow_http (bool): whether to allow calls to the metadata system when the URL begins "http" instead of the
                 more secure "https". Defaults to False.
             verify_ssl (bool): Although it is good practice for API requests to verify SSL, some systems do not allow
@@ -227,6 +229,13 @@ class MetadataEditor:
             projects = pd.DataFrame.from_dict(projects).set_index("id")
         except KeyError:  # is this the best way to cope with times when there are no projects?
             return pd.DataFrame(columns=["id", "created"]).set_index("id")
+
+        try:
+            new_index = projects.index.astype(int)
+        except ValueError:
+            pass
+        else:
+            projects.index = new_index
         return projects
 
     def get_project_by_id(self, id: int) -> pd.Series:
@@ -246,6 +255,45 @@ class MetadataEditor:
         response = self._apinterface.get_request(get_project_template, id=id)
         return pd.Series(response["project"])
 
+    ####################################################################################################################
+    # Generic API requests
+    ####################################################################################################################
+
+    def generic_api_request(
+        self,
+        method: str,
+        endpoint: str,
+        params: Optional[Dict] = None,
+        data: Optional[Dict] = None,
+        json: Optional[Dict] = None,
+        files: Optional[Dict[str, BufferedReader]] = None,
+    ) -> Dict:
+        """Make a generic API request to the Metadata Editor API.
+
+        It's generally better to use the specific functions such as list_projects, create_project_log etc. but this
+        function is provided for flexibility and to allow for future changes in the API.
+
+        Args:
+            method (str): Either 'POST' or 'GET'
+            endpoint (str): The path appended to the API_URL to which a GET or POST request is sent.
+            params (optional dict): additional parameters to send with a GET request.
+            data (optional dict): The data to send with a POST request.
+            json (optional dict): The JSON data to send with a POST request.
+            files (optional dict[str, BufferedReader]): The files to send with a POST request in the form
+                ```{"filename": open(filename, "rb")}```.
+
+        Returns:
+            Dict: The response from the API as a dictionary.
+        """
+        return self._apinterface._request(
+            method=method,
+            pth=endpoint,
+            params=params,
+            data=data,
+            json=json,
+            files=files,
+        )
+
     ####################################################################################################################
     # METADATA Input and Output
     ####################################################################################################################
@@ -379,12 +427,21 @@ class MetadataEditor:
         else:
             metadata_type = self._templates[template_uid]["metadata_type"]
             if "class" not in self._templates[template_uid] or apply_template_rules is False:
-                parent_schema = self._mm.metadata_class_from_name(metadata_type)
+                try:
+                    parent_schema = self._mm.metadata_class_from_name(metadata_type)
+                except ValueError:
+                    warnings.warn(
+                        f"Could not find a metadata class for template type '{metadata_type}', UID '{template_uid}'. "
+                        f"Building a class from the template without a parent schema."
+                    )
+                    parent_schema = SchemaBaseModel
+
                 klass = pydantic_from_template(
                     temp.template,
                     parent_schema=parent_schema,
                     uid=template_uid,
                     name=temp.name,
+                    metadata_type=metadata_type,
                     apply_rules=apply_template_rules,
                 )
                 if apply_template_rules:
@@ -644,7 +701,7 @@ class MetadataEditor:
         llm_model_name="gpt-4o",
         tokenizer_model="o200k_base",
         max_tokens=128_000,
-        public_llm_base_url: Optional[str] = None,
+        llm_base_url: Optional[str] = None,
         azure_llm_base_url: Optional[str] = None,
     ) -> Union[BaseModel, Dict, str]:
         """Automatically generate *draft* metadata for a project based on local files or web pages.
@@ -681,11 +738,11 @@ class MetadataEditor:
                 The option is provided in case OpenAI deprecated the 4o model.
             max_tokens (int): The maximum number of tokens to use when sending the content to OpenAI.
                 Defaults to 128_000, which has been the typical maximum for the 4o model.
-            public_llm_base_url (Optional[str]): The base URL for the LLM API. If None, the default URL is used which
+            llm_base_url (Optional[str]): The base URL for the LLM API. If None, the default URL is used which
                 sends the request to OpenAI. This argument is ignored if an azure_llm_base_url is provided.
             azure_llm_base_url (Optional[str]): The base URL for the Azure LLM API. Typically used when an organization
                 has its own deployment of an LLM model, possibly for privacy reasons. The Azure endpoint will be used
-                even if a public_llm_base_url is provided. If None, then the public_llm_base_url endpoint is used. If
+                even if a llm_base_url is provided. If None, then the llm_base_url endpoint is used. If
                 that's also None, then OpenAI is used.
 
         Returns:
@@ -706,6 +763,19 @@ class MetadataEditor:
             title="My Metadata",
         )
 
+        # Example with a local model running on Ollama:
+        me.draft_metadata_from_files(
+            llm_api_key="ollama",  # pragma: allowlist secret
+            files=["/path/to/word_file1.docx", "http://www.example.com/report.pdf"],
+            output_mode="pydantic",
+            metadata_type_or_template_uid="indicator",
+            metadata_producer_organization="My Organization",
+            filename="output.xlsx",
+            title="My Metadata",
+            llm_base_url="http://localhost:11434/v1/",
+            llm_model_name="llama3.1"
+        )
+
         # Example with an Azure instance of a Large Language Model:
         me.draft_metadata_from_files(
             llm_api_key="...",
@@ -726,7 +796,7 @@ class MetadataEditor:
         )
         enc = tiktoken.get_encoding(tokenizer_model)
         if azure_llm_base_url is None:
-            client = OpenAI(api_key=llm_api_key, base_url=public_llm_base_url)
+            client = OpenAI(api_key=llm_api_key, base_url=llm_base_url)
         else:
             client = AzureOpenAI(api_key=llm_api_key, base_url=azure_llm_base_url, api_version="2024-10-01")
 
@@ -765,29 +835,29 @@ class MetadataEditor:
                     f" {max_tokens}, truncating the content and proceeding."
                 )
             else:
-                print(f"Read in {doc}, running token count is {num_tokens}")
+                print(f"Reading {doc}, running token count is {num_tokens}")
         messages += user_message
 
         endpoint_name = (
-            azure_llm_base_url
-            if azure_llm_base_url is not None
-            else "OpenAI"
-            if public_llm_base_url is None
-            else public_llm_base_url
+            azure_llm_base_url if azure_llm_base_url is not None else "OpenAI" if llm_base_url is None else llm_base_url
         )
         print(f"Sending to {endpoint_name}, this may take a few minutes...")
-        completion = client.beta.chat.completions.parse(
-            model=llm_model_name,
-            messages=messages,
-            response_format=metadata_class_no_rules,
-        )
-
-        message = completion.choices[0].message
-        if not message.parsed:
-            raise ValueError(message.refusal)
+        try:
+            completion = client.beta.chat.completions.parse(
+                model=llm_model_name,
+                messages=messages,
+                response_format=metadata_class_no_rules,
+            )
 
-        metadata_dict = message.parsed.model_dump(exclude_none=True, exclude_unset=True)
+            message = completion.choices[0].message
+            if not message.parsed:
+                raise ValueError(message.refusal)
 
+            metadata_dict = message.parsed.model_dump(exclude_none=True, exclude_unset=True)
+        except ValidationError:
+            metadata_dict = call_per_field(metadata_class_no_rules, client, llm_model_name, messages).model_dump(
+                exclude_none=True, exclude_unset=True
+            )
         # if prefix:
         #     metadata_dict = self._prepend_draft_drop_non_str(metadata_dict, prefix)
         metadata_class_with_rules = self.get_metadata_class(metadata_type_or_template_uid)
@@ -814,7 +884,7 @@ class MetadataEditor:
         llm_model_name="gpt-4o",
         tokenizer_model="o200k_base",
         max_tokens=128_000,
-        public_llm_base_url: Optional[str] = None,
+        llm_base_url: Optional[str] = None,
         azure_llm_base_url: Optional[str] = None,
     ) -> Union[BaseModel, Dict, str]:
         """Augment existing metadata with information from files or web pages.
@@ -843,11 +913,11 @@ class MetadataEditor:
                 The option is provided in case OpenAI deprecated the 4o model.
             max_tokens (int): The maximum number of tokens to use when sending the content to OpenAI.
                 Defaults to 128_000, which has been the typical maximum for the 4o model.
-            public_llm_base_url (Optional[str]): The base URL for the LLM API. If None, the default URL is used which
+            llm_base_url (Optional[str]): The base URL for the LLM API. If None, the default URL is used which
                 sends the request to OpenAI.
             azure_llm_base_url (Optional[str]): The base URL for the Azure LLM API. Typically used when an organization
                 has its own deployment of an LLM model, possibly for privacy reasons. The Azure endpoint will be used
-                even if a public_llm_base_url is provided. If None, then the public_llm_base_url endpoint is used. If
+                even if a llm_base_url is provided. If None, then the llm_base_url endpoint is used. If
                 that's also None, then OpenAI is used.
 
         Returns:
@@ -946,7 +1016,7 @@ class MetadataEditor:
                 llm_model_name=llm_model_name,
                 tokenizer_model=tokenizer_model,
                 max_tokens=max_tokens,
-                public_llm_base_url=public_llm_base_url,
+                llm_base_url=llm_base_url,
                 azure_llm_base_url=azure_llm_base_url,
             )
 
@@ -982,7 +1052,7 @@ class MetadataEditor:
                     document, geospatial, image, indicator, indicators_db, microdata, resource, script, table, video
                 In this case we will use the default template for that metadata type.
                 Alternatively you can pass in the UID of a template. This is required if the metadata is a dictionary
-                    otherwise ignored.
+                    otherwise the UID associated with the pydantic model or the Excel file will be used.
 
         Returns:
             int: The ID of the newly created document metadata
@@ -1002,6 +1072,37 @@ class MetadataEditor:
         )
         return ret["id"]
 
+    # def log_project_admin_metadata(
+    #         self, id: int, metadata: Union[BaseModel, Dict, str], metadata_type_or_template_uid: Optional[str] = None
+    # ) -> int:
+    #     """Validates and logs admin metadata which can be a dictionary, a pydantic model or a path to an Excel file.
+
+    #     Args:
+    #         id (int): The ID of the project to associate the metadata with.
+    #         metadata (dictionary or BaseModel or str): If str, it's assumed this is a path to an appropriately
+    #             formatted Excel file.
+    #         admin_metadata_template_uid (str, optional): The UID of the admin metadata template to use.
+    #             otherwise the UID associated with the pydantic model or the Excel file will be used.
+
+    #     Returns:
+    #         int: The ID of the newly created admin metadata
+    #     """
+    #     metadata, metadata_type, uid = self._process_metadata_input(metadata, metadata_type_or_template_uid)
+    #     post_request_pth = "admin-metadata/data/"
+    #     post_json = {
+    #         "project_id": id,
+    #         "template_uid": uid,
+    #         "metadata": remove_empty_from_dict(metadata.model_dump(mode="json",
+    #                                                                exclude_none=True,
+    #                                                                exclude_unset=True)),
+    #     }
+    #     ret = self._apinterface.post_request(
+    #         pth=post_request_pth,
+    #         json=post_json,
+    #         id=id,
+    #     )
+    #     return ret
+
     def update_project_log_by_id(self, id: int, new_metadata: Union[BaseModel, Dict, str]):
         """Updates the record of the metadata.
 
@@ -1111,7 +1212,7 @@ class MetadataEditor:
         - "add": Adds a value to the specified path. If the path already exists, the value is replaced.
         - "remove": Removes the value at the specified path.
         - "replace": Replaces the value at the specified path with a new value.
-        - "test": Tests that the value at the specified path matches a given value.
+        - "test": Tests that the specified path contains the given value.
 
         The `path` is a string that uses a slash (`/`) notation to specify the location within the JSON document.
         For example, `/author` refers to the "author" field, and `/metadata/title` refers to the "title" field
@@ -1132,7 +1233,18 @@ class MetadataEditor:
         Example:
         ```python
         me = MetadataEditor(api_url = api_url, api_key = api_key)
+
+        # set the author of the project with ID 123 to "John Doe"
         me.patch_update_project_log_by_id(id=123, op="add", path="/author", value="John Doe")
+
+        # test that the author is "John Doe"
+        me.patch_update_project_log_by_id(id=123, op="test", path="/author", value="John Doe")
+
+        # change the author to "Jane Doe"
+        me.patch_update_project_log_by_id(id=123, op="replace", path="/author", value="Jane Doe")
+
+        # remove the value of author
+        me.patch_update_project_log_by_id(id=123, op="remove", path="/author")
         ```
         """
         project_data = self.get_project_by_id(id)
@@ -1144,7 +1256,7 @@ class MetadataEditor:
             patches["value"] = value
         if isinstance(patches, dict):
             patches = [patches]
-        patches = validate_json_patches(patches)  # depend on serverside validation
+        patches = validate_json_patches(patches)  # could instead only depend on serverside validation...
         self._apinterface.post_request(pth=pth + "{}", id=id, json={"patches": patches, "validate": False})
 
     ####################################################################################################################
@@ -1175,6 +1287,12 @@ class MetadataEditor:
                         return row["data_type"]
 
                 self._default_templates["data_type"] = self._default_templates.apply(standardize, axis=1)
+
+            try:
+                new_index = templates.index.astype(int)
+            except ValueError:
+                pass
+                templates.index = new_index
             return templates
 
     def get_template_by_uid(self, uid: str) -> pd.Series:
@@ -1193,10 +1311,23 @@ class MetadataEditor:
             raise KeyError(f"No result returned.\nResponse: {response}")
 
         temp = pd.Series(response["result"], name=response["result"]["name"])
-        self._templates[uid] = {"template": temp, "metadata_type": self._mm.standardize_metadata_name(temp.data_type)}
         try:
-            parent_schema = self._mm.metadata_class_from_name(temp.data_type)
-            klass = pydantic_from_template(temp.template, parent_schema=parent_schema, uid=uid, name=temp.name)
+            standard_type_name = self._mm.standardize_metadata_name(temp.data_type)
+        except ValueError:
+            standard_type_name = temp.data_type
+        self._templates[uid] = {"template": temp, "metadata_type": standard_type_name}
+        try:
+            try:
+                parent_schema = self._mm.metadata_class_from_name(temp.data_type)
+            except ValueError:
+                warnings.warn(
+                    f"Could not find a metadata class for template type '{standard_type_name}', UID '{uid}'. "
+                    f"Building a class from the template without a parent schema."
+                )
+                parent_schema = SchemaBaseModel
+            klass = pydantic_from_template(
+                temp.template, parent_schema=parent_schema, uid=uid, name=temp.name, metadata_type=standard_type_name
+            )
             self._templates[uid]["class"] = klass
         finally:
             return temp
@@ -1242,9 +1373,12 @@ class MetadataEditor:
         if not (output_template_uid is None or output_template_uid == uid):
             new_klass, new_type, _ = self._get_metadata_class_and_type_and_UID(output_template_uid)
             try:
-                metadata = new_klass.model_validate(
-                    remove_empty_from_dict(metadata.model_dump(mode="json", exclude_none=True, exclude_unset=True)),
-                    strict=False,
+                # metadata = new_klass.model_validate(
+                #     remove_empty_from_dict(metadata.model_dump(mode="json", exclude_none=True, exclude_unset=True)),
+                #     strict=False,
+                # )
+                metadata = _iterated_validated_update_to_outline(
+                    new_klass, updates=metadata.model_dump(mode="json", exclude_none=True, exclude_unset=True)
                 )
             except ValidationError as e:
                 raise TemplateError(
@@ -1332,7 +1466,14 @@ class MetadataEditor:
         response = self._apinterface.get_request("collections")
         if "collections" not in response or len(response["collections"]) == 0:
             return pd.DataFrame([], columns=["id", "title", "created"]).set_index("id")
-        return pd.DataFrame(response["collections"]).set_index("id")
+        df = pd.DataFrame(response["collections"]).set_index("id")
+        try:
+            new_index = df.index.astype(int)
+        except ValueError:
+            pass
+        else:
+            df.index = new_index
+        return df
 
     def get_collection_by_id(self, id: int) -> pd.Series:
         """Get information about a collection like title, description, created date.
@@ -1385,6 +1526,26 @@ class MetadataEditor:
             metadata["description"] = description
         self._apinterface.post_request("collections/update/{}", id=id, json=metadata)
 
+    def copy_collection(self, source_id: int, target_id: int):
+        """Copy projects and users from one collection to another.
+
+        Args:
+            source_id (int): The ID of the source collection.
+            target_id (int): The ID of the target collection.
+
+        """
+        self._apinterface.post_request("collections/copy/", json={"source_id": source_id, "target_id": target_id})
+
+    def move_collection(self, source_id, target_id):
+        """Move source collection to be a sub-collection of the target collection.
+
+        Args:
+            source_id (int): The ID of the source collection.
+            target_id (int): The ID of the target collection.
+
+        """
+        self._apinterface.post_request("collections/move/", json={"source_id": source_id, "target_id": target_id})
+
     def count_projects_in_collection(self, collection: int) -> int:
         """Count the number of projects you have access to.
 
@@ -1449,7 +1610,15 @@ class MetadataEditor:
         ret = self._apinterface.get_request("editor?collection={}", id=collection, params=params)
         if len(ret["projects"]) == 0:
             return pd.DataFrame([], columns=["id", "type", "idno", "title"]).set_index("id")
-        return pd.DataFrame(ret["projects"]).set_index("id")
+        df = pd.DataFrame(ret["projects"]).set_index("id")
+
+        try:
+            new_index = df.index.astype(int)
+        except ValueError:
+            pass
+        else:
+            df.index = new_index
+        return df
 
     def add_projects_to_collection(
         self, collection: Union[int, List[int]], id_format: str, projects: Union[int, List[int], str, List[str]]
@@ -1580,7 +1749,14 @@ class MetadataEditor:
         """
         response = self._apinterface.get_request("resources/{}", id=id)
         if "resources" in response and len(response["resources"]) > 0:
-            return pd.DataFrame(response["resources"])
+            df = pd.DataFrame(response["resources"])
+            try:
+                new_index = df.index.astype(int)
+            except ValueError:
+                pass
+            else:
+                df.index = new_index
+            return df
         else:
             return pd.DataFrame(columns=["id", "sid", "dctype", "title", "subtitle", "author", "filename", "dcformat"])
 

pymetadataeditor-0.3.1/pymetadataeditor/llm_helpers.py

@@ -0,0 +1,257 @@
+"""Helper functions for the LLM Metadata Editor when working with Pydantic models and LLM outputs."""
+
+from datetime import datetime
+from typing import Any, List, get_args
+
+from metadataschemas.utils.quick_start import make_skeleton
+from metadataschemas.utils.utils import (
+    get_subtype_of_optional_or_list,
+    is_list_annotation,
+    is_optional_annotation,
+    is_optional_list,
+    subset_pydantic_model_type,
+)
+from openai import APITimeoutError
+from pydantic import BaseModel, ValidationError
+
+# from tqdm import tqdm
+from tqdm import tqdm
+
+
+def _prepend_draft_drop_non_str(d: Any, prefix: str) -> dict | list | str | None:
+    """Recursively prepend a prefix to all strings in a dictionary or list and drop empty strings and non-strings.
+
+    Args:
+        d (Any): The dictionary or list to process.
+        prefix (str): The prefix to prepend to all strings.
+
+    Returns:
+        dict | list | str | None: The processed dictionary or list, or the string with the prefix prepended.
+    """
+    if isinstance(d, dict):
+        out = {}
+        for k, v in d.items():
+            new_v = _prepend_draft_drop_non_str(v, prefix)
+            if new_v is not None:
+                out[k] = new_v
+        if len(out) > 0:
+            return out
+        else:
+            return None
+    elif isinstance(d, list):
+        out = []
+        for v in d:
+            new_v = _prepend_draft_drop_non_str(v, prefix)
+            if new_v is not None:
+                out.append(new_v)
+        if len(out) > 0:
+            return out
+        else:
+            return None
+    elif isinstance(d, str):
+        if d.strip() == "":
+            return None
+        else:
+            return prefix + d
+    else:
+        return None
+
+
+def _iterated_validated_update_to_outline(model_def: type[BaseModel], updates: dict, verbose=False) -> BaseModel:
+    """Recursively updates a model definition with given updates and validates the result.
+
+    This function takes a model definition and a dictionary of updates, makes a skeleton and applies the updates
+    to that skeleton, and validates the updated model. If validation fails for any update value, the
+    original skeleton value is retained.
+
+    Args:
+        model_def (type[BaseModel]): The model definition class to be updated.
+        updates (dict): A dictionary containing the updates to be applied to the model.
+        verbose (bool, optional): If True, prints detailed information about validation failures. Defaults to False.
+
+    Returns:
+        BaseModel: The validated model instance with the applied updates.
+
+    Raises:
+        ValidationError: If the final model validation fails.
+
+    Steps:
+        1. Create a skeleton of the model and dump its initial state.
+        2. Validate the initial state of the model.
+        3. Iterate over the updates dictionary.
+        4. For each update, check if the key exists in the original model.
+        5. Determine the type of the field (annotation) and handle optional and list types.
+        6. Recursively update nested models if the value is a dictionary or a list of dictionaries.
+        7. Apply the candidate value to the original model.
+        8. Validate the updated model.
+        9. If validation fails, revert to the original value and optionally print the error.
+        10. Return the validated model instance.
+    """
+    original_model = make_skeleton(model_def).model_dump()
+    # print(original_model)
+    model_def.model_validate(original_model, strict=False)
+
+    model_fields = model_def.model_fields
+    for key, value in updates.items():
+        if key not in original_model:
+            continue
+
+        # get annotation aka type of the field
+        annotation = model_fields[key].annotation
+        if is_optional_annotation(annotation):
+            annotation = get_args(annotation)[0]
+        is_list = False
+        if is_list_annotation(annotation):
+            annotation = get_args(annotation)[0]
+            is_list = True
+        # print(annotation)
+
+        # get candidate value
+        if isinstance(value, dict) and issubclass(annotation, BaseModel):
+            candidate_value = _iterated_validated_update_to_outline(annotation, value, verbose=verbose).model_dump()
+        elif is_list and isinstance(value, list) and issubclass(annotation, BaseModel):
+            candidate_value = []
+            for item in value:
+                candidate_value.append(
+                    _iterated_validated_update_to_outline(annotation, item, verbose=verbose).model_dump()
+                )
+        else:
+            candidate_value = value
+
+        # try the candidate value
+        original_value = original_model[key]
+        original_model[key] = candidate_value
+        try:
+            model_def.model_validate(original_model, strict=False)
+        except ValidationError as e:
+            # If validation fails, leave the original value
+            if verbose:
+                print(f"Skipping {key}({annotation})={candidate_value} because of {e}\n")
+            original_model[key] = original_value
+        # print(original_model)
+
+    return model_def.model_validate(original_model, strict=False)
+
+
+def get_date_as_text():
+    """Returns the current date as a formatted string with an ordinal suffix for the day.
+
+    The format of the returned date string is "Month DaySuffix, Year", where:
+    - Month is the full name of the month (e.g., January, February).
+    - DaySuffix is the day of the month with an appropriate ordinal suffix (e.g., 1st, 2nd, 3rd, 4th, etc.).
+    - Year is the four-digit year.
+
+    Example:
+        If today's date is October 21, 2023, the function will return "October 21st, 2023".
+
+    Returns:
+        str: The formatted date string.
+    """
+    today = datetime.today()
+    day = today.day
+    # Determine ordinal suffix
+    if 10 <= day % 100 <= 20:
+        suffix = "th"
+    else:
+        suffix = {1: "st", 2: "nd", 3: "rd"}.get(day % 10, "th")
+
+    # Format the date
+    return today.strftime(f"%B %-d{suffix}, %Y")
+
+
+def json_to_markdown(data, level=1):
+    """Converts a JSON object into Markdown.  Future versions of MarkItDown will support this natively.
+
+    Args:
+        data: The JSON object (dict, list, or primitive type).
+        level: Current Markdown heading level for nested keys.
+
+    Returns:
+        A Markdown string.
+    """
+    markdown = ""
+
+    if isinstance(data, dict):
+        for key, value in data.items():
+            markdown += f"{'#' * level} {key}\n\n"  # Add heading for the key
+            markdown += json_to_markdown(value, level + 1)  # Recursively process value
+    elif isinstance(data, list):
+        for item in data:
+            markdown += f"- {json_to_markdown(item, level + 1).strip()}\n"  # Format as a list
+    else:
+        # Format primitive types (string, number, boolean, null)
+        markdown += f"{data}\n\n"
+
+    return markdown
+
+
+def call_per_field(
+    klass: type[BaseModel], client, model: str, messages: list[dict[str, str]], show_progress_bar: bool = False
+) -> BaseModel:
+    """Calls the LLM per field and returns a validated model.
+
+    This function is typically used when the LLM model is less powerful such as when the model is running locally.
+
+    Args:
+        klass (type[BaseModel]): The Pydantic model class to be updated.
+        client: The LLM client. such as OpenAI(api_key=..., base_url=...) or AzureOpenAI(api_key=..., base_url=...)
+        model (str): The model name, for example "gpt-4o" or "llama3.1".
+        messages (list[dict[str, str]]): The messages to be sent to the LLM.
+        show_progress_bar (bool): If True, shows a progress bar. Defaults to False but is set to True nested calls.
+
+
+    Returns:
+        BaseModel: The validated model instance with the applied updates.
+
+    Example:
+    ```python
+    klass = metadata_class_no_rules, metadata_type, _ = MetadataEditor._get_metadata_class_and_type_and_UID(
+            metadata_type_or_template_uid, apply_template_rules=False
+        )
+    client = OpenAI(api_key=..., base_url=...)
+    model = "gpt-4o"
+    messages = [{"role": "system", "content": "You are a helpful assistant who writes metadata."},
+                {"role": "user", "content": "A report on the state of the world"}]
+    updated_model = call_per_field(klass, client, model, messages)
+    ```
+    1. The function iterates over each field in the Pydantic model.
+    2. For each field, it sends a request to the LLM with the provided messages.
+    3. If the LLM response is valid, it updates the field in the model.
+    4. If the LLM response is invalid, it attempts to handle nested fields or lists.
+    5. The function returns the updated and validated model.
+    """
+    final_dict = {}
+    for field in tqdm(klass.model_fields.keys(), disable=not show_progress_bar):
+        try:
+            completion = client.beta.chat.completions.parse(
+                model=model,
+                messages=messages,
+                response_format=subset_pydantic_model_type(klass, [field]),
+                temperature=0.0,
+                timeout=60 * 1,
+            )
+        except APITimeoutError:
+            # print(f"{' '*(indent+2)}Timeout")
+            continue
+        except ValidationError:
+            # print(f"{' '*indent}Validation Error for {field}, attempting to step through subfields")
+            field_annotation = klass.model_fields[field].annotation
+            # print(f"{' '*indent}field")
+            if is_optional_list(field_annotation) or is_list_annotation(field_annotation):
+                # print(get_subtype_of_optional_or_list(field_annotation))
+                class subtype(BaseModel):
+                    field: List[get_subtype_of_optional_or_list(field_annotation)]
+
+                final_dict[field] = call_per_field(subtype, client, model, messages, show_progress_bar=True)
+            elif is_optional_annotation(field_annotation):
+                # print(get_subtype_of_optional_or_list(field_annotation))
+                final_dict[field] = call_per_field(
+                    get_subtype_of_optional_or_list(field_annotation), client, model, messages, show_progress_bar=True
+                )
+            else:
+                continue
+        else:
+            message = completion.choices[0].message
+            metadata_dict = message.parsed.model_dump(exclude_none=True, exclude_unset=True)
+            final_dict = {**final_dict, **metadata_dict}
+    return _iterated_validated_update_to_outline(klass, final_dict)

{pymetadataeditor-0.2.0 → pymetadataeditor-0.3.1}/pymetadataeditor/requester.py

@@ -1,3 +1,9 @@
+"""A module to handle HTTP requests with specific error handling for SSL and JSON decoding errors.
+
+This module defines a class `RequestsWithSpecificErrors` that provides methods for making GET and POST requests
+to a specified API URL. It includes error handling for SSL errors, HTTP errors, and JSON decoding errors.
+"""
+
 from io import BufferedReader
 from json import JSONDecodeError
 from ssl import SSLError as ssl_SSLError
@@ -9,6 +15,8 @@ from requests.exceptions import HTTPError, SSLError
 
 
 class RequestsWithSpecificErrors(BaseModel):
+    """A class to handle HTTP requests with specific error handling for SSL and JSON decoding errors."""
+
     model_config = ConfigDict(extra="forbid", validate_assignment=True)
 
     api_url: AnyHttpUrl
@@ -30,7 +38,7 @@ class RequestsWithSpecificErrors(BaseModel):
         else:
             raise ValueError(
                 f"URL scheme should be 'https' but got {self.api_url}"
-                "To allow the less secure use of 'http', set allow_unsecure=True"
+                "To allow the less secure use of 'http', set allow_http=True"
             )
 
     def _request(
@@ -43,12 +51,17 @@ class RequestsWithSpecificErrors(BaseModel):
         data: Optional[Dict[str, str]] = None,
         files: Optional[Dict[str, BufferedReader]] = None,
     ) -> Dict:
-        """
-        Sends a GET or POST request to the specified URL with the API key in the headers and returns the JSON response.
+        """Perform a GET or POST request to the specified URL with the API key in the headers.
 
         Args:
             method (str): Either post or get
             pth (str): The path appended to the API_URL to which the GET or POST request is sent.
+            json (optional dict): The JSON data to send with the POST request.
+            params (optional dict): additional parameters to send with the get request such as 'keywords'
+            id (optional int or str): The id of a specific collection or project.
+                                        If not none, then pth should contain '{}' where the id ought to go.
+            data (optional dict): The data to send with the POST request.
+            files (optional dict): The files to send with the POST request.
 
         Returns:
             Dict[str, str]: The JSON response from the server, parsed into a dictionary.
@@ -102,14 +115,14 @@ class RequestsWithSpecificErrors(BaseModel):
                 raise HTTPError(error_msg) from None
             elif response.status_code == 403:
                 raise PermissionError(
-                    f"Access to that URL is denied for {url} " "Check that the API key is correct"
+                    f"Access to that URL is denied for {url} Check that the API key is correct"
                 ) from e
-            elif response.status_code == 400 and "message" in response.text:
-                if isinstance(response.text, dict):
-                    error_message = response.text["message"]
-                else:
-                    error_message = response.text
-                raise PermissionError(error_message) from e
+            # elif response.status_code == 400 and "message" in response.text:
+            #     if isinstance(response.text, dict):
+            #         error_message = response.text["message"]
+            #     else:
+            #         error_message = response.text
+            #     raise PermissionError(error_message) from e
             else:
                 raise HTTPError(f"Status Code: {response.status_code}, Response: {response.text}") from e
         try:
@@ -125,7 +138,8 @@ class RequestsWithSpecificErrors(BaseModel):
     def get_request(
         self, pth: str, id: Optional[Union[int, str]] = None, params: Optional[Dict[str, Union[str, List[str]]]] = None
     ) -> Dict:
-        """
+        """Performs a GET request to the specified URL with the API key in the headers and returns the JSON response.
+
         Args:
             pth (str): The path appended to the API_URL to which the GET request is sent.
             id (optional int or str): The id of a specific collection or project.
@@ -145,7 +159,8 @@ class RequestsWithSpecificErrors(BaseModel):
         data: Optional[Dict[str, str]] = None,
         files: Optional[Dict[str, BufferedReader]] = None,
     ):
-        """
+        """Performs a POST request to the specified URL with the API key in the headers and returns the JSON response.
+
         Args:
             pth (str): The path appended to the API_URL to which the POST request is sent.
             json (optional dict): The JSON data to send with the POST request.

{pymetadataeditor-0.2.0 → pymetadataeditor-0.3.1}/pymetadataeditor/templates.py

@@ -617,7 +617,12 @@ def append_variables_and_data_files(model_elements, parent_schema, apply_rules:
 
 
 def pydantic_from_template(
-    template: Dict, parent_schema: Type[SchemaBaseModel], uid: str, name: Optional[str] = None, apply_rules: bool = True
+    template: Dict,
+    parent_schema: Type[SchemaBaseModel],
+    uid: str,
+    name: Optional[str] = None,
+    metadata_type: Optional[str] = None,
+    apply_rules: bool = True,
 ) -> Type[BaseModel]:
     """Generate a Pydantic model from a given template.
 
@@ -626,6 +631,8 @@ def pydantic_from_template(
         parent_schema (Type[SchemaBaseModel]): The parent schema model to base the new model on.
         uid (str): A unique identifier for the new model.
         name (Optional[str], optional): The name of the new model. Defaults to None.
+        metadata_type (Optional[str], optional): The metadata type for the new model used if the parent_schema is
+            SchemaBaseModel. Otherwise the metadata_type of the parent_schema is used. Defaults to None.
         apply_rules (bool, optional): Whether to apply rules to the model elements. Defaults to True.
 
     Returns:
@@ -659,4 +666,12 @@ def pydantic_from_template(
         hasattr(parent_schema._metadata_type__, "default") and parent_schema._metadata_type__.default == "microdata"
     ):
         model_elements = append_variables_and_data_files(model_elements, parent_schema, apply_rules=apply_rules)
-    return create_model_for_template(model_elements, parent_schema, name, uid=uid)
+    new_model = create_model_for_template(model_elements, parent_schema, name, uid=uid)
+    if parent_schema is SchemaBaseModel:
+        if metadata_type is None:
+            raise ValueError("metadata_type must be provided if parent_schema is SchemaBaseModel. ")
+        new_model._template_name__ = f"{name.replace('-', '_').replace(' ', '_')}"
+        new_model._template_uid__ = uid
+        new_model._metadata_type__ = metadata_type.replace("-", "_").replace(" ", "_")
+        new_model._metadata_type_version__ = "unknown"
+    return new_model

{pymetadataeditor-0.2.0 → pymetadataeditor-0.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pymetadataeditor"
-version = "0.2.0"
+version = "0.3.1"
 description = "Metadata Editor client for python"
 authors = ["Gordon Blackadder <gblackadder@worldbank.org>"]
 readme = "README.md"
@@ -15,7 +15,7 @@ pydantic = "^2.7.4"
 openai = "^1.59.4"
 tiktoken = "^0.8.0"
 markitdown = "^0.0.1a3"
-metadataschemas = "^0.1.2"
+metadataschemas = "^0.1.3"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.2.2"
@@ -35,6 +35,8 @@ fix = true
 [tool.ruff.lint]
 extend-select = ["I","E","F","D"]
 ignore = ["D203","D204","D213","D215","D400","D401","D404","D406","D407","D408","D409","D413"]
+# Ignore missing docstring warnings in test files since unit tests don't need them
+per-file-ignores = { "tests/*.py" = ["D103", "D100"] }
 
 [tool.ruff.lint.pydocstyle]
 convention = "google"

pymetadataeditor-0.2.0/pymetadataeditor/llm_helpers.py

@@ -1,145 +0,0 @@
-from datetime import datetime
-from typing import Any, get_args
-
-from metadataschemas.utils.quick_start import make_skeleton
-from metadataschemas.utils.utils import is_list_annotation, is_optional_annotation
-from pydantic import BaseModel, ValidationError
-
-
-def _prepend_draft_drop_non_str(d: Any, prefix: str) -> dict | list | str | None:
-    """
-    Recursively prepend a prefix to all strings in a dictionary or list and drop empty strings and non-strings.
-
-    Args:
-
-        d (Any): The dictionary or list to process.
-    """
-
-    if isinstance(d, dict):
-        out = {}
-        for k, v in d.items():
-            new_v = _prepend_draft_drop_non_str(v, prefix)
-            if new_v is not None:
-                out[k] = new_v
-        if len(out) > 0:
-            return out
-        else:
-            return None
-    elif isinstance(d, list):
-        out = []
-        for v in d:
-            new_v = _prepend_draft_drop_non_str(v, prefix)
-            if new_v is not None:
-                out.append(new_v)
-        if len(out) > 0:
-            return out
-        else:
-            return None
-    elif isinstance(d, str):
-        if d.strip() == "":
-            return None
-        else:
-            return prefix + d
-    else:
-        return None
-
-
-def _iterated_validated_update_to_outline(model_def: type[BaseModel], updates: dict, verbose=False) -> BaseModel:
-    original_model = make_skeleton(model_def).model_dump()
-    # print(original_model)
-    model_def.model_validate(original_model, strict=False)
-
-    model_fields = model_def.model_fields
-    for key, value in updates.items():
-        if key not in original_model:
-            continue
-
-        # get annotation aka type of the field
-        annotation = model_fields[key].annotation
-        if is_optional_annotation(annotation):
-            annotation = get_args(annotation)[0]
-        is_list = False
-        if is_list_annotation(annotation):
-            annotation = get_args(annotation)[0]
-            is_list = True
-        # print(annotation)
-
-        # get candidate value
-        if isinstance(value, dict) and issubclass(annotation, BaseModel):
-            candidate_value = _iterated_validated_update_to_outline(annotation, value, verbose=verbose).model_dump()
-        elif is_list and isinstance(value, list) and issubclass(annotation, BaseModel):
-            candidate_value = []
-            for item in value:
-                candidate_value.append(
-                    _iterated_validated_update_to_outline(annotation, item, verbose=verbose).model_dump()
-                )
-        else:
-            candidate_value = value
-
-        # try the candidate value
-        original_value = original_model[key]
-        original_model[key] = candidate_value
-        try:
-            model_def.model_validate(original_model, strict=False)
-        except ValidationError as e:
-            # If validation fails, leave the original value
-            if verbose:
-                print(f"Skipping {key}({annotation})={candidate_value} because of {e}\n")
-            original_model[key] = original_value
-        # print(original_model)
-
-    return model_def.model_validate(original_model, strict=False)
-
-
-def get_date_as_text():
-    """
-    Returns the current date as a formatted string with an ordinal suffix for the day.
-
-    The format of the returned date string is "Month DaySuffix, Year", where:
-    - Month is the full name of the month (e.g., January, February).
-    - DaySuffix is the day of the month with an appropriate ordinal suffix (e.g., 1st, 2nd, 3rd, 4th, etc.).
-    - Year is the four-digit year.
-
-    Example:
-        If today's date is October 21, 2023, the function will return "October 21st, 2023".
-
-    Returns:
-        str: The formatted date string.
-    """
-    today = datetime.today()
-    day = today.day
-    # Determine ordinal suffix
-    if 10 <= day % 100 <= 20:
-        suffix = "th"
-    else:
-        suffix = {1: "st", 2: "nd", 3: "rd"}.get(day % 10, "th")
-
-    # Format the date
-    return today.strftime(f"%B %-d{suffix}, %Y")
-
-
-def json_to_markdown(data, level=1):
-    """
-    Converts a JSON object into Markdown.  Future versions of MarkItDown will support this natively.
-
-    Args:
-        data: The JSON object (dict, list, or primitive type).
-        level: Current Markdown heading level for nested keys.
-
-    Returns:
-        A Markdown string.
-    """
-    markdown = ""
-
-    if isinstance(data, dict):
-        for key, value in data.items():
-            markdown += f"{'#' * level} {key}\n\n"  # Add heading for the key
-            markdown += json_to_markdown(value, level + 1)  # Recursively process value
-    elif isinstance(data, list):
-        for item in data:
-            markdown += f"- {json_to_markdown(item, level + 1).strip()}\n"  # Format as a list
-    else:
-        # Format primitive types (string, number, boolean, null)
-        markdown += f"{data}\n\n"
-
-    return markdown