pymetadataeditor 0.3.0__tar.gz → 0.3.2__tar.gz

{pymetadataeditor-0.3.0 → pymetadataeditor-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pymetadataeditor
-Version: 0.3.0
+Version: 0.3.2
 Summary: Metadata Editor client for python
 License: MIT
 Author: Gordon Blackadder
@@ -11,8 +11,8 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: markitdown (>=0.0.1a3,<0.0.2)
-Requires-Dist: metadataschemas (>=0.1.2,<0.2.0)
-Requires-Dist: openai (>=1.59.4,<2.0.0)
+Requires-Dist: metadataschemas (>=0.1.3,<0.2.0)
+Requires-Dist: openai (>=2.8.0,<3.0.0)
 Requires-Dist: pandas (>=2.2.2,<3.0.0)
 Requires-Dist: pydantic (>=2.7.4,<3.0.0)
 Requires-Dist: requests (>=2.32.3,<3.0.0)

{pymetadataeditor-0.3.0 → pymetadataeditor-0.3.2}/pymetadataeditor/interface.py

@@ -427,12 +427,21 @@ class MetadataEditor:
         else:
             metadata_type = self._templates[template_uid]["metadata_type"]
             if "class" not in self._templates[template_uid] or apply_template_rules is False:
-                parent_schema = self._mm.metadata_class_from_name(metadata_type)
+                try:
+                    parent_schema = self._mm.metadata_class_from_name(metadata_type)
+                except ValueError:
+                    warnings.warn(
+                        f"Could not find a metadata class for template type '{metadata_type}', UID '{template_uid}'. "
+                        f"Building a class from the template without a parent schema."
+                    )
+                    parent_schema = SchemaBaseModel
+
                 klass = pydantic_from_template(
                     temp.template,
                     parent_schema=parent_schema,
                     uid=template_uid,
                     name=temp.name,
+                    metadata_type=metadata_type,
                     apply_rules=apply_template_rules,
                 )
                 if apply_template_rules:
@@ -693,7 +702,7 @@ class MetadataEditor:
         tokenizer_model="o200k_base",
         max_tokens=128_000,
         llm_base_url: Optional[str] = None,
-        azure_llm_base_url: Optional[str] = None,
+        azure_deployment_name: Optional[str] = None,
     ) -> Union[BaseModel, Dict, str]:
         """Automatically generate *draft* metadata for a project based on local files or web pages.
 
@@ -730,11 +739,13 @@ class MetadataEditor:
             max_tokens (int): The maximum number of tokens to use when sending the content to OpenAI.
                 Defaults to 128_000, which has been the typical maximum for the 4o model.
             llm_base_url (Optional[str]): The base URL for the LLM API. If None, the default URL is used which
-                sends the request to OpenAI. This argument is ignored if an azure_llm_base_url is provided.
-            azure_llm_base_url (Optional[str]): The base URL for the Azure LLM API. Typically used when an organization
-                has its own deployment of an LLM model, possibly for privacy reasons. The Azure endpoint will be used
-                even if a llm_base_url is provided. If None, then the llm_base_url endpoint is used. If
-                that's also None, then OpenAI is used.
+                sends the request to OpenAI. Alternatively, this can be the base URL for a local LLM instance such as
+                Ollama or a private deployment of an LLM model. If using Azure OpenAI, this should be the Azure
+                endpoint URL (eg "https://my-azure-openai-resource.openai.azure.com/") and the azure_deployment_name
+                parameter should also be set.
+            azure_deployment_name (Optional[str]): Used when an organization has its own deployment of an LLM model in
+                Azure, possibly for privacy reasons. Be sure to provide the llm_base_url parameter as well which should
+                be the Azure endpoint URL.
 
         Returns:
             (Union[BaseModel, Dict, str]): If mode == 'dict', a dictionary is returned. If mode == 'pydantic', a
@@ -776,7 +787,8 @@ class MetadataEditor:
             metadata_producer_organization="My Organization",
             filename="output.xlsx",
             title="My Metadata",
-            azure_llm_base_url="https://my-azure-openai-resource.openai.azure.com/",
+            llm_base_url="https://my-azure-openai-resource.openai.azure.com/",
+            azure_deployment_name="my-llm-deployment"
         )
         ```
         """
@@ -786,10 +798,15 @@ class MetadataEditor:
             metadata_type_or_template_uid, apply_template_rules=False
         )
         enc = tiktoken.get_encoding(tokenizer_model)
-        if azure_llm_base_url is None:
+        if azure_deployment_name is None:
             client = OpenAI(api_key=llm_api_key, base_url=llm_base_url)
         else:
-            client = AzureOpenAI(api_key=llm_api_key, base_url=azure_llm_base_url, api_version="2024-10-01")
+            client = AzureOpenAI(
+                api_key=llm_api_key,
+                azure_endpoint=llm_base_url,
+                azure_deployment=azure_deployment_name,
+                api_version="2024-10-01",
+            )
 
         system_prompt = f"You are an expert on producing {metadata_type} documentation. "
         system_prompt += (
@@ -829,9 +846,7 @@ class MetadataEditor:
                 print(f"Reading {doc}, running token count is {num_tokens}")
         messages += user_message
 
-        endpoint_name = (
-            azure_llm_base_url if azure_llm_base_url is not None else "OpenAI" if llm_base_url is None else llm_base_url
-        )
+        endpoint_name = llm_base_url if llm_base_url is not None else "OpenAI"
         print(f"Sending to {endpoint_name}, this may take a few minutes...")
         try:
             completion = client.beta.chat.completions.parse(
@@ -876,7 +891,7 @@ class MetadataEditor:
         tokenizer_model="o200k_base",
         max_tokens=128_000,
         llm_base_url: Optional[str] = None,
-        azure_llm_base_url: Optional[str] = None,
+        azure_deployment_name: Optional[str] = None,
     ) -> Union[BaseModel, Dict, str]:
         """Augment existing metadata with information from files or web pages.
 
@@ -905,11 +920,13 @@ class MetadataEditor:
             max_tokens (int): The maximum number of tokens to use when sending the content to OpenAI.
                 Defaults to 128_000, which has been the typical maximum for the 4o model.
             llm_base_url (Optional[str]): The base URL for the LLM API. If None, the default URL is used which
-                sends the request to OpenAI.
-            azure_llm_base_url (Optional[str]): The base URL for the Azure LLM API. Typically used when an organization
-                has its own deployment of an LLM model, possibly for privacy reasons. The Azure endpoint will be used
-                even if a llm_base_url is provided. If None, then the llm_base_url endpoint is used. If
-                that's also None, then OpenAI is used.
+                sends the request to OpenAI. Alternatively, this can be the base URL for a local LLM instance such as
+                Ollama or a private deployment of an LLM model. If using Azure OpenAI, this should be the Azure
+                endpoint URL (eg "https://my-azure-openai-resource.openai.azure.com/") and the azure_deployment_name
+                parameter should also be set.
+            azure_deployment_name (Optional[str]): Used when an organization has its own deployment of an LLM model in
+                Azure, possibly for privacy reasons. Be sure to provide the llm_base_url parameter as well which should
+                be the Azure endpoint URL.
 
         Returns:
             Union[BaseModel, Dict, str]: The augmented metadata.
@@ -936,7 +953,8 @@ class MetadataEditor:
             output_mode="pydantic",
             metadata_producer_organization="My Organization",
             prefix="<AI>",
-            azure_llm_base_url="https://my-azure-openai-resource.openai.azure.com/",
+            llm_base_url="https://my-azure-openai-resource.openai.azure.com/",
+            azure_deployment_name="my-llm-deployment"
         )
         ```
         """
@@ -1008,7 +1026,7 @@ class MetadataEditor:
                 tokenizer_model=tokenizer_model,
                 max_tokens=max_tokens,
                 llm_base_url=llm_base_url,
-                azure_llm_base_url=azure_llm_base_url,
+                azure_deployment_name=azure_deployment_name,
             )
 
         if new_metadata is None or len(new_metadata) == 0:
@@ -1043,7 +1061,7 @@ class MetadataEditor:
                     document, geospatial, image, indicator, indicators_db, microdata, resource, script, table, video
                 In this case we will use the default template for that metadata type.
                 Alternatively you can pass in the UID of a template. This is required if the metadata is a dictionary
-                    otherwise ignored.
+                    otherwise the UID associated with the pydantic model or the Excel file will be used.
 
         Returns:
             int: The ID of the newly created document metadata
@@ -1063,6 +1081,37 @@ class MetadataEditor:
         )
         return ret["id"]
 
+    # def log_project_admin_metadata(
+    #         self, id: int, metadata: Union[BaseModel, Dict, str], metadata_type_or_template_uid: Optional[str] = None
+    # ) -> int:
+    #     """Validates and logs admin metadata which can be a dictionary, a pydantic model or a path to an Excel file.
+
+    #     Args:
+    #         id (int): The ID of the project to associate the metadata with.
+    #         metadata (dictionary or BaseModel or str): If str, it's assumed this is a path to an appropriately
+    #             formatted Excel file.
+    #         admin_metadata_template_uid (str, optional): The UID of the admin metadata template to use.
+    #             otherwise the UID associated with the pydantic model or the Excel file will be used.
+
+    #     Returns:
+    #         int: The ID of the newly created admin metadata
+    #     """
+    #     metadata, metadata_type, uid = self._process_metadata_input(metadata, metadata_type_or_template_uid)
+    #     post_request_pth = "admin-metadata/data/"
+    #     post_json = {
+    #         "project_id": id,
+    #         "template_uid": uid,
+    #         "metadata": remove_empty_from_dict(metadata.model_dump(mode="json",
+    #                                                                exclude_none=True,
+    #                                                                exclude_unset=True)),
+    #     }
+    #     ret = self._apinterface.post_request(
+    #         pth=post_request_pth,
+    #         json=post_json,
+    #         id=id,
+    #     )
+    #     return ret
+
     def update_project_log_by_id(self, id: int, new_metadata: Union[BaseModel, Dict, str]):
         """Updates the record of the metadata.
 
@@ -1252,7 +1301,6 @@ class MetadataEditor:
                 new_index = templates.index.astype(int)
             except ValueError:
                 pass
-            else:
                 templates.index = new_index
             return templates
 
@@ -1272,10 +1320,23 @@ class MetadataEditor:
             raise KeyError(f"No result returned.\nResponse: {response}")
 
         temp = pd.Series(response["result"], name=response["result"]["name"])
-        self._templates[uid] = {"template": temp, "metadata_type": self._mm.standardize_metadata_name(temp.data_type)}
         try:
-            parent_schema = self._mm.metadata_class_from_name(temp.data_type)
-            klass = pydantic_from_template(temp.template, parent_schema=parent_schema, uid=uid, name=temp.name)
+            standard_type_name = self._mm.standardize_metadata_name(temp.data_type)
+        except ValueError:
+            standard_type_name = temp.data_type
+        self._templates[uid] = {"template": temp, "metadata_type": standard_type_name}
+        try:
+            try:
+                parent_schema = self._mm.metadata_class_from_name(temp.data_type)
+            except ValueError:
+                warnings.warn(
+                    f"Could not find a metadata class for template type '{standard_type_name}', UID '{uid}'. "
+                    f"Building a class from the template without a parent schema."
+                )
+                parent_schema = SchemaBaseModel
+            klass = pydantic_from_template(
+                temp.template, parent_schema=parent_schema, uid=uid, name=temp.name, metadata_type=standard_type_name
+            )
             self._templates[uid]["class"] = klass
         finally:
             return temp
@@ -1474,6 +1535,26 @@ class MetadataEditor:
             metadata["description"] = description
         self._apinterface.post_request("collections/update/{}", id=id, json=metadata)
 
+    def copy_collection(self, source_id: int, target_id: int):
+        """Copy projects and users from one collection to another.
+
+        Args:
+            source_id (int): The ID of the source collection.
+            target_id (int): The ID of the target collection.
+
+        """
+        self._apinterface.post_request("collections/copy/", json={"source_id": source_id, "target_id": target_id})
+
+    def move_collection(self, source_id, target_id):
+        """Move source collection to be a sub-collection of the target collection.
+
+        Args:
+            source_id (int): The ID of the source collection.
+            target_id (int): The ID of the target collection.
+
+        """
+        self._apinterface.post_request("collections/move/", json={"source_id": source_id, "target_id": target_id})
+
     def count_projects_in_collection(self, collection: int) -> int:
         """Count the number of projects you have access to.
 

{pymetadataeditor-0.3.0 → pymetadataeditor-0.3.2}/pymetadataeditor/templates.py

@@ -617,7 +617,12 @@ def append_variables_and_data_files(model_elements, parent_schema, apply_rules:
 
 
 def pydantic_from_template(
-    template: Dict, parent_schema: Type[SchemaBaseModel], uid: str, name: Optional[str] = None, apply_rules: bool = True
+    template: Dict,
+    parent_schema: Type[SchemaBaseModel],
+    uid: str,
+    name: Optional[str] = None,
+    metadata_type: Optional[str] = None,
+    apply_rules: bool = True,
 ) -> Type[BaseModel]:
     """Generate a Pydantic model from a given template.
 
@@ -626,6 +631,8 @@ def pydantic_from_template(
         parent_schema (Type[SchemaBaseModel]): The parent schema model to base the new model on.
         uid (str): A unique identifier for the new model.
         name (Optional[str], optional): The name of the new model. Defaults to None.
+        metadata_type (Optional[str], optional): The metadata type for the new model used if the parent_schema is
+            SchemaBaseModel. Otherwise the metadata_type of the parent_schema is used. Defaults to None.
         apply_rules (bool, optional): Whether to apply rules to the model elements. Defaults to True.
 
     Returns:
@@ -659,4 +666,12 @@ def pydantic_from_template(
         hasattr(parent_schema._metadata_type__, "default") and parent_schema._metadata_type__.default == "microdata"
     ):
         model_elements = append_variables_and_data_files(model_elements, parent_schema, apply_rules=apply_rules)
-    return create_model_for_template(model_elements, parent_schema, name, uid=uid)
+    new_model = create_model_for_template(model_elements, parent_schema, name, uid=uid)
+    if parent_schema is SchemaBaseModel:
+        if metadata_type is None:
+            raise ValueError("metadata_type must be provided if parent_schema is SchemaBaseModel. ")
+        new_model._template_name__ = f"{name.replace('-', '_').replace(' ', '_')}"
+        new_model._template_uid__ = uid
+        new_model._metadata_type__ = metadata_type.replace("-", "_").replace(" ", "_")
+        new_model._metadata_type_version__ = "unknown"
+    return new_model

{pymetadataeditor-0.3.0 → pymetadataeditor-0.3.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pymetadataeditor"
-version = "0.3.0"
+version = "0.3.2"
 description = "Metadata Editor client for python"
 authors = ["Gordon Blackadder <gblackadder@worldbank.org>"]
 readme = "README.md"
@@ -12,10 +12,10 @@ requests = "^2.32.3"
 urllib3 = "^2.2.2"  # earlier versions have known security issues
 pandas = "^2.2.2"
 pydantic = "^2.7.4"
-openai = "^1.59.4"
+openai = "^2.8.0"
 tiktoken = "^0.8.0"
 markitdown = "^0.0.1a3"
-metadataschemas = "^0.1.2"
+metadataschemas = "^0.1.3"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.2.2"