pymetadataeditor 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pymetadataeditor/__init__.py +1 -0
- pymetadataeditor/interface.py +1641 -0
- pymetadataeditor/llm_helpers.py +145 -0
- pymetadataeditor/requester.py +160 -0
- pymetadataeditor/templates.py +617 -0
- pymetadataeditor/utils.py +262 -0
- pymetadataeditor-0.1.0.dist-info/LICENSE +21 -0
- pymetadataeditor-0.1.0.dist-info/METADATA +333 -0
- pymetadataeditor-0.1.0.dist-info/RECORD +10 -0
- pymetadataeditor-0.1.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
from typing import Annotated, Any, Dict, List, Optional, Type, Union, get_args, get_origin
|
|
2
|
+
|
|
3
|
+
from metadataschemas.utils.utils import is_list_annotation, is_optional_annotation, is_optional_list
|
|
4
|
+
from pydantic import BaseModel, Field
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def validate_json_patches(patches):
|
|
8
|
+
"""
|
|
9
|
+
Validates a list of JSON patches according to the JSON Patch specification.
|
|
10
|
+
|
|
11
|
+
Args:
|
|
12
|
+
patches (list): A list of JSON patch dictionaries.
|
|
13
|
+
|
|
14
|
+
Returns:
|
|
15
|
+
list: The validated patches with corrected paths if necessary.
|
|
16
|
+
|
|
17
|
+
Raises:
|
|
18
|
+
ValueError: If a patch is invalid or contains an error.
|
|
19
|
+
"""
|
|
20
|
+
valid_ops = {"add", "remove", "replace", "move", "copy", "test"}
|
|
21
|
+
validated_patches = []
|
|
22
|
+
|
|
23
|
+
for patch in patches:
|
|
24
|
+
if not isinstance(patch, dict):
|
|
25
|
+
raise ValueError(f"Invalid patch format: {patch}. Each patch must be a dictionary.")
|
|
26
|
+
|
|
27
|
+
op = patch.get("op")
|
|
28
|
+
path = patch.get("path")
|
|
29
|
+
|
|
30
|
+
if op not in valid_ops:
|
|
31
|
+
raise ValueError(f"Invalid operation '{op}'. Valid operations are {valid_ops}.")
|
|
32
|
+
|
|
33
|
+
if not isinstance(path, str):
|
|
34
|
+
raise ValueError(f"Invalid path: {path}. Path must be a string.")
|
|
35
|
+
|
|
36
|
+
if not path.startswith("/"):
|
|
37
|
+
path = "/" + path
|
|
38
|
+
|
|
39
|
+
# Additional checks for specific operations
|
|
40
|
+
if op in {"add", "replace", "test"} and "value" not in patch:
|
|
41
|
+
raise ValueError(f"Operation '{op}' requires a 'value' field.")
|
|
42
|
+
|
|
43
|
+
if op in {"move", "copy"} and "from" not in patch:
|
|
44
|
+
raise ValueError(f"Operation '{op}' requires a 'from' field.")
|
|
45
|
+
|
|
46
|
+
validated_patches.append({**patch, "path": path})
|
|
47
|
+
|
|
48
|
+
return validated_patches
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def remove_empty_from_dict(old_dict: Dict) -> Dict:
|
|
52
|
+
"""Helper function for removing entries from dictionaries that look like:
|
|
53
|
+
[{'name': ''}]
|
|
54
|
+
"""
|
|
55
|
+
new_dict = {}
|
|
56
|
+
for k, v in old_dict.items():
|
|
57
|
+
if v is None:
|
|
58
|
+
continue
|
|
59
|
+
if isinstance(v, str):
|
|
60
|
+
if v != "":
|
|
61
|
+
new_dict[k] = v
|
|
62
|
+
elif isinstance(v, dict):
|
|
63
|
+
new_sub_dict = remove_empty_from_dict(v)
|
|
64
|
+
if len(new_sub_dict) > 0:
|
|
65
|
+
new_dict[k] = new_sub_dict
|
|
66
|
+
elif isinstance(v, list):
|
|
67
|
+
new_list = remove_empty_from_list(v)
|
|
68
|
+
if len(new_list) > 0:
|
|
69
|
+
new_dict[k] = new_list
|
|
70
|
+
else:
|
|
71
|
+
new_dict[k] = v
|
|
72
|
+
return new_dict
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def remove_empty_from_list(v):
|
|
76
|
+
new_list = []
|
|
77
|
+
for elem in v:
|
|
78
|
+
if elem is None:
|
|
79
|
+
continue
|
|
80
|
+
elif isinstance(elem, dict):
|
|
81
|
+
new_elem = remove_empty_from_dict(elem)
|
|
82
|
+
if len(new_elem) > 0:
|
|
83
|
+
new_list.append(new_elem)
|
|
84
|
+
elif isinstance(elem, str):
|
|
85
|
+
if elem != "":
|
|
86
|
+
new_list.append(elem)
|
|
87
|
+
elif isinstance(elem, list):
|
|
88
|
+
new_elem = remove_empty_from_list(elem)
|
|
89
|
+
if len(new_elem) > 0:
|
|
90
|
+
new_list.append(new_elem)
|
|
91
|
+
else:
|
|
92
|
+
new_list.append(elem)
|
|
93
|
+
return new_list
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# def strip_model_rules(original_model: Type[BaseModel]) -> Type[BaseModel]:
|
|
97
|
+
# """
|
|
98
|
+
# Create a copy of a Pydantic model class (and its nested models) with validation rules stripped,
|
|
99
|
+
# but retaining titles and descriptions.
|
|
100
|
+
# """
|
|
101
|
+
# # Process fields
|
|
102
|
+
# stripped_fields = {}
|
|
103
|
+
# annotations = {}
|
|
104
|
+
|
|
105
|
+
# for name, field in original_model.model_fields.items():
|
|
106
|
+
# annotation = field.annotation
|
|
107
|
+
# default_value = field.default
|
|
108
|
+
|
|
109
|
+
# # Prepare metadata to retain title and description
|
|
110
|
+
# extra_kwargs = {}
|
|
111
|
+
# if field.title:
|
|
112
|
+
# extra_kwargs["title"] = field.title
|
|
113
|
+
# if field.description:
|
|
114
|
+
# extra_kwargs["description"] = field.description
|
|
115
|
+
|
|
116
|
+
# # Check if the annotation refers to another Pydantic model
|
|
117
|
+
# if isinstance(annotation, type) and issubclass(annotation, BaseModel):
|
|
118
|
+
# # Recursively process the nested model
|
|
119
|
+
# stripped_sub_model = strip_model_rules(annotation)
|
|
120
|
+
# stripped_fields[name] = (
|
|
121
|
+
# stripped_sub_model,
|
|
122
|
+
# Field(default=default_value, **extra_kwargs),
|
|
123
|
+
# )
|
|
124
|
+
# # Handle lists of models
|
|
125
|
+
# elif get_origin(annotation) is list or get_origin(annotation) is List:
|
|
126
|
+
# item_type = get_args(annotation)[0]
|
|
127
|
+
# if isinstance(item_type, type) and issubclass(item_type, BaseModel):
|
|
128
|
+
# # Recursively process the nested model
|
|
129
|
+
# stripped_item_type = strip_model_rules(item_type)
|
|
130
|
+
# stripped_fields[name] = (
|
|
131
|
+
# List[stripped_item_type],
|
|
132
|
+
# Field(default=default_value, **extra_kwargs),
|
|
133
|
+
# )
|
|
134
|
+
# else:
|
|
135
|
+
# # Leave other types untouched
|
|
136
|
+
# stripped_fields[name] = (annotation, Field(default=default_value, **extra_kwargs))
|
|
137
|
+
# else:
|
|
138
|
+
# # For non-model fields, just copy the type, default value, and metadata
|
|
139
|
+
# stripped_fields[name] = (annotation, Field(default=default_value, **extra_kwargs))
|
|
140
|
+
|
|
141
|
+
# # Update annotations
|
|
142
|
+
# annotations[name] = stripped_fields[name][0]
|
|
143
|
+
|
|
144
|
+
# # Create the new stripped model
|
|
145
|
+
# new_model = type(
|
|
146
|
+
# f"{original_model.__name__}",
|
|
147
|
+
# (SchemaBaseModel,),
|
|
148
|
+
# {"__annotations__": annotations, **{name: field[1] for name, field in stripped_fields.items()}},
|
|
149
|
+
# )
|
|
150
|
+
|
|
151
|
+
# return new_model
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def strip_constraints_from_annotated(annotation: Any) -> Any:
|
|
155
|
+
"""
|
|
156
|
+
Strip constraints from Annotated types, Optional, List, and combinations,
|
|
157
|
+
while preserving the base type.
|
|
158
|
+
"""
|
|
159
|
+
# # Handle Annotated types, e.g., Annotated[str, constr(min_length=1)]
|
|
160
|
+
# if get_origin(annotation) is Annotated:
|
|
161
|
+
# base_type = get_args(annotation)[0]
|
|
162
|
+
# return base_type
|
|
163
|
+
|
|
164
|
+
# # Handle Optional (Union[Type, None])
|
|
165
|
+
# if get_origin(annotation) is Union:
|
|
166
|
+
# args = get_args(annotation)
|
|
167
|
+
# if NoneType in args:
|
|
168
|
+
# # Get the other type in Union and return it
|
|
169
|
+
# other_type = [arg for arg in args if arg is not None][0]
|
|
170
|
+
# return strip_constraints_from_annotated(other_type)
|
|
171
|
+
|
|
172
|
+
# # Handle List and List of models
|
|
173
|
+
# if get_origin(annotation) is List:
|
|
174
|
+
# item_type = get_args(annotation)[0]
|
|
175
|
+
# return List[strip_constraints_from_annotated(item_type)]
|
|
176
|
+
|
|
177
|
+
if is_optional_list(annotation):
|
|
178
|
+
inner_annotation = get_args(get_args(annotation)[0])[0]
|
|
179
|
+
if get_origin(inner_annotation) is Annotated:
|
|
180
|
+
return Optional[List[get_args(inner_annotation)[0]]]
|
|
181
|
+
elif is_optional_annotation(annotation):
|
|
182
|
+
inner_annotation = get_args(annotation)[0]
|
|
183
|
+
if get_origin(inner_annotation) is Annotated:
|
|
184
|
+
return Optional[get_args(inner_annotation)[0]]
|
|
185
|
+
elif is_list_annotation(annotation):
|
|
186
|
+
inner_annotation = get_args(annotation)[0]
|
|
187
|
+
if get_origin(inner_annotation) is Annotated:
|
|
188
|
+
return List[get_args(inner_annotation)[0]]
|
|
189
|
+
elif get_origin(annotation) is Annotated:
|
|
190
|
+
return get_args(annotation)[0]
|
|
191
|
+
|
|
192
|
+
# If it's not a generic type or Annotated, return the original annotation
|
|
193
|
+
return annotation
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def strip_model_rules(original_model: Type[BaseModel]) -> Type[BaseModel]:
|
|
197
|
+
"""
|
|
198
|
+
Create a copy of a Pydantic model class (and its nested models) with validation rules stripped,
|
|
199
|
+
but retaining titles and descriptions. Handles Annotated types like StringConstraints.
|
|
200
|
+
"""
|
|
201
|
+
stripped_fields = {}
|
|
202
|
+
annotations = {}
|
|
203
|
+
|
|
204
|
+
for name, field in original_model.model_fields.items():
|
|
205
|
+
annotation = field.annotation
|
|
206
|
+
default_value = field.default
|
|
207
|
+
|
|
208
|
+
# Prepare metadata to retain title and description
|
|
209
|
+
extra_kwargs = {}
|
|
210
|
+
if field.title:
|
|
211
|
+
extra_kwargs["title"] = field.title
|
|
212
|
+
if field.description:
|
|
213
|
+
extra_kwargs["description"] = field.description
|
|
214
|
+
|
|
215
|
+
# Handle Annotated types (e.g., Annotated[str, StringConstraints])
|
|
216
|
+
annotation = strip_constraints_from_annotated(annotation)
|
|
217
|
+
|
|
218
|
+
# Check if the annotation refers to another Pydantic model
|
|
219
|
+
if isinstance(annotation, type) and issubclass(annotation, BaseModel):
|
|
220
|
+
# Recursively process the nested model
|
|
221
|
+
stripped_sub_model = strip_model_rules(annotation)
|
|
222
|
+
stripped_fields[name] = (
|
|
223
|
+
stripped_sub_model,
|
|
224
|
+
Field(default=default_value, **extra_kwargs),
|
|
225
|
+
)
|
|
226
|
+
# Handle lists of models
|
|
227
|
+
elif get_origin(annotation) is list or get_origin(annotation) is List:
|
|
228
|
+
item_type = get_args(annotation)[0]
|
|
229
|
+
if isinstance(item_type, type) and issubclass(item_type, BaseModel):
|
|
230
|
+
# Recursively process the nested model
|
|
231
|
+
stripped_item_type = strip_model_rules(item_type)
|
|
232
|
+
stripped_fields[name] = (
|
|
233
|
+
List[stripped_item_type],
|
|
234
|
+
Field(default=default_value, **extra_kwargs),
|
|
235
|
+
)
|
|
236
|
+
else:
|
|
237
|
+
# Leave other types untouched
|
|
238
|
+
stripped_fields[name] = (annotation, Field(default=default_value, **extra_kwargs))
|
|
239
|
+
# Handle Union types with Annotated or nested models
|
|
240
|
+
elif get_origin(annotation) is Union:
|
|
241
|
+
stripped_union_args = tuple(
|
|
242
|
+
strip_constraints_from_annotated(arg) if isinstance(arg, type) else arg for arg in get_args(annotation)
|
|
243
|
+
)
|
|
244
|
+
stripped_fields[name] = (
|
|
245
|
+
Union[stripped_union_args],
|
|
246
|
+
Field(default=default_value, **extra_kwargs),
|
|
247
|
+
)
|
|
248
|
+
else:
|
|
249
|
+
# For non-model fields, just copy the type, default value, and metadata
|
|
250
|
+
stripped_fields[name] = (annotation, Field(default=default_value, **extra_kwargs))
|
|
251
|
+
|
|
252
|
+
# Update annotations
|
|
253
|
+
annotations[name] = stripped_fields[name][0]
|
|
254
|
+
|
|
255
|
+
# Create the new stripped model
|
|
256
|
+
new_model = type(
|
|
257
|
+
f"{original_model.__name__}",
|
|
258
|
+
(BaseModel,),
|
|
259
|
+
{"__annotations__": annotations, **{name: field[1] for name, field in stripped_fields.items()}},
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
return new_model
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 World Bank
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: pymetadataeditor
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Metadata editor client for python
|
|
5
|
+
License: MIT
|
|
6
|
+
Author: Gordon Blackadder
|
|
7
|
+
Author-email: gblackadder@worldbank.org
|
|
8
|
+
Requires-Python: >=3.11,<4.0
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Requires-Dist: markitdown (>=0.0.1a3,<0.0.2)
|
|
14
|
+
Requires-Dist: metadataschemas (>=0.1.1,<0.2.0)
|
|
15
|
+
Requires-Dist: openai (>=1.59.4,<2.0.0)
|
|
16
|
+
Requires-Dist: pandas (>=2.2.2,<3.0.0)
|
|
17
|
+
Requires-Dist: pydantic (>=2.7.4,<3.0.0)
|
|
18
|
+
Requires-Dist: requests (>=2.32.3,<3.0.0)
|
|
19
|
+
Requires-Dist: tiktoken (>=0.8.0,<0.9.0)
|
|
20
|
+
Requires-Dist: urllib3 (>=2.2.2,<3.0.0)
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# pyMetadataEditor
|
|
24
|
+
|
|
25
|
+
A tool connected to Metadata Editor for creating, editing and managing metadata for microdata, indicators, geospatial data, documents, scripts, images and videos.
|
|
26
|
+
|
|
27
|
+
# How to use pyMetadataEditor
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from pymetadataeditor import MetadataEditor
|
|
32
|
+
import os
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
your_api_key = os.getenv("API_KEY")
|
|
39
|
+
api_url = os.getenv("API_URL")
|
|
40
|
+
me = MetadataEditor(api_url=api_url, api_key=your_api_key, verify_ssl=False)
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Listing your projects
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
me.list_projects(limit=8)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
<div>
|
|
51
|
+
<style scoped>
|
|
52
|
+
.dataframe tbody tr th:only-of-type {
|
|
53
|
+
vertical-align: middle;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
.dataframe tbody tr th {
|
|
57
|
+
vertical-align: top;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
.dataframe thead th {
|
|
61
|
+
text-align: right;
|
|
62
|
+
}
|
|
63
|
+
</style>
|
|
64
|
+
<table border="1" class="dataframe">
|
|
65
|
+
<thead>
|
|
66
|
+
<tr style="text-align: right;">
|
|
67
|
+
<th></th>
|
|
68
|
+
<th>type</th>
|
|
69
|
+
<th>idno</th>
|
|
70
|
+
<th>study_idno</th>
|
|
71
|
+
<th>title</th>
|
|
72
|
+
<th>abbreviation</th>
|
|
73
|
+
<th>nation</th>
|
|
74
|
+
<th>year_start</th>
|
|
75
|
+
<th>year_end</th>
|
|
76
|
+
</tr>
|
|
77
|
+
<tr>
|
|
78
|
+
<th>id</th>
|
|
79
|
+
<th></th>
|
|
80
|
+
<th></th>
|
|
81
|
+
<th></th>
|
|
82
|
+
<th></th>
|
|
83
|
+
<th></th>
|
|
84
|
+
<th></th>
|
|
85
|
+
<th></th>
|
|
86
|
+
<th></th>
|
|
87
|
+
</tr>
|
|
88
|
+
</thead>
|
|
89
|
+
<tbody>
|
|
90
|
+
<tr>
|
|
91
|
+
<th>1003</th>
|
|
92
|
+
<td>document</td>
|
|
93
|
+
<td>12345</td>
|
|
94
|
+
<td>DOC_001</td>
|
|
95
|
+
<td>Sample Document 1</td>
|
|
96
|
+
<td>SD1</td>
|
|
97
|
+
<td>Example Nation</td>
|
|
98
|
+
<td>2020</td>
|
|
99
|
+
<td>2025</td>
|
|
100
|
+
</tr>
|
|
101
|
+
<tr>
|
|
102
|
+
<th>1002</th>
|
|
103
|
+
<td>survey</td>
|
|
104
|
+
<td>67890</td>
|
|
105
|
+
<td>SURVEY_002</td>
|
|
106
|
+
<td>Sample Survey 2</td>
|
|
107
|
+
<td>SS2</td>
|
|
108
|
+
<td>Example Nation</td>
|
|
109
|
+
<td>2019</td>
|
|
110
|
+
<td>2024</td>
|
|
111
|
+
</tr>
|
|
112
|
+
<tr>
|
|
113
|
+
<th>1001</th>
|
|
114
|
+
<td>timeseries</td>
|
|
115
|
+
<td>54321</td>
|
|
116
|
+
<td>TS_003</td>
|
|
117
|
+
<td>Time Series 3</td>
|
|
118
|
+
<td>TS3</td>
|
|
119
|
+
<td>Another Nation</td>
|
|
120
|
+
<td>2021</td>
|
|
121
|
+
<td>2026</td>
|
|
122
|
+
</tr>
|
|
123
|
+
</tbody>
|
|
124
|
+
</table>
|
|
125
|
+
</div>
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
## Creating a new indicator project
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
demo_name = "GB20241030_demo"
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
series_description = {
|
|
140
|
+
"idno": demo_name,
|
|
141
|
+
"doi": "V1",
|
|
142
|
+
"name": "Version 1",
|
|
143
|
+
"display_name": "Version 1"
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
indicator_id = me.create_project_log({"idno": demo_name, "series_description": series_description}, "indicator")
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
# Starting with outlines
|
|
151
|
+
|
|
152
|
+
The metadata can be both large and hierarchical. Starting with a skeleton outline makes things easier.
|
|
153
|
+
|
|
154
|
+
Outlines are available in three modes - dictionary, pydantic model and as an Excel file.
|
|
155
|
+
|
|
156
|
+
## Dictionaries
|
|
157
|
+
Dictionaries are created like so:
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
indicator_dict = me.make_metadata_outline('indicator', output_mode='dict')
|
|
162
|
+
indicator_dict
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
{'metadata_information': {'title': None,
|
|
169
|
+
'idno': None,
|
|
170
|
+
'producers': [{'name': '', 'abbr': None, 'affiliation': None, 'role': None}],
|
|
171
|
+
'prod_date': None,
|
|
172
|
+
...
|
|
173
|
+
'email': None,
|
|
174
|
+
'telephone': None,
|
|
175
|
+
'uri': None}]},
|
|
176
|
+
'tags': [{'tag': None, 'tag_group': None}]}
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
## Pydantic
|
|
180
|
+
|
|
181
|
+
Pydantic is a nice python library for defining and validating data schemas. An outline for the indicator schema can be created like so:
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
```python
|
|
185
|
+
indicator_pydantic = me.make_metadata_outline('indicator', 'pydantic')
|
|
186
|
+
indicator_pydantic
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
giving
|
|
191
|
+
|
|
192
|
+
IHSN_INDICATOR_1-0_Template_v01_EN(metadata_information=metadata_information(title=None, idno=None, producers=[Producer(name='', abbr=None, affiliation=None, role=None)], prod_date=None, ...
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
It can be updated using dot notation, for example:
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
```python
|
|
200
|
+
indicator_pydantic.metadata_information.producers[0].name = "example_producer"
|
|
201
|
+
indicator_pydantic
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
giving
|
|
206
|
+
|
|
207
|
+
IHSN_INDICATOR_1-0_Template_v01_EN(metadata_information=metadata_information(title=None, idno=None, producers=[Producer(name='example_producer', abbr=None, affiliation=None, role=None)], prod_date=None, ...
|
|
208
|
+
|
|
209
|
+
## Excel
|
|
210
|
+
|
|
211
|
+
Finally, a nicely formatted Excel file can be created into which the metadata can be written, with the name of the metadata type or of the default template used as the filename if no filename is explicitly given.
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
```python
|
|
215
|
+
outline_filename = me.make_metadata_outline('indicator', 'excel')
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
And then read back in from Excel like so:
|
|
219
|
+
|
|
220
|
+
```python
|
|
221
|
+
indicator_excel = me.read_metadata_from_excel(outline_filename)
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
## Retreiving existing metadata
|
|
225
|
+
|
|
226
|
+
Likewise, existing projects can be downloaded as either dictionaries, pydantic models or as excel spreadsheets.
|
|
227
|
+
|
|
228
|
+
Asking for the metadata as a pydantic object
|
|
229
|
+
|
|
230
|
+
```python
|
|
231
|
+
demo_pydantic = me.get_project_metadata_by_id(indicator_id, 'pydantic')
|
|
232
|
+
demo_pydantic
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
which gives:
|
|
236
|
+
|
|
237
|
+
IHSN_INDICATOR_1-0_Template_v01_EN(metadata_information=metadata_information(title=None, idno=None, producers=[Producer(name='', abbr=None, affiliation=None, role=None)], prod_date=None, ...
|
|
238
|
+
|
|
239
|
+
# Automatic Metadata Creation and Augmentation from Sources
|
|
240
|
+
|
|
241
|
+
We can use a Large Language Model to make a first draft of metadata from a source document or documents.
|
|
242
|
+
|
|
243
|
+
We can create metadata from source files such as:
|
|
244
|
+
- pdfs
|
|
245
|
+
- word
|
|
246
|
+
- excel
|
|
247
|
+
- powerpoint
|
|
248
|
+
- text files
|
|
249
|
+
- csv
|
|
250
|
+
- XML
|
|
251
|
+
- ZIP files
|
|
252
|
+
- Images
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
```python
|
|
258
|
+
docs = ["survey_records/cambodia/cambodia_lsms_basic_information_document.pdf", "survey_records/cambodia/cambodia_living_standards_measurement_study_plus_manual_english.pdf"]
|
|
259
|
+
|
|
260
|
+
example = me.draft_metadata_from_files(openai_api_key=openai_key,
|
|
261
|
+
files=docs,
|
|
262
|
+
metadata_type_or_template_uid='microdata',
|
|
263
|
+
output_mode='pydantic',
|
|
264
|
+
metadata_producer_organization="The World Bank Group, DEC - Development Data Group"
|
|
265
|
+
)
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
The files are read in and sent to the LLM for processing.
|
|
269
|
+
|
|
270
|
+
Read in survey_records/cambodia/cambodia_lsms_basic_information_document.pdf, running token count is 6373
|
|
271
|
+
Read in survey_records/cambodia/cambodia_living_standards_measurement_study_plus_manual_english.pdf, running token count is 24901
|
|
272
|
+
Sending to OpenAI, this may take a few minutes...
|
|
273
|
+
|
|
274
|
+
We can then view the new metadata
|
|
275
|
+
```python
|
|
276
|
+
example.pretty_print()
|
|
277
|
+
```
|
|
278
|
+
which gives
|
|
279
|
+
|
|
280
|
+
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace">IHSN_DDI_2-<span style="color: #800080; text-decoration-color: #800080; font-weight: bold">5_Template_v01_EN</span><span style="font-weight: bold">(</span>
|
|
281
|
+
<span style="color: #808000; text-decoration-color: #808000">doc_desc</span>=<span style="color: #800080; text-decoration-color: #800080; font-weight: bold">doc_desc</span><span style="font-weight: bold">(</span>
|
|
282
|
+
<span style="color: #808000; text-decoration-color: #808000">producers</span>=<span style="font-weight: bold">[</span>
|
|
283
|
+
<span style="color: #800080; text-decoration-color: #800080; font-weight: bold">Producer</span><span style="font-weight: bold">(</span>
|
|
284
|
+
<span style="color: #808000; text-decoration-color: #808000">name</span>=<span style="color: #008000; text-decoration-color: #008000">'The World Bank Group, DEC - Development Data Group'</span>,
|
|
285
|
+
<span style="color: #808000; text-decoration-color: #808000">abbr</span>=<span style="color: #008000; text-decoration-color: #008000">'WBG'</span>,
|
|
286
|
+
<span style="color: #808000; text-decoration-color: #808000">affiliation</span>=<span style="color: #008000; text-decoration-color: #008000">'World Bank'</span>,
|
|
287
|
+
<span style="color: #808000; text-decoration-color: #808000">role</span>=<span style="color: #008000; text-decoration-color: #008000">'Metadata producer'</span>
|
|
288
|
+
<span style="font-weight: bold">)</span>
|
|
289
|
+
<span style="font-weight: bold">]</span>,
|
|
290
|
+
<span style="color: #808000; text-decoration-color: #808000">prod_date</span>=<span style="color: #008000; text-decoration-color: #008000">'2025-01-28'</span>,
|
|
291
|
+
<span style="color: #808000; text-decoration-color: #808000">idno</span>=<span style="color: #008000; text-decoration-color: #008000">'CAMBODIA_LSMS_PLUS_2019_2020_v01_EN'</span>,
|
|
292
|
+
<span style="color: #808000; text-decoration-color: #808000">version_statement</span>=<span style="color: #800080; text-decoration-color: #800080; font-weight: bold">version_statement</span><span style="font-weight: bold">(</span>
|
|
293
|
+
<span style="color: #808000; text-decoration-color: #808000">version</span>=<span style="color: #008000; text-decoration-color: #008000">'1.0'</span>,
|
|
294
|
+
<span style="color: #808000; text-decoration-color: #808000">version_date</span>=<span style="color: #008000; text-decoration-color: #008000">'2025-01-28'</span>,
|
|
295
|
+
<span style="color: #808000; text-decoration-color: #808000">version_resp</span>=<span style="color: #008000; text-decoration-color: #008000">''</span>,
|
|
296
|
+
<span style="color: #808000; text-decoration-color: #808000">version_notes</span>=<span style="color: #008000; text-decoration-color: #008000">'First draft of the metadata for the Cambodia Living Standards Measurement Study - Plus </span>
|
|
297
|
+
<span style="color: #008000; text-decoration-color: #008000">(LSMS+) 2019-20.'</span>
|
|
298
|
+
<span style="font-weight: bold">)</span>
|
|
299
|
+
<span style="font-weight: bold">)</span>,
|
|
300
|
+
...
|
|
301
|
+
|
|
302
|
+
<span style="font-weight: bold">)</span>
|
|
303
|
+
</pre>
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
# Contributing
|
|
309
|
+
## Setting up the python environment
|
|
310
|
+
|
|
311
|
+
This library uses Poetry for dependency management (https://python-poetry.org/docs/basic-usage/).
|
|
312
|
+
|
|
313
|
+
In your python environment run `pip install poetry` then navigate to the pymetadataeditor folder and run `poetry install` or, if that doesn't work, try `python -m poetry install`.
|
|
314
|
+
|
|
315
|
+
### Development python environment
|
|
316
|
+
|
|
317
|
+
If you want to make changes to this repo then you also need to install the tools used for development but which aren't used otherwise, for example pytest.
|
|
318
|
+
|
|
319
|
+
Run:
|
|
320
|
+
|
|
321
|
+
`poetry install --with dev`
|
|
322
|
+
`poetry run pre-commit install`
|
|
323
|
+
|
|
324
|
+
### Poetry troubleshooting
|
|
325
|
+
|
|
326
|
+
If you are running on Windows and see errors about numpy installation errors then it could be an issue with Windows file paths. With default settings, file paths that exceed a few hundred characters can cause installation problems. To overcome this you can either
|
|
327
|
+
|
|
328
|
+
1) enable long path support in Windows (https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=powershell#enable-long-paths-in-windows-10-version-1607-and-later)
|
|
329
|
+
2) install python libraries in a folder in the current directory by running `poetry config virtualenvs.in-project true` and then running `poetry install`
|
|
330
|
+
|
|
331
|
+
## Notes
|
|
332
|
+
|
|
333
|
+
In keeping with World Bank Group practice, it should be noted that parts of this code base were written with the assistance of ChatGPT.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
pymetadataeditor/__init__.py,sha256=7iI9eHfeI_mY1n1BWIsg8ydSSBZWPN_A9ctSF6SkntQ,46
|
|
2
|
+
pymetadataeditor/interface.py,sha256=ZHcCtaYaTi8Sph0_GF5beAa_giR1PMdGwu5utbF_WdU,75990
|
|
3
|
+
pymetadataeditor/llm_helpers.py,sha256=25HzB1aw_TDSyhOBrgmtVVkLyiQyMwR-keS_htIhXEw,4908
|
|
4
|
+
pymetadataeditor/requester.py,sha256=uBzr1J3dbq-V8QUMPeE3bT1wss1A6xwACOUj31vdK6I,7151
|
|
5
|
+
pymetadataeditor/templates.py,sha256=LmrOsN1PlY-Fb5SKVoOhOYKruI1nP8Ed3QxHxyIz8Nk,25984
|
|
6
|
+
pymetadataeditor/utils.py,sha256=SZO04_wkP-eh1EP7X6BhrCtUGMYwOH54FfqT90Go_ic,10109
|
|
7
|
+
pymetadataeditor-0.1.0.dist-info/LICENSE,sha256=FOubTO8qPACQIWI8ycK24BJGVyQ6ojw-xiymEYZthlI,1067
|
|
8
|
+
pymetadataeditor-0.1.0.dist-info/METADATA,sha256=0jX_tfhz6rybNK8hQMySKuyEqmFZ0CTfH1ov97ZPe3A,11593
|
|
9
|
+
pymetadataeditor-0.1.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
10
|
+
pymetadataeditor-0.1.0.dist-info/RECORD,,
|