eadpy 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- eadpy/__init__.py +8 -0
- eadpy/cli.py +46 -0
- eadpy/ead.py +637 -0
- eadpy/py.typed +0 -0
- eadpy-0.1.0.dist-info/METADATA +86 -0
- eadpy-0.1.0.dist-info/RECORD +8 -0
- eadpy-0.1.0.dist-info/WHEEL +4 -0
- eadpy-0.1.0.dist-info/entry_points.txt +2 -0
eadpy/__init__.py
ADDED
eadpy/cli.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Command-line interface for the EADPy library.
|
|
3
|
+
"""
|
|
4
|
+
import sys
|
|
5
|
+
import os
|
|
6
|
+
from eadpy import Ead
|
|
7
|
+
|
|
8
|
+
def main():
|
|
9
|
+
"""Main entry point for the EADPy command-line interface."""
|
|
10
|
+
if len(sys.argv) < 2:
|
|
11
|
+
print("Usage: eadpy path_to_ead.xml [output_file.json]")
|
|
12
|
+
sys.exit(1)
|
|
13
|
+
|
|
14
|
+
ead_file = sys.argv[1]
|
|
15
|
+
print(f"Loading EAD file: {ead_file}")
|
|
16
|
+
|
|
17
|
+
try:
|
|
18
|
+
if not os.path.exists(ead_file):
|
|
19
|
+
print(f"Error: File '{ead_file}' does not exist.")
|
|
20
|
+
print(f"Current working directory: {os.getcwd()}")
|
|
21
|
+
sys.exit(1)
|
|
22
|
+
|
|
23
|
+
ead = Ead(ead_file)
|
|
24
|
+
print(f"Successfully parsed EAD file: {ead_file}")
|
|
25
|
+
|
|
26
|
+
output_file = None
|
|
27
|
+
if len(sys.argv) == 3:
|
|
28
|
+
output_file = sys.argv[2]
|
|
29
|
+
print(f"Output file: {output_file}")
|
|
30
|
+
else:
|
|
31
|
+
print("No output file specified. Using default: ead.json")
|
|
32
|
+
output_file = "ead.json"
|
|
33
|
+
|
|
34
|
+
ead.create_and_save_chunks(output_file)
|
|
35
|
+
print(f"Successfully wrote output to: {output_file}")
|
|
36
|
+
|
|
37
|
+
except FileNotFoundError as e:
|
|
38
|
+
print(f"Error: {str(e)}")
|
|
39
|
+
print(f"Current working directory: {os.getcwd()}")
|
|
40
|
+
sys.exit(1)
|
|
41
|
+
except Exception as e:
|
|
42
|
+
print(f"Error: {str(e)}")
|
|
43
|
+
sys.exit(1)
|
|
44
|
+
|
|
45
|
+
if __name__ == "__main__":
|
|
46
|
+
main()
|
eadpy/ead.py
ADDED
|
@@ -0,0 +1,637 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import time
|
|
3
|
+
import os
|
|
4
|
+
from lxml import etree
|
|
5
|
+
|
|
6
|
+
class Ead:
|
|
7
|
+
NAME_ELEMENTS = ["corpname", "famname", "name", "persname"]
|
|
8
|
+
|
|
9
|
+
SEARCHABLE_NOTES_FIELDS = [
|
|
10
|
+
"accessrestrict", "accruals", "altformavail", "appraisal", "arrangement",
|
|
11
|
+
"bibliography", "bioghist", "custodhist", "fileplan", "note", "odd",
|
|
12
|
+
"originalsloc", "otherfindaid", "phystech", "prefercite", "processinfo",
|
|
13
|
+
"relatedmaterial", "scopecontent", "separatedmaterial", "userestrict"
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
DID_SEARCHABLE_NOTES_FIELDS = [
|
|
17
|
+
"abstract", "materialspec", "physloc", "note"
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
def __init__(self, file_path):
|
|
21
|
+
"""
|
|
22
|
+
Initialize an Ead object with a file path and parse it immediately.
|
|
23
|
+
|
|
24
|
+
Parameters:
|
|
25
|
+
file_path (str): Path to the EAD XML file
|
|
26
|
+
"""
|
|
27
|
+
self.file_path = file_path
|
|
28
|
+
# A simple counter for generating fallback MD5 IDs
|
|
29
|
+
self.counter = 0
|
|
30
|
+
# Parse the file immediately
|
|
31
|
+
self.data = self.parse()
|
|
32
|
+
|
|
33
|
+
def parse(self):
|
|
34
|
+
"""
|
|
35
|
+
Parse the EAD XML file and return a dictionary representing
|
|
36
|
+
the parsed structure.
|
|
37
|
+
"""
|
|
38
|
+
# Check if the file exists
|
|
39
|
+
if not os.path.exists(self.file_path):
|
|
40
|
+
raise FileNotFoundError(f"EAD file not found: '{self.file_path}'. Please provide a valid file path.")
|
|
41
|
+
|
|
42
|
+
if not os.path.isfile(self.file_path):
|
|
43
|
+
raise IsADirectoryError(f"'{self.file_path}' is a directory, not a file. Please provide a valid XML file.")
|
|
44
|
+
|
|
45
|
+
# Check if the file is readable
|
|
46
|
+
if not os.access(self.file_path, os.R_OK):
|
|
47
|
+
raise PermissionError(f"Permission denied: Unable to read '{self.file_path}'.")
|
|
48
|
+
|
|
49
|
+
try:
|
|
50
|
+
# Parse XML and strip namespaces
|
|
51
|
+
parser = etree.XMLParser(remove_blank_text=True)
|
|
52
|
+
tree = etree.parse(self.file_path, parser)
|
|
53
|
+
self._remove_namespaces(tree)
|
|
54
|
+
|
|
55
|
+
# We will use the root element for subsequent xpath queries
|
|
56
|
+
root = tree.getroot()
|
|
57
|
+
|
|
58
|
+
# Parse the top-level collection
|
|
59
|
+
collection = self._parse_collection(root)
|
|
60
|
+
|
|
61
|
+
# Identify all top-level components (c, c01..c12)
|
|
62
|
+
component_nodes = root.xpath(
|
|
63
|
+
"//archdesc/dsc/c | " + " | ".join(f"//archdesc/dsc/c{i:02d}" for i in range(1, 13))
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
# Parse child components under the main collection
|
|
67
|
+
collection["components"] = self._parse_components(component_nodes, collection["id"])
|
|
68
|
+
|
|
69
|
+
return collection
|
|
70
|
+
except etree.XMLSyntaxError as e:
|
|
71
|
+
raise ValueError(f"Invalid XML in '{self.file_path}': {str(e)}")
|
|
72
|
+
except Exception as e:
|
|
73
|
+
raise RuntimeError(f"Error parsing EAD file '{self.file_path}': {str(e)}")
|
|
74
|
+
|
|
75
|
+
def create_item_chunks(self):
|
|
76
|
+
"""
|
|
77
|
+
Create item-focused chunks that include relevant information from their parent hierarchy.
|
|
78
|
+
Returns a list of chunks ready for embedding.
|
|
79
|
+
"""
|
|
80
|
+
chunks = []
|
|
81
|
+
|
|
82
|
+
def process_items(component, ancestors=None):
|
|
83
|
+
if ancestors is None:
|
|
84
|
+
ancestors = []
|
|
85
|
+
|
|
86
|
+
# Build current component info with relevant details
|
|
87
|
+
current_component = {
|
|
88
|
+
"id": component.get("id", ""),
|
|
89
|
+
"title": component.get("title", ""),
|
|
90
|
+
"level": component.get("level", ""),
|
|
91
|
+
"date": component.get("normalized_date", ""),
|
|
92
|
+
"extent": component.get("extent", [])
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
# Current hierarchy including this component
|
|
96
|
+
current_ancestors = ancestors + [current_component]
|
|
97
|
+
|
|
98
|
+
# For leaf items or any item-level component, create detailed chunks
|
|
99
|
+
is_leaf = "components" not in component or not component["components"]
|
|
100
|
+
is_item = component.get("level") == "item"
|
|
101
|
+
|
|
102
|
+
if is_leaf or is_item:
|
|
103
|
+
# Start with hierarchical path
|
|
104
|
+
hierarchy_titles = [a.get("title") or "" for a in current_ancestors]
|
|
105
|
+
hierarchy_path = " > ".join(hierarchy_titles)
|
|
106
|
+
|
|
107
|
+
# Gather important dates and extent info from ancestors
|
|
108
|
+
ancestor_dates = []
|
|
109
|
+
ancestor_extents = []
|
|
110
|
+
|
|
111
|
+
for ancestor in current_ancestors[:-1]: # Exclude current component
|
|
112
|
+
if ancestor["date"] and ancestor["date"] not in ancestor_dates:
|
|
113
|
+
ancestor_dates.append(ancestor["date"])
|
|
114
|
+
|
|
115
|
+
for extent in ancestor["extent"]:
|
|
116
|
+
if extent and extent not in ancestor_extents:
|
|
117
|
+
ancestor_extents.append(extent)
|
|
118
|
+
|
|
119
|
+
# Create chunk data
|
|
120
|
+
chunk_data = {
|
|
121
|
+
"id": current_component["id"],
|
|
122
|
+
"title": current_component["title"],
|
|
123
|
+
"path": hierarchy_path,
|
|
124
|
+
"level": current_component["level"],
|
|
125
|
+
"date": current_component["date"],
|
|
126
|
+
"ancestor_dates": ancestor_dates,
|
|
127
|
+
"ancestor_extents": ancestor_extents,
|
|
128
|
+
"content": []
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
# Add notes from the current component
|
|
132
|
+
if component.get("notes"):
|
|
133
|
+
for note_type, notes in component["notes"].items():
|
|
134
|
+
if isinstance(notes, list):
|
|
135
|
+
for note in notes:
|
|
136
|
+
if isinstance(note, dict) and "content" in note:
|
|
137
|
+
chunk_data["content"].append({
|
|
138
|
+
"type": note_type,
|
|
139
|
+
"text": " ".join(note["content"])
|
|
140
|
+
})
|
|
141
|
+
else:
|
|
142
|
+
chunk_data["content"].append({
|
|
143
|
+
"type": note_type,
|
|
144
|
+
"text": str(note)
|
|
145
|
+
})
|
|
146
|
+
|
|
147
|
+
# Add extent info from current component
|
|
148
|
+
if current_component["extent"]:
|
|
149
|
+
chunk_data["content"].append({
|
|
150
|
+
"type": "extent",
|
|
151
|
+
"text": ", ".join(current_component["extent"])
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
# Add subjects
|
|
155
|
+
if component.get("access_subjects"):
|
|
156
|
+
chunk_data["content"].append({
|
|
157
|
+
"type": "subjects",
|
|
158
|
+
"text": ", ".join(component["access_subjects"])
|
|
159
|
+
})
|
|
160
|
+
|
|
161
|
+
# Include digital objects if any
|
|
162
|
+
if component.get("digital_objects"):
|
|
163
|
+
digital_texts = []
|
|
164
|
+
for obj in component["digital_objects"]:
|
|
165
|
+
if obj.get("label"):
|
|
166
|
+
digital_texts.append(f"{obj['label']}: {obj.get('href', '')}")
|
|
167
|
+
else:
|
|
168
|
+
digital_texts.append(obj.get('href', ''))
|
|
169
|
+
|
|
170
|
+
if digital_texts:
|
|
171
|
+
chunk_data["content"].append({
|
|
172
|
+
"type": "digital_objects",
|
|
173
|
+
"text": "; ".join(digital_texts)
|
|
174
|
+
})
|
|
175
|
+
|
|
176
|
+
# Add creators if available
|
|
177
|
+
if component.get("creators"):
|
|
178
|
+
creator_texts = []
|
|
179
|
+
for creator in component["creators"]:
|
|
180
|
+
if creator.get("name"):
|
|
181
|
+
creator_texts.append(f"{creator['name']} ({creator.get('type', '')})")
|
|
182
|
+
|
|
183
|
+
if creator_texts:
|
|
184
|
+
chunk_data["content"].append({
|
|
185
|
+
"type": "creators",
|
|
186
|
+
"text": "; ".join(creator_texts)
|
|
187
|
+
})
|
|
188
|
+
|
|
189
|
+
# Generate final text for embedding
|
|
190
|
+
text_parts = [f"Path: {chunk_data['path']}"]
|
|
191
|
+
text_parts.append(f"Title: {chunk_data['title']}")
|
|
192
|
+
|
|
193
|
+
if chunk_data["date"]:
|
|
194
|
+
text_parts.append(f"Date: {chunk_data['date']}")
|
|
195
|
+
|
|
196
|
+
# Include ancestor dates if different from item date
|
|
197
|
+
if ancestor_dates:
|
|
198
|
+
text_parts.append(f"Collection Dates: {', '.join(ancestor_dates)}")
|
|
199
|
+
|
|
200
|
+
# Include ancestor extents
|
|
201
|
+
if ancestor_extents:
|
|
202
|
+
text_parts.append(f"Collection Extent: {', '.join(ancestor_extents)}")
|
|
203
|
+
|
|
204
|
+
# Add all content elements
|
|
205
|
+
for content in chunk_data["content"]:
|
|
206
|
+
text_parts.append(f"{content['type'].capitalize()}: {content['text']}")
|
|
207
|
+
|
|
208
|
+
chunks.append({
|
|
209
|
+
"text": "\n".join(text_parts),
|
|
210
|
+
"metadata": {
|
|
211
|
+
"id": chunk_data["id"],
|
|
212
|
+
"title": chunk_data["title"],
|
|
213
|
+
"level": chunk_data["level"],
|
|
214
|
+
"path": hierarchy_path,
|
|
215
|
+
"date": chunk_data["date"],
|
|
216
|
+
"ancestors": [a["id"] for a in current_ancestors[:-1]],
|
|
217
|
+
"ancestor_titles": hierarchy_titles[:-1]
|
|
218
|
+
}
|
|
219
|
+
})
|
|
220
|
+
|
|
221
|
+
# Recursively process children
|
|
222
|
+
if "components" in component:
|
|
223
|
+
for child in component["components"]:
|
|
224
|
+
process_items(child, current_ancestors)
|
|
225
|
+
|
|
226
|
+
# Start with the collection (self.data is the parsed EAD)
|
|
227
|
+
process_items(self.data)
|
|
228
|
+
|
|
229
|
+
return chunks
|
|
230
|
+
|
|
231
|
+
def save_chunks_to_json(self, chunks, output_file):
|
|
232
|
+
"""
|
|
233
|
+
Save chunks to a JSON file.
|
|
234
|
+
|
|
235
|
+
Parameters:
|
|
236
|
+
chunks (list): List of chunks to save
|
|
237
|
+
output_file (str): Path to the output JSON file
|
|
238
|
+
"""
|
|
239
|
+
import json
|
|
240
|
+
|
|
241
|
+
with open(output_file, 'w') as f:
|
|
242
|
+
json.dump(chunks, f, indent=2)
|
|
243
|
+
|
|
244
|
+
def create_and_save_chunks(self, output_file):
|
|
245
|
+
"""
|
|
246
|
+
Create item-focused chunks and save them to a JSON file.
|
|
247
|
+
|
|
248
|
+
Parameters:
|
|
249
|
+
output_file (str): Path to the output JSON file
|
|
250
|
+
|
|
251
|
+
Returns:
|
|
252
|
+
list: The chunks that were created and saved
|
|
253
|
+
"""
|
|
254
|
+
chunks = self.create_item_chunks()
|
|
255
|
+
self.save_chunks_to_json(chunks, output_file)
|
|
256
|
+
return chunks
|
|
257
|
+
|
|
258
|
+
def _remove_namespaces(self, tree):
|
|
259
|
+
"""
|
|
260
|
+
Remove namespaces in-place from an lxml ElementTree.
|
|
261
|
+
"""
|
|
262
|
+
for elem in tree.getiterator():
|
|
263
|
+
if elem.tag and isinstance(elem.tag, str) and elem.tag.startswith("{"):
|
|
264
|
+
elem.tag = elem.tag.split("}", 1)[1]
|
|
265
|
+
etree.cleanup_namespaces(tree)
|
|
266
|
+
|
|
267
|
+
def _generate_id(self, reference_id, parent_id=None):
|
|
268
|
+
"""
|
|
269
|
+
Generate unique identifier if reference_id is None, otherwise
|
|
270
|
+
prepend parent_id if present.
|
|
271
|
+
"""
|
|
272
|
+
if reference_id:
|
|
273
|
+
return f"{parent_id}_{reference_id}" if parent_id else reference_id
|
|
274
|
+
else:
|
|
275
|
+
# fallback to random hash
|
|
276
|
+
random_str = str(time.time())
|
|
277
|
+
md5_hash = hashlib.md5(random_str.encode("utf-8")).hexdigest()[:9]
|
|
278
|
+
return f"{parent_id}_{md5_hash}" if parent_id else md5_hash
|
|
279
|
+
|
|
280
|
+
def _parse_collection(self, root):
|
|
281
|
+
"""
|
|
282
|
+
Parse the top-level <archdesc> as a 'collection'.
|
|
283
|
+
"""
|
|
284
|
+
ead_id_node = root.xpath("//eadheader/eadid")
|
|
285
|
+
ead_id = ead_id_node[0].text.strip() if ead_id_node else None
|
|
286
|
+
|
|
287
|
+
title = self._parse_title(root)
|
|
288
|
+
normalized_date = self._parse_normalized_date(root)
|
|
289
|
+
|
|
290
|
+
collection = {
|
|
291
|
+
"id": ead_id,
|
|
292
|
+
"level": "collection",
|
|
293
|
+
"title": title,
|
|
294
|
+
"normalized_title": self._normalize_title(title, normalized_date),
|
|
295
|
+
"dates": self._parse_dates(root),
|
|
296
|
+
"normalized_date": normalized_date,
|
|
297
|
+
"creators": self._parse_creators(root),
|
|
298
|
+
"extent": self._parse_extent(root),
|
|
299
|
+
"language": [x.strip() for x in root.xpath("//archdesc/did/langmaterial/text()")],
|
|
300
|
+
"physdesc": self._parse_physdesc(root),
|
|
301
|
+
"repository": self._extract_all_text(root.xpath("//repository")),
|
|
302
|
+
"unitid": self._safe_strip(root.xpath("//archdesc/did/unitid/text()")),
|
|
303
|
+
"notes": self._parse_notes(root),
|
|
304
|
+
"access_subjects": self._parse_access_subjects(root),
|
|
305
|
+
"geo_names": [x.strip() for x in root.xpath("//archdesc/controlaccess/geogname/text()")],
|
|
306
|
+
"digital_objects": self._parse_digital_objects(
|
|
307
|
+
root.xpath("//archdesc/did/dao | //archdesc/dao")
|
|
308
|
+
),
|
|
309
|
+
"has_online_content": len(root.xpath("//dao")) > 0
|
|
310
|
+
}
|
|
311
|
+
return collection
|
|
312
|
+
|
|
313
|
+
def _parse_components(self, component_nodes, parent_id):
|
|
314
|
+
"""
|
|
315
|
+
Recursively parse all <cXX> child components.
|
|
316
|
+
"""
|
|
317
|
+
components = []
|
|
318
|
+
for node in component_nodes:
|
|
319
|
+
ref_id = node.get("id")
|
|
320
|
+
if not ref_id:
|
|
321
|
+
# fallback for missing @id in the node
|
|
322
|
+
self.counter += 1
|
|
323
|
+
fallback = f"{parent_id}_{self.counter}"
|
|
324
|
+
ref_id = hashlib.md5(fallback.encode("utf-8")).hexdigest()[:9]
|
|
325
|
+
|
|
326
|
+
component_id = self._generate_id(ref_id, parent_id)
|
|
327
|
+
|
|
328
|
+
title = self._safe_strip(node.xpath("./did/unittitle/text()"))
|
|
329
|
+
normalized_date = self._parse_normalized_component_date(node)
|
|
330
|
+
|
|
331
|
+
component = {
|
|
332
|
+
"id": component_id,
|
|
333
|
+
"ref_id": ref_id,
|
|
334
|
+
"parent_id": parent_id,
|
|
335
|
+
"level": self._parse_level(node),
|
|
336
|
+
"title": title,
|
|
337
|
+
"normalized_title": self._normalize_title(title, normalized_date),
|
|
338
|
+
"dates": self._parse_component_dates(node),
|
|
339
|
+
"normalized_date": normalized_date,
|
|
340
|
+
"unitid": self._safe_strip(node.xpath("./did/unitid/text()")),
|
|
341
|
+
"creators": self._parse_component_creators(node),
|
|
342
|
+
"extent": self._parse_component_extent(node),
|
|
343
|
+
"notes": self._parse_component_notes(node),
|
|
344
|
+
"containers": self._parse_containers(node),
|
|
345
|
+
"access_subjects": self._parse_component_access_subjects(node),
|
|
346
|
+
"digital_objects": self._parse_digital_objects(
|
|
347
|
+
node.xpath("./dao | ./did/dao")
|
|
348
|
+
),
|
|
349
|
+
"has_online_content": len(node.xpath(".//dao")) > 0,
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
# Recursively parse children of this node
|
|
353
|
+
child_selector = "./c" + "".join(f"|./c{i:02d}" for i in range(1, 13))
|
|
354
|
+
child_nodes = node.xpath(child_selector)
|
|
355
|
+
if child_nodes:
|
|
356
|
+
component["components"] = self._parse_components(child_nodes, component_id)
|
|
357
|
+
|
|
358
|
+
components.append(component)
|
|
359
|
+
|
|
360
|
+
return components
|
|
361
|
+
|
|
362
|
+
def _normalize_title(self, title, date_str):
|
|
363
|
+
"""
|
|
364
|
+
If both title and date_str exist, combine them.
|
|
365
|
+
"""
|
|
366
|
+
if not title or not date_str:
|
|
367
|
+
return title
|
|
368
|
+
return f"{title}, {date_str}"
|
|
369
|
+
|
|
370
|
+
def _parse_title(self, root):
|
|
371
|
+
"""
|
|
372
|
+
Extract the collection-level title (unittitle).
|
|
373
|
+
"""
|
|
374
|
+
title_node = root.xpath("//archdesc/did/unittitle/text()")
|
|
375
|
+
return title_node[0].strip() if title_node else None
|
|
376
|
+
|
|
377
|
+
def _parse_dates(self, root):
|
|
378
|
+
"""
|
|
379
|
+
Return a dict of date types: inclusive, bulk, and other at the collection level.
|
|
380
|
+
"""
|
|
381
|
+
return {
|
|
382
|
+
"inclusive": [x.strip() for x in root.xpath('//archdesc/did/unitdate[@type="inclusive"]/text()')],
|
|
383
|
+
"bulk": [x.strip() for x in root.xpath('//archdesc/did/unitdate[@type="bulk"]/text()')],
|
|
384
|
+
"other": [x.strip() for x in root.xpath('//archdesc/did/unitdate[not(@type)]/text()')]
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
def _parse_component_dates(self, node):
|
|
388
|
+
"""
|
|
389
|
+
Return a dict of date types: inclusive, bulk, and other for a component.
|
|
390
|
+
"""
|
|
391
|
+
return {
|
|
392
|
+
"inclusive": [x.strip() for x in node.xpath('./did/unitdate[@type="inclusive"]/text()')],
|
|
393
|
+
"bulk": [x.strip() for x in node.xpath('./did/unitdate[@type="bulk"]/text()')],
|
|
394
|
+
"other": [x.strip() for x in node.xpath('./did/unitdate[not(@type)]/text()')]
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
def _parse_normalized_date(self, root):
|
|
398
|
+
"""
|
|
399
|
+
Concatenate inclusive, bulk, other collection-level dates into a single string.
|
|
400
|
+
"""
|
|
401
|
+
inclusive = [x.strip() for x in root.xpath('//archdesc/did/unitdate[@type="inclusive"]/text()')]
|
|
402
|
+
bulk = [x.strip() for x in root.xpath('//archdesc/did/unitdate[@type="bulk"]/text()')]
|
|
403
|
+
other = [x.strip() for x in root.xpath('//archdesc/did/unitdate[not(@type)]/text()')]
|
|
404
|
+
|
|
405
|
+
normalized = []
|
|
406
|
+
if inclusive:
|
|
407
|
+
normalized.extend(inclusive)
|
|
408
|
+
if bulk:
|
|
409
|
+
normalized.append(f"bulk {', '.join(bulk)}")
|
|
410
|
+
if other:
|
|
411
|
+
normalized.extend(other)
|
|
412
|
+
|
|
413
|
+
return ", ".join(normalized) if normalized else None
|
|
414
|
+
|
|
415
|
+
def _parse_normalized_component_date(self, node):
|
|
416
|
+
"""
|
|
417
|
+
Concatenate inclusive, bulk, other component-level dates into a single string.
|
|
418
|
+
"""
|
|
419
|
+
inclusive = [x.strip() for x in node.xpath('./did/unitdate[@type="inclusive"]/text()')]
|
|
420
|
+
bulk = [x.strip() for x in node.xpath('./did/unitdate[@type="bulk"]/text()')]
|
|
421
|
+
other = [x.strip() for x in node.xpath('./did/unitdate[not(@type)]/text()')]
|
|
422
|
+
|
|
423
|
+
normalized = []
|
|
424
|
+
if inclusive:
|
|
425
|
+
normalized.extend(inclusive)
|
|
426
|
+
if bulk:
|
|
427
|
+
normalized.append(f"bulk {', '.join(bulk)}")
|
|
428
|
+
if other:
|
|
429
|
+
normalized.extend(other)
|
|
430
|
+
|
|
431
|
+
return ", ".join(normalized) if normalized else None
|
|
432
|
+
|
|
433
|
+
def _parse_extent(self, root):
|
|
434
|
+
"""
|
|
435
|
+
Collect <extent> under the collection-level <physdesc>.
|
|
436
|
+
"""
|
|
437
|
+
return [x.strip() for x in root.xpath('//archdesc/did/physdesc/extent/text()')]
|
|
438
|
+
|
|
439
|
+
def _parse_component_extent(self, node):
|
|
440
|
+
"""
|
|
441
|
+
Collect <extent> for a component-level <physdesc>.
|
|
442
|
+
"""
|
|
443
|
+
return [x.strip() for x in node.xpath('./did/physdesc/extent/text()')]
|
|
444
|
+
|
|
445
|
+
def _parse_physdesc(self, root):
|
|
446
|
+
"""
|
|
447
|
+
Extract textual content from <physdesc> for the collection-level.
|
|
448
|
+
"""
|
|
449
|
+
entries = []
|
|
450
|
+
physdesc_nodes = root.xpath('//archdesc/did/physdesc')
|
|
451
|
+
for pnode in physdesc_nodes:
|
|
452
|
+
# Join all non-element child text
|
|
453
|
+
text_parts = []
|
|
454
|
+
for child in pnode.itertext():
|
|
455
|
+
if child.strip():
|
|
456
|
+
text_parts.append(child.strip())
|
|
457
|
+
joined_text = " ".join(text_parts).strip()
|
|
458
|
+
if joined_text:
|
|
459
|
+
entries.append(joined_text)
|
|
460
|
+
return entries
|
|
461
|
+
|
|
462
|
+
def _parse_creators(self, root):
|
|
463
|
+
"""
|
|
464
|
+
Collect <origination> name elements for the collection.
|
|
465
|
+
"""
|
|
466
|
+
creators = []
|
|
467
|
+
for name_el in self.NAME_ELEMENTS:
|
|
468
|
+
path = f"//archdesc/did/origination/{name_el}/text()"
|
|
469
|
+
for text_node in root.xpath(path):
|
|
470
|
+
creators.append({"type": name_el, "name": text_node.strip()})
|
|
471
|
+
return creators
|
|
472
|
+
|
|
473
|
+
def _parse_component_creators(self, node):
|
|
474
|
+
"""
|
|
475
|
+
Collect <origination> name elements for a component.
|
|
476
|
+
"""
|
|
477
|
+
creators = []
|
|
478
|
+
for name_el in self.NAME_ELEMENTS:
|
|
479
|
+
path = f"./did/origination/{name_el}/text()"
|
|
480
|
+
for text_node in node.xpath(path):
|
|
481
|
+
creators.append({"type": name_el, "name": text_node.strip()})
|
|
482
|
+
return creators
|
|
483
|
+
|
|
484
|
+
def _parse_level(self, node):
|
|
485
|
+
"""
|
|
486
|
+
Return the component's level, falling back to 'otherlevel' if appropriate.
|
|
487
|
+
"""
|
|
488
|
+
level = node.get("level")
|
|
489
|
+
other_level = node.get("otherlevel")
|
|
490
|
+
if level == "otherlevel" and other_level:
|
|
491
|
+
return other_level
|
|
492
|
+
return level
|
|
493
|
+
|
|
494
|
+
def _parse_containers(self, node):
|
|
495
|
+
"""
|
|
496
|
+
Collect all <container> info for a given component.
|
|
497
|
+
"""
|
|
498
|
+
containers = []
|
|
499
|
+
container_nodes = node.xpath('./did/container')
|
|
500
|
+
for c in container_nodes:
|
|
501
|
+
containers.append({
|
|
502
|
+
"type": c.get("type"),
|
|
503
|
+
"value": c.text.strip() if c.text else None
|
|
504
|
+
})
|
|
505
|
+
return containers
|
|
506
|
+
|
|
507
|
+
def _parse_notes(self, root):
|
|
508
|
+
"""
|
|
509
|
+
Parse <archdesc> notes. Some are directly under <archdesc>,
|
|
510
|
+
some are under <archdesc>/did.
|
|
511
|
+
"""
|
|
512
|
+
notes = {}
|
|
513
|
+
|
|
514
|
+
# Parse top-level (archdesc) notes
|
|
515
|
+
for field in self.SEARCHABLE_NOTES_FIELDS:
|
|
516
|
+
content_nodes = root.xpath(f"//archdesc/{field}")
|
|
517
|
+
if content_nodes:
|
|
518
|
+
field_values = []
|
|
519
|
+
for node in content_nodes:
|
|
520
|
+
heading = "".join(node.xpath('./head/text()')).strip()
|
|
521
|
+
content_texts = []
|
|
522
|
+
# gather all text except <head>
|
|
523
|
+
for child in node.xpath('./*[local-name()!="head"]'):
|
|
524
|
+
content_texts.append("".join(child.itertext()).strip())
|
|
525
|
+
|
|
526
|
+
field_values.append({
|
|
527
|
+
"heading": heading,
|
|
528
|
+
"content": content_texts
|
|
529
|
+
})
|
|
530
|
+
notes[field] = field_values
|
|
531
|
+
|
|
532
|
+
# Parse <did> notes fields
|
|
533
|
+
for field in self.DID_SEARCHABLE_NOTES_FIELDS:
|
|
534
|
+
content_nodes = root.xpath(f"//archdesc/did/{field}/text()")
|
|
535
|
+
if content_nodes:
|
|
536
|
+
notes[field] = [c.strip() for c in content_nodes if c.strip()]
|
|
537
|
+
|
|
538
|
+
return notes
|
|
539
|
+
|
|
540
|
+
def _parse_component_notes(self, node):
|
|
541
|
+
"""
|
|
542
|
+
Parse notes for a particular component node (e.g. <cXX>).
|
|
543
|
+
"""
|
|
544
|
+
notes = {}
|
|
545
|
+
|
|
546
|
+
for field in self.SEARCHABLE_NOTES_FIELDS:
|
|
547
|
+
content_nodes = node.xpath(f"./{field}")
|
|
548
|
+
if content_nodes:
|
|
549
|
+
field_values = []
|
|
550
|
+
for cnode in content_nodes:
|
|
551
|
+
heading = "".join(cnode.xpath('./head/text()')).strip()
|
|
552
|
+
content_texts = []
|
|
553
|
+
for child in cnode.xpath('./*[local-name()!="head"]'):
|
|
554
|
+
content_texts.append("".join(child.itertext()).strip())
|
|
555
|
+
|
|
556
|
+
field_values.append({
|
|
557
|
+
"heading": heading,
|
|
558
|
+
"content": content_texts
|
|
559
|
+
})
|
|
560
|
+
notes[field] = field_values
|
|
561
|
+
|
|
562
|
+
for field in self.DID_SEARCHABLE_NOTES_FIELDS:
|
|
563
|
+
content_nodes = node.xpath(f"./did/{field}/text()")
|
|
564
|
+
if content_nodes:
|
|
565
|
+
notes[field] = [c.strip() for c in content_nodes if c.strip()]
|
|
566
|
+
|
|
567
|
+
return notes
|
|
568
|
+
|
|
569
|
+
def _parse_access_subjects(self, root):
|
|
570
|
+
"""
|
|
571
|
+
Collect subject, function, occupation, genreform under <controlaccess> at collection-level.
|
|
572
|
+
"""
|
|
573
|
+
subjects = []
|
|
574
|
+
control_access_nodes = root.xpath("//archdesc/controlaccess")
|
|
575
|
+
for canode in control_access_nodes:
|
|
576
|
+
for selector in ["subject", "function", "occupation", "genreform"]:
|
|
577
|
+
for text_node in canode.xpath(f".//{selector}/text()"):
|
|
578
|
+
if text_node.strip():
|
|
579
|
+
subjects.append(text_node.strip())
|
|
580
|
+
return subjects
|
|
581
|
+
|
|
582
|
+
def _parse_component_access_subjects(self, node):
|
|
583
|
+
"""
|
|
584
|
+
Collect subject, function, occupation, genreform under <controlaccess> at component-level.
|
|
585
|
+
"""
|
|
586
|
+
subjects = []
|
|
587
|
+
control_access_nodes = node.xpath("./controlaccess")
|
|
588
|
+
for canode in control_access_nodes:
|
|
589
|
+
for selector in ["subject", "function", "occupation", "genreform"]:
|
|
590
|
+
for text_node in canode.xpath(f".//{selector}/text()"):
|
|
591
|
+
if text_node.strip():
|
|
592
|
+
subjects.append(text_node.strip())
|
|
593
|
+
return subjects
|
|
594
|
+
|
|
595
|
+
def _parse_digital_objects(self, dao_nodes):
|
|
596
|
+
"""
|
|
597
|
+
Collect digital object references from <dao> or <did/dao>.
|
|
598
|
+
"""
|
|
599
|
+
digital_objects = []
|
|
600
|
+
for dao in dao_nodes:
|
|
601
|
+
# Look for @title attribute or nested <daodesc><p> text
|
|
602
|
+
label = dao.get("title")
|
|
603
|
+
if not label:
|
|
604
|
+
# fallback: look for <daodesc><p> text
|
|
605
|
+
label_candidate = dao.xpath("daodesc/p/text()")
|
|
606
|
+
label = label_candidate[0].strip() if label_candidate else None
|
|
607
|
+
|
|
608
|
+
# Check for href or xlink:href
|
|
609
|
+
href = dao.get("href")
|
|
610
|
+
if not href:
|
|
611
|
+
href = dao.get("{http://www.w3.org/1999/xlink}href")
|
|
612
|
+
|
|
613
|
+
if href:
|
|
614
|
+
digital_objects.append({"label": label, "href": href})
|
|
615
|
+
|
|
616
|
+
return digital_objects
|
|
617
|
+
|
|
618
|
+
def _safe_strip(self, nodes):
|
|
619
|
+
"""
|
|
620
|
+
Return the first node as stripped text if present, else None.
|
|
621
|
+
"""
|
|
622
|
+
if not nodes:
|
|
623
|
+
return None
|
|
624
|
+
if isinstance(nodes, list):
|
|
625
|
+
# lxml returns a list from xpath
|
|
626
|
+
return nodes[0].strip() if nodes[0] else None
|
|
627
|
+
return nodes.strip() if nodes else None
|
|
628
|
+
|
|
629
|
+
def _extract_all_text(self, nodes):
|
|
630
|
+
"""
|
|
631
|
+
Extract all text content from an element including nested elements.
|
|
632
|
+
"""
|
|
633
|
+
if not nodes:
|
|
634
|
+
return None
|
|
635
|
+
|
|
636
|
+
node = nodes[0] if isinstance(nodes, list) else nodes
|
|
637
|
+
return " ".join(node.itertext()).strip() if node is not None else None
|
eadpy/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: eadpy
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Add your description here
|
|
5
|
+
Author-email: Brendan Quinn <brendan-quinn@northwestern.edu>
|
|
6
|
+
Requires-Python: >=3.11
|
|
7
|
+
Requires-Dist: lxml>=5.3.1
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
|
|
10
|
+
# EADPy
|
|
11
|
+
|
|
12
|
+
[](https://pypi.org/project/eadpy/)
|
|
13
|
+
[](https://opensource.org/licenses/MIT)
|
|
14
|
+
|
|
15
|
+
A Python library for working with Encoded Archival Description (EAD) XML documents.
|
|
16
|
+
|
|
17
|
+
## Features
|
|
18
|
+
|
|
19
|
+
- Parse and manipulate EAD XML documents
|
|
20
|
+
- Convert EAD to various formats (JSON, CSV, etc.)
|
|
21
|
+
- Validate EAD documents against schemas
|
|
22
|
+
- Tools for batch processing of EAD files
|
|
23
|
+
|
|
24
|
+
## Installation
|
|
25
|
+
|
|
26
|
+
Install EADPy using pip:
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install eadpy
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Usage
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from eadpy import Ead
|
|
36
|
+
|
|
37
|
+
# Load an EAD file and process it
|
|
38
|
+
ead = Ead("path/to/finding_aid.xml")
|
|
39
|
+
ead.create_and_save_chunks("path/to/output.json")
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Development
|
|
43
|
+
|
|
44
|
+
### Setting up the development environment
|
|
45
|
+
|
|
46
|
+
EADPy uses [uv](https://github.com/astral-sh/uv) for dependency management and virtual environment setup.
|
|
47
|
+
|
|
48
|
+
1. Clone the repository:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
git clone https://github.com/yourusername/eadpy.git
|
|
52
|
+
cd eadpy
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
2. Create and activate a virtual environment:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
uv venv --python 3.13
|
|
59
|
+
source .venv/bin/activate # On Unix/macOS
|
|
60
|
+
# or
|
|
61
|
+
.venv\Scripts\activate # On Windows
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
3. Install development dependencies:
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
uv pip install -e ".[dev]"
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### Running tests
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
pytest
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Documentation
|
|
77
|
+
|
|
78
|
+
For full documentation, visit [eadpy.readthedocs.io](https://eadpy.readthedocs.io).
|
|
79
|
+
|
|
80
|
+
## Contributing
|
|
81
|
+
|
|
82
|
+
Contributions are welcome! Please feel free to submit a Pull Request.
|
|
83
|
+
|
|
84
|
+
## License
|
|
85
|
+
|
|
86
|
+
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
eadpy/__init__.py,sha256=yx9l_4WE4CjEiMEiuML9BjSlBuEuJESy1Zg53k4XdMU,158
|
|
2
|
+
eadpy/cli.py,sha256=2VSGQvS_kTHuqLuXt5zXsEhm-2Y15Iqd9acY6Uwjf7I,1330
|
|
3
|
+
eadpy/ead.py,sha256=o4crRg0J0cWDMSUH1JfikezdpDupDKw2LgLeDiM82ao,26106
|
|
4
|
+
eadpy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
eadpy-0.1.0.dist-info/METADATA,sha256=i4MA4bQpiWyBytGJ35hVu89P6Y_okTh02dXi-YlAH-M,1813
|
|
6
|
+
eadpy-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
7
|
+
eadpy-0.1.0.dist-info/entry_points.txt,sha256=MHbW6xnyrn9A_visLPekorYbcVoeUza2qgBqzpEnooI,41
|
|
8
|
+
eadpy-0.1.0.dist-info/RECORD,,
|