regexml 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- regexml/__init__.py +24 -0
- regexml/attributes.py +60 -0
- regexml/extractor.py +121 -0
- regexml/inspector.py +85 -0
- regexml/models.py +176 -0
- regexml/nested.py +150 -0
- regexml/utils.py +29 -0
- regexml-0.1.0.dist-info/METADATA +168 -0
- regexml-0.1.0.dist-info/RECORD +11 -0
- regexml-0.1.0.dist-info/WHEEL +4 -0
- regexml-0.1.0.dist-info/licenses/LICENSE +21 -0
regexml/__init__.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""regexml: A lightweight, fault-tolerant, text-first XML extraction library for Python."""
|
|
2
|
+
|
|
3
|
+
from regexml.models import TagMatch, TagReport, XMLText
|
|
4
|
+
from regexml.extractor import (
|
|
5
|
+
tagvalue,
|
|
6
|
+
tagvalues,
|
|
7
|
+
tagmatch,
|
|
8
|
+
tagmatches,
|
|
9
|
+
extract_tags,
|
|
10
|
+
)
|
|
11
|
+
from regexml.inspector import check_tags
|
|
12
|
+
|
|
13
|
+
__version__ = "0.1.0"
|
|
14
|
+
__all__ = [
|
|
15
|
+
"XMLText",
|
|
16
|
+
"TagMatch",
|
|
17
|
+
"TagReport",
|
|
18
|
+
"tagvalue",
|
|
19
|
+
"tagvalues",
|
|
20
|
+
"tagmatch",
|
|
21
|
+
"tagmatches",
|
|
22
|
+
"extract_tags",
|
|
23
|
+
"check_tags",
|
|
24
|
+
]
|
regexml/attributes.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Resilient attribute parsing for regexml."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
import re
|
|
5
|
+
from typing import Dict, Optional
|
|
6
|
+
|
|
7
|
+
# Resilient attribute regex supporting double quotes, single quotes, unquoted values, and boolean flags
|
|
8
|
+
ATTR_PATTERN = re.compile(
|
|
9
|
+
r'([a-zA-Z0-9_:-]+)(?:\s*=\s*(?:"([^"]*)"|\'([^\']*)\'|([^\s>]+)))?',
|
|
10
|
+
re.MULTILINE,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def parse_attributes(attr_string: str) -> Dict[str, str]:
|
|
15
|
+
"""
|
|
16
|
+
Parse XML/HTML attributes from a tag's attribute string into a dictionary.
|
|
17
|
+
Handles unquoted values, single quotes, double quotes, and boolean attributes.
|
|
18
|
+
"""
|
|
19
|
+
if not attr_string or not attr_string.strip():
|
|
20
|
+
return {}
|
|
21
|
+
|
|
22
|
+
attributes: Dict[str, str] = {}
|
|
23
|
+
for match in ATTR_PATTERN.finditer(attr_string):
|
|
24
|
+
name = match.group(1)
|
|
25
|
+
if not name:
|
|
26
|
+
continue
|
|
27
|
+
|
|
28
|
+
# Determine value among the three quote capture groups
|
|
29
|
+
val_double = match.group(2)
|
|
30
|
+
val_single = match.group(3)
|
|
31
|
+
val_unquoted = match.group(4)
|
|
32
|
+
|
|
33
|
+
if val_double is not None:
|
|
34
|
+
value = val_double
|
|
35
|
+
elif val_single is not None:
|
|
36
|
+
value = val_single
|
|
37
|
+
elif val_unquoted is not None:
|
|
38
|
+
value = val_unquoted
|
|
39
|
+
else:
|
|
40
|
+
# Boolean attribute without value (e.g. <input disabled>)
|
|
41
|
+
value = ""
|
|
42
|
+
|
|
43
|
+
attributes[name] = value
|
|
44
|
+
|
|
45
|
+
return attributes
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def matches_attributes(
|
|
49
|
+
parsed_attrs: Dict[str, str],
|
|
50
|
+
filter_attrs: Optional[Dict[str, str]],
|
|
51
|
+
) -> bool:
|
|
52
|
+
"""Check if all key-value pairs in filter_attrs match the parsed attributes."""
|
|
53
|
+
if not filter_attrs:
|
|
54
|
+
return True
|
|
55
|
+
|
|
56
|
+
for k, v in filter_attrs.items():
|
|
57
|
+
if parsed_attrs.get(k) != v:
|
|
58
|
+
return False
|
|
59
|
+
|
|
60
|
+
return True
|
regexml/extractor.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Core XML extraction engine."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
from typing import Any, Dict, List, Optional
|
|
5
|
+
from regexml.models import _MISSING, TagMatch, XMLText
|
|
6
|
+
from regexml.nested import find_balanced_tags
|
|
7
|
+
from regexml.utils import remove_comments, strip_all_tags, unwrap_cdata
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def tagmatches(
|
|
11
|
+
text: str,
|
|
12
|
+
tag: str,
|
|
13
|
+
ignore_case: bool = False,
|
|
14
|
+
match_attributes: Optional[Dict[str, str]] = None,
|
|
15
|
+
allow_unclosed: bool = True,
|
|
16
|
+
ignore_comments: bool = True,
|
|
17
|
+
match_namespace: bool = False,
|
|
18
|
+
**kwargs: Any,
|
|
19
|
+
) -> List[TagMatch]:
|
|
20
|
+
"""
|
|
21
|
+
Extract all matching occurrences of a tag as TagMatch objects,
|
|
22
|
+
using balanced depth scanning to accurately capture nested tags.
|
|
23
|
+
"""
|
|
24
|
+
if text is None:
|
|
25
|
+
return []
|
|
26
|
+
|
|
27
|
+
input_text = str(text)
|
|
28
|
+
if ignore_comments:
|
|
29
|
+
input_text = remove_comments(input_text)
|
|
30
|
+
|
|
31
|
+
return find_balanced_tags(
|
|
32
|
+
text=input_text,
|
|
33
|
+
tag_name=tag,
|
|
34
|
+
ignore_case=ignore_case,
|
|
35
|
+
match_attributes=match_attributes,
|
|
36
|
+
allow_unclosed=allow_unclosed,
|
|
37
|
+
match_namespace=match_namespace,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def tagmatch(
|
|
42
|
+
text: str,
|
|
43
|
+
tag: str,
|
|
44
|
+
default: Any = None,
|
|
45
|
+
**kwargs: Any,
|
|
46
|
+
) -> Optional[TagMatch]:
|
|
47
|
+
"""Extract the first matching TagMatch object for a tag."""
|
|
48
|
+
matches = tagmatches(text, tag, **kwargs)
|
|
49
|
+
return matches[0] if matches else default
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def tagvalues(
|
|
53
|
+
text: str,
|
|
54
|
+
tag: str,
|
|
55
|
+
include_tags: bool = False,
|
|
56
|
+
strip_inner_tags: bool = False,
|
|
57
|
+
strip_whitespace: bool = True,
|
|
58
|
+
unwrap_cdata_content: bool = True,
|
|
59
|
+
**kwargs: Any,
|
|
60
|
+
) -> List[XMLText]:
|
|
61
|
+
"""
|
|
62
|
+
Extract all matching values for a tag name, returned as a list of XMLText instances.
|
|
63
|
+
"""
|
|
64
|
+
matches = tagmatches(text, tag, **kwargs)
|
|
65
|
+
results: List[XMLText] = []
|
|
66
|
+
|
|
67
|
+
for m in matches:
|
|
68
|
+
content = m.raw if include_tags else m.content
|
|
69
|
+
|
|
70
|
+
if unwrap_cdata_content:
|
|
71
|
+
content = unwrap_cdata(content)
|
|
72
|
+
|
|
73
|
+
if strip_inner_tags:
|
|
74
|
+
content = strip_all_tags(content)
|
|
75
|
+
|
|
76
|
+
if strip_whitespace:
|
|
77
|
+
content = content.strip()
|
|
78
|
+
|
|
79
|
+
results.append(
|
|
80
|
+
XMLText(
|
|
81
|
+
content,
|
|
82
|
+
is_truncated=m.is_truncated,
|
|
83
|
+
attributes=m.attributes,
|
|
84
|
+
tag=m.tag,
|
|
85
|
+
)
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
return results
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def tagvalue(
|
|
92
|
+
text: str,
|
|
93
|
+
tag: str,
|
|
94
|
+
default: Any = _MISSING,
|
|
95
|
+
**kwargs: Any,
|
|
96
|
+
) -> Any:
|
|
97
|
+
"""
|
|
98
|
+
Extract the value of the first matching tag.
|
|
99
|
+
Returns XMLText instance or default if not found.
|
|
100
|
+
"""
|
|
101
|
+
values = tagvalues(text, tag, **kwargs)
|
|
102
|
+
if values:
|
|
103
|
+
return values[0]
|
|
104
|
+
if default is not _MISSING:
|
|
105
|
+
return default
|
|
106
|
+
return XMLText("", is_none=True)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def extract_tags(
|
|
110
|
+
text: str,
|
|
111
|
+
tags: List[str],
|
|
112
|
+
**kwargs: Any,
|
|
113
|
+
) -> Dict[str, List[XMLText]]:
|
|
114
|
+
"""
|
|
115
|
+
Batch extract multiple distinct tag names from the input text.
|
|
116
|
+
Returns a dictionary mapping tag name to list of XMLText values.
|
|
117
|
+
"""
|
|
118
|
+
output: Dict[str, List[XMLText]] = {}
|
|
119
|
+
for t in tags:
|
|
120
|
+
output[t] = tagvalues(text, t, **kwargs)
|
|
121
|
+
return output
|
regexml/inspector.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""Tag inspection, hierarchy discovery, and unclosed tag diagnostics."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
import re
|
|
5
|
+
from typing import Any, Dict, List
|
|
6
|
+
from regexml.models import TagReport
|
|
7
|
+
from regexml.utils import remove_comments
|
|
8
|
+
|
|
9
|
+
TAG_TOKEN_RE = re.compile(
|
|
10
|
+
r"(?P<closing></(?P<c_tag>[a-zA-Z0-9_.:-]+)\s*>)|"
|
|
11
|
+
r"(?P<self_closing><(?P<s_tag>[a-zA-Z0-9_.:-]+)(?:\s+[^>]*?)?\s*/>)|"
|
|
12
|
+
r"(?P<opening><(?P<o_tag>[a-zA-Z0-9_.:-]+)(?:\s+[^>]*?)?>)",
|
|
13
|
+
re.DOTALL,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def check_tags(
|
|
18
|
+
text: str,
|
|
19
|
+
ignore_comments: bool = True,
|
|
20
|
+
) -> TagReport:
|
|
21
|
+
"""
|
|
22
|
+
Inspect all XML tags within the provided text, building a tag hierarchy
|
|
23
|
+
and identifying any unclosed/truncated tags without validating strict well-formedness.
|
|
24
|
+
"""
|
|
25
|
+
if text is None:
|
|
26
|
+
return TagReport(tags=[], tag_counts={}, unclosed_tags=[], hierarchy={}, total_tags=0)
|
|
27
|
+
|
|
28
|
+
input_text = str(text)
|
|
29
|
+
if ignore_comments:
|
|
30
|
+
input_text = remove_comments(input_text)
|
|
31
|
+
|
|
32
|
+
tag_counts: Dict[str, int] = {}
|
|
33
|
+
stack: List[str] = []
|
|
34
|
+
|
|
35
|
+
# Hierarchy tree root
|
|
36
|
+
root_hierarchy: Dict[str, Any] = {}
|
|
37
|
+
node_stack: List[Dict[str, Any]] = [root_hierarchy]
|
|
38
|
+
|
|
39
|
+
total_tags = 0
|
|
40
|
+
|
|
41
|
+
for match in TAG_TOKEN_RE.finditer(input_text):
|
|
42
|
+
if match.group("self_closing"):
|
|
43
|
+
s_tag = match.group("s_tag")
|
|
44
|
+
total_tags += 1
|
|
45
|
+
tag_counts[s_tag] = tag_counts.get(s_tag, 0) + 1
|
|
46
|
+
current_node = node_stack[-1]
|
|
47
|
+
if s_tag not in current_node:
|
|
48
|
+
current_node[s_tag] = {}
|
|
49
|
+
|
|
50
|
+
elif match.group("opening"):
|
|
51
|
+
o_tag = match.group("o_tag")
|
|
52
|
+
total_tags += 1
|
|
53
|
+
tag_counts[o_tag] = tag_counts.get(o_tag, 0) + 1
|
|
54
|
+
|
|
55
|
+
current_node = node_stack[-1]
|
|
56
|
+
if o_tag not in current_node:
|
|
57
|
+
current_node[o_tag] = {}
|
|
58
|
+
|
|
59
|
+
# Push to stack
|
|
60
|
+
stack.append(o_tag)
|
|
61
|
+
node_stack.append(current_node[o_tag])
|
|
62
|
+
|
|
63
|
+
elif match.group("closing"):
|
|
64
|
+
c_tag = match.group("c_tag")
|
|
65
|
+
# If matching opening tag exists on stack
|
|
66
|
+
if c_tag in stack:
|
|
67
|
+
# Pop back up to matching tag to handle missing intermediate tags
|
|
68
|
+
while stack:
|
|
69
|
+
popped = stack.pop()
|
|
70
|
+
if len(node_stack) > 1:
|
|
71
|
+
node_stack.pop()
|
|
72
|
+
if popped == c_tag:
|
|
73
|
+
break
|
|
74
|
+
|
|
75
|
+
# Any tags remaining on the stack at EOF are unclosed/truncated tags
|
|
76
|
+
unclosed_tags = list(stack)
|
|
77
|
+
tags_list = list(tag_counts.keys())
|
|
78
|
+
|
|
79
|
+
return TagReport(
|
|
80
|
+
tags=tags_list,
|
|
81
|
+
tag_counts=tag_counts,
|
|
82
|
+
unclosed_tags=unclosed_tags,
|
|
83
|
+
hierarchy=root_hierarchy,
|
|
84
|
+
total_tags=total_tags,
|
|
85
|
+
)
|
regexml/models.py
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
"""Data models and XMLText wrapper class for regexml."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import Any, Dict, List, Optional, Union
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class TagMatch:
|
|
10
|
+
"""Represents a matched tag with its content, attributes, and location."""
|
|
11
|
+
tag: str
|
|
12
|
+
content: str
|
|
13
|
+
attributes: Dict[str, str] = field(default_factory=dict)
|
|
14
|
+
raw: str = ""
|
|
15
|
+
start_pos: int = -1
|
|
16
|
+
end_pos: int = -1
|
|
17
|
+
is_truncated: bool = False
|
|
18
|
+
|
|
19
|
+
def to_xml_text(self) -> XMLText:
|
|
20
|
+
"""Convert match content to an XMLText instance carrying metadata."""
|
|
21
|
+
return XMLText(
|
|
22
|
+
self.content,
|
|
23
|
+
is_truncated=self.is_truncated,
|
|
24
|
+
attributes=self.attributes,
|
|
25
|
+
tag=self.tag,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class TagReport:
|
|
31
|
+
"""Diagnostic report produced by check_tags()."""
|
|
32
|
+
tags: List[str]
|
|
33
|
+
tag_counts: Dict[str, int]
|
|
34
|
+
unclosed_tags: List[str]
|
|
35
|
+
hierarchy: Dict[str, Any]
|
|
36
|
+
total_tags: int = 0
|
|
37
|
+
|
|
38
|
+
def tree(self) -> str:
|
|
39
|
+
"""Render a terminal-friendly visual ASCII tree of the tag hierarchy."""
|
|
40
|
+
lines: List[str] = []
|
|
41
|
+
|
|
42
|
+
def _render_node(node: Dict[str, Any], prefix: str = ""):
|
|
43
|
+
items = list(node.items())
|
|
44
|
+
for i, (tag_name, children) in enumerate(items):
|
|
45
|
+
is_last = i == len(items) - 1
|
|
46
|
+
connector = "└── " if is_last else "├── "
|
|
47
|
+
unclosed_flag = " (unclosed)" if tag_name in self.unclosed_tags else ""
|
|
48
|
+
lines.append(f"{prefix}{connector}{tag_name}{unclosed_flag}")
|
|
49
|
+
new_prefix = prefix + (" " if is_last else "│ ")
|
|
50
|
+
if isinstance(children, dict) and children:
|
|
51
|
+
_render_node(children, new_prefix)
|
|
52
|
+
|
|
53
|
+
if not self.hierarchy:
|
|
54
|
+
return "(empty or no tags found)"
|
|
55
|
+
|
|
56
|
+
_render_node(self.hierarchy)
|
|
57
|
+
return "\n".join(lines)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
_MISSING = object()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class XMLText(str):
|
|
64
|
+
"""
|
|
65
|
+
A string subclass that adds resilient XML extraction methods (.tagvalue, .tagvalues, .check_tags).
|
|
66
|
+
|
|
67
|
+
Acts as a drop-in replacement for standard Python strings with 100% compatibility.
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
def __new__(
|
|
71
|
+
cls,
|
|
72
|
+
content: str = "",
|
|
73
|
+
is_truncated: bool = False,
|
|
74
|
+
is_none: bool = False,
|
|
75
|
+
attributes: Optional[Dict[str, str]] = None,
|
|
76
|
+
tag: Optional[str] = None,
|
|
77
|
+
):
|
|
78
|
+
obj = super().__new__(cls, content)
|
|
79
|
+
obj._is_truncated = is_truncated
|
|
80
|
+
obj._is_none = is_none
|
|
81
|
+
obj._attributes = attributes or {}
|
|
82
|
+
obj._tag = tag
|
|
83
|
+
return obj
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def is_truncated(self) -> bool:
|
|
87
|
+
"""True if the tag content was cut off before its closing tag (e.g., EOF reached)."""
|
|
88
|
+
return self._is_truncated
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def is_none(self) -> bool:
|
|
92
|
+
"""True if this represents an empty/missing tag in a chain."""
|
|
93
|
+
return self._is_none
|
|
94
|
+
|
|
95
|
+
@property
|
|
96
|
+
def attributes(self) -> Dict[str, str]:
|
|
97
|
+
"""Parsed attributes associated with this tag match."""
|
|
98
|
+
return self._attributes
|
|
99
|
+
|
|
100
|
+
@property
|
|
101
|
+
def tag(self) -> Optional[str]:
|
|
102
|
+
"""Name of the tag this content was extracted from."""
|
|
103
|
+
return self._tag
|
|
104
|
+
|
|
105
|
+
def __bool__(self) -> bool:
|
|
106
|
+
"""Evaluate falsy if is_none is True or content is empty."""
|
|
107
|
+
if self._is_none:
|
|
108
|
+
return False
|
|
109
|
+
return len(self) > 0
|
|
110
|
+
|
|
111
|
+
def tagvalue(
|
|
112
|
+
self,
|
|
113
|
+
tag_path: str,
|
|
114
|
+
default: Any = _MISSING,
|
|
115
|
+
**kwargs: Any,
|
|
116
|
+
) -> Any:
|
|
117
|
+
"""
|
|
118
|
+
Extract the value of a tag or a nested path of tags (e.g. 'div/name' or 'div.name').
|
|
119
|
+
Supports method chaining: doc.tagvalue('div').tagvalue('name').
|
|
120
|
+
"""
|
|
121
|
+
from regexml.extractor import tagvalue as _extract_tagvalue
|
|
122
|
+
|
|
123
|
+
# Safe chaining: if this node was already missing in a previous chain call
|
|
124
|
+
if self._is_none:
|
|
125
|
+
if default is not _MISSING:
|
|
126
|
+
return default
|
|
127
|
+
return XMLText("", is_none=True)
|
|
128
|
+
|
|
129
|
+
# Handle path notation: e.g. "div/name/first_name" or "div.name.first_name"
|
|
130
|
+
if "/" in tag_path:
|
|
131
|
+
parts = [p.strip() for p in tag_path.split("/") if p.strip()]
|
|
132
|
+
elif "." in tag_path:
|
|
133
|
+
parts = [p.strip() for p in tag_path.split(".") if p.strip()]
|
|
134
|
+
else:
|
|
135
|
+
parts = [tag_path.strip()]
|
|
136
|
+
|
|
137
|
+
current: Any = self
|
|
138
|
+
for idx, part in enumerate(parts):
|
|
139
|
+
is_last = idx == len(parts) - 1
|
|
140
|
+
step_default = default if is_last else _MISSING
|
|
141
|
+
|
|
142
|
+
if hasattr(current, "is_none") and current.is_none:
|
|
143
|
+
if step_default is not _MISSING:
|
|
144
|
+
return step_default
|
|
145
|
+
return XMLText("", is_none=True)
|
|
146
|
+
|
|
147
|
+
current = _extract_tagvalue(str(current), part, default=step_default, **kwargs)
|
|
148
|
+
if current is None or (isinstance(current, XMLText) and current.is_none):
|
|
149
|
+
if is_last and default is not _MISSING:
|
|
150
|
+
return default
|
|
151
|
+
current = XMLText("", is_none=True)
|
|
152
|
+
|
|
153
|
+
if isinstance(current, XMLText) and current.is_none and default is not _MISSING:
|
|
154
|
+
return default
|
|
155
|
+
|
|
156
|
+
return current
|
|
157
|
+
|
|
158
|
+
def tagvalues(
|
|
159
|
+
self,
|
|
160
|
+
tag: str,
|
|
161
|
+
**kwargs: Any,
|
|
162
|
+
) -> List[XMLText]:
|
|
163
|
+
"""Extract all occurrences of a tag as a list of XMLText objects."""
|
|
164
|
+
from regexml.extractor import tagvalues as _extract_tagvalues
|
|
165
|
+
|
|
166
|
+
if self._is_none:
|
|
167
|
+
return []
|
|
168
|
+
return _extract_tagvalues(str(self), tag, **kwargs)
|
|
169
|
+
|
|
170
|
+
def check_tags(self, **kwargs: Any) -> TagReport:
|
|
171
|
+
"""Inspect all tags, nesting, and unclosed/truncated tags."""
|
|
172
|
+
from regexml.inspector import check_tags as _check_tags
|
|
173
|
+
|
|
174
|
+
if self._is_none:
|
|
175
|
+
return TagReport(tags=[], tag_counts={}, unclosed_tags=[], hierarchy={}, total_tags=0)
|
|
176
|
+
return _check_tags(str(self), **kwargs)
|
regexml/nested.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"""Balanced depth-counter scanner for nested XML tags."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
import re
|
|
5
|
+
from typing import Dict, List, Optional
|
|
6
|
+
from regexml.models import TagMatch
|
|
7
|
+
from regexml.attributes import parse_attributes, matches_attributes
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def find_balanced_tags(
|
|
11
|
+
text: str,
|
|
12
|
+
tag_name: str,
|
|
13
|
+
ignore_case: bool = False,
|
|
14
|
+
match_attributes: Optional[Dict[str, str]] = None,
|
|
15
|
+
allow_unclosed: bool = True,
|
|
16
|
+
match_namespace: bool = False,
|
|
17
|
+
) -> List[TagMatch]:
|
|
18
|
+
"""
|
|
19
|
+
Find balanced matches for a tag name using a depth counter.
|
|
20
|
+
|
|
21
|
+
Ensures that when a tag contains nested child tags of the same name:
|
|
22
|
+
<div id="outer">
|
|
23
|
+
<div id="inner">...</div>
|
|
24
|
+
</div>
|
|
25
|
+
the entire outer block is captured from the first opening tag to its corresponding
|
|
26
|
+
balanced closing tag, rather than stopping prematurely at the first closing tag.
|
|
27
|
+
"""
|
|
28
|
+
flags = re.DOTALL
|
|
29
|
+
if ignore_case:
|
|
30
|
+
flags |= re.IGNORECASE
|
|
31
|
+
|
|
32
|
+
escaped_tag = re.escape(tag_name)
|
|
33
|
+
tag_pattern_str = escaped_tag if match_namespace else rf"(?:[a-zA-Z0-9_.-]+:)?{escaped_tag}"
|
|
34
|
+
|
|
35
|
+
# Token regex matches:
|
|
36
|
+
# 1. Opening tag: <tag ...>
|
|
37
|
+
# 2. Closing tag: </tag>
|
|
38
|
+
# 3. Self-closing tag: <tag ... />
|
|
39
|
+
token_re = re.compile(
|
|
40
|
+
rf"(?P<closing></(?P<c_tag>{tag_pattern_str})\s*>)|"
|
|
41
|
+
rf"(?P<self_closing><(?P<s_tag>{tag_pattern_str})(?P<s_attrs>(?:\s+[^>]*?)?)\s*/>)|"
|
|
42
|
+
rf"(?P<opening><(?P<o_tag>{tag_pattern_str})(?P<o_attrs>(?:\s+[^>]*?)?)>)",
|
|
43
|
+
flags,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
matches: List[TagMatch] = []
|
|
47
|
+
text_len = len(text)
|
|
48
|
+
idx = 0
|
|
49
|
+
|
|
50
|
+
while idx < text_len:
|
|
51
|
+
# Search for the next token from current idx
|
|
52
|
+
m = token_re.search(text, idx)
|
|
53
|
+
if not m:
|
|
54
|
+
break
|
|
55
|
+
|
|
56
|
+
# Check token type
|
|
57
|
+
if m.group("self_closing"):
|
|
58
|
+
# Self-closing tag: completes immediately
|
|
59
|
+
tag_name_found = m.group("s_tag")
|
|
60
|
+
attrs_str = m.group("s_attrs") or ""
|
|
61
|
+
parsed_attrs = parse_attributes(attrs_str)
|
|
62
|
+
|
|
63
|
+
if matches_attributes(parsed_attrs, match_attributes):
|
|
64
|
+
match_obj = TagMatch(
|
|
65
|
+
tag=tag_name_found,
|
|
66
|
+
content="",
|
|
67
|
+
attributes=parsed_attrs,
|
|
68
|
+
raw=m.group(0),
|
|
69
|
+
start_pos=m.start(),
|
|
70
|
+
end_pos=m.end(),
|
|
71
|
+
is_truncated=False,
|
|
72
|
+
)
|
|
73
|
+
matches.append(match_obj)
|
|
74
|
+
|
|
75
|
+
idx = m.end()
|
|
76
|
+
continue
|
|
77
|
+
|
|
78
|
+
elif m.group("opening"):
|
|
79
|
+
# Opening tag: start tracking balanced depth
|
|
80
|
+
tag_name_found = m.group("o_tag")
|
|
81
|
+
attrs_str = m.group("o_attrs") or ""
|
|
82
|
+
parsed_attrs = parse_attributes(attrs_str)
|
|
83
|
+
|
|
84
|
+
# Check if this opening tag satisfies attribute filter
|
|
85
|
+
if not matches_attributes(parsed_attrs, match_attributes):
|
|
86
|
+
idx = m.end()
|
|
87
|
+
continue
|
|
88
|
+
|
|
89
|
+
outer_start = m.start()
|
|
90
|
+
content_start = m.end()
|
|
91
|
+
depth = 1
|
|
92
|
+
inner_idx = content_start
|
|
93
|
+
found_closing = False
|
|
94
|
+
|
|
95
|
+
while inner_idx < text_len:
|
|
96
|
+
inner_m = token_re.search(text, inner_idx)
|
|
97
|
+
if not inner_m:
|
|
98
|
+
break
|
|
99
|
+
|
|
100
|
+
if inner_m.group("self_closing"):
|
|
101
|
+
# Self-closing does not alter depth
|
|
102
|
+
inner_idx = inner_m.end()
|
|
103
|
+
elif inner_m.group("opening"):
|
|
104
|
+
# Nested opening tag of same name
|
|
105
|
+
depth += 1
|
|
106
|
+
inner_idx = inner_m.end()
|
|
107
|
+
elif inner_m.group("closing"):
|
|
108
|
+
# Closing tag found
|
|
109
|
+
depth -= 1
|
|
110
|
+
if depth == 0:
|
|
111
|
+
# Balanced closing tag reached!
|
|
112
|
+
content = text[content_start:inner_m.start()]
|
|
113
|
+
raw = text[outer_start:inner_m.end()]
|
|
114
|
+
match_obj = TagMatch(
|
|
115
|
+
tag=tag_name_found,
|
|
116
|
+
content=content,
|
|
117
|
+
attributes=parsed_attrs,
|
|
118
|
+
raw=raw,
|
|
119
|
+
start_pos=outer_start,
|
|
120
|
+
end_pos=inner_m.end(),
|
|
121
|
+
is_truncated=False,
|
|
122
|
+
)
|
|
123
|
+
matches.append(match_obj)
|
|
124
|
+
found_closing = True
|
|
125
|
+
idx = inner_m.end()
|
|
126
|
+
break
|
|
127
|
+
else:
|
|
128
|
+
inner_idx = inner_m.end()
|
|
129
|
+
|
|
130
|
+
if not found_closing:
|
|
131
|
+
# If closing tag was not found and allow_unclosed is enabled (truncated XML)
|
|
132
|
+
if allow_unclosed:
|
|
133
|
+
content = text[content_start:]
|
|
134
|
+
raw = text[outer_start:]
|
|
135
|
+
match_obj = TagMatch(
|
|
136
|
+
tag=tag_name_found,
|
|
137
|
+
content=content,
|
|
138
|
+
attributes=parsed_attrs,
|
|
139
|
+
raw=raw,
|
|
140
|
+
start_pos=outer_start,
|
|
141
|
+
end_pos=text_len,
|
|
142
|
+
is_truncated=True,
|
|
143
|
+
)
|
|
144
|
+
matches.append(match_obj)
|
|
145
|
+
break
|
|
146
|
+
else:
|
|
147
|
+
# Standalone closing tag without opening; skip
|
|
148
|
+
idx = m.end()
|
|
149
|
+
|
|
150
|
+
return matches
|
regexml/utils.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""String cleanup and regex helper utilities for regexml."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
import re
|
|
5
|
+
|
|
6
|
+
TAG_STRIP_PATTERN = re.compile(r"<[^>]+>", re.DOTALL)
|
|
7
|
+
COMMENT_PATTERN = re.compile(r"<!--.*?-->", re.DOTALL)
|
|
8
|
+
CDATA_PATTERN = re.compile(r"<!\[CDATA\[(.*?)\]\]>", re.DOTALL)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def strip_all_tags(text: str) -> str:
|
|
12
|
+
"""Strip all XML/HTML markup tags from text, leaving only the inner content."""
|
|
13
|
+
if not text:
|
|
14
|
+
return ""
|
|
15
|
+
return TAG_STRIP_PATTERN.sub("", text)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def remove_comments(text: str) -> str:
|
|
19
|
+
"""Remove XML comments (<!-- ... -->) from text."""
|
|
20
|
+
if not text:
|
|
21
|
+
return ""
|
|
22
|
+
return COMMENT_PATTERN.sub("", text)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def unwrap_cdata(text: str) -> str:
|
|
26
|
+
"""Extract contents of CDATA blocks: <![CDATA[ content ]]> -> content."""
|
|
27
|
+
if not text:
|
|
28
|
+
return ""
|
|
29
|
+
return CDATA_PATTERN.sub(r"\1", text)
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: regexml
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A lightweight, fault-tolerant, text-first XML extraction library for Python.
|
|
5
|
+
Project-URL: Homepage, https://github.com/visitishan/regexml
|
|
6
|
+
Project-URL: Repository, https://github.com/visitishan/regexml
|
|
7
|
+
Project-URL: Issues, https://github.com/visitishan/regexml/issues
|
|
8
|
+
Author-email: Ishan <ishan@example.com>
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: extractor,fault-tolerant,llm,parser,regex,scraping,xml
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Text Processing :: Markup :: XML
|
|
22
|
+
Requires-Python: >=3.9
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
|
|
25
|
+
Requires-Dist: pytest>=7.0.0; extra == 'dev'
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# regexml 🚀
|
|
29
|
+
|
|
30
|
+
[](https://pypi.org/project/regexml/)
|
|
31
|
+
[](https://python.org)
|
|
32
|
+
[](LICENSE)
|
|
33
|
+
|
|
34
|
+
A lightweight, fault-tolerant, text-first XML extraction library for Python. Available on PyPI: **[pypi.org/project/regexml](https://pypi.org/project/regexml/)**.
|
|
35
|
+
|
|
36
|
+
Unlike traditional XML parsers (`xml.etree.ElementTree`, `lxml`) that enforce strict document validation and crash with `ParseError` on incomplete or malformed markup, **`regexml`** treats XML purely as plain text. It extracts values directly using regular expressions and resilient pattern matching.
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
40
|
+
## 🌟 Why `regexml`?
|
|
41
|
+
|
|
42
|
+
- **Fault-Tolerant & Resilient:** Doesn't crash on unescaped `&`, `<`, `>`, missing closing tags, or broken XML headers.
|
|
43
|
+
- **Perfect for LLM Outputs:** Cleanly extract tags like `<thought>`, `<answer>`, or `<code>` from AI responses.
|
|
44
|
+
- **Fluent Dot-Syntax Chaining:** Call `data.tagvalue("div").tagvalue("name")` seamlessly.
|
|
45
|
+
- **Balanced Outermost Tag Extraction:** Accurately captures outer tags with nested tags of the same name without premature cut-offs.
|
|
46
|
+
- **Truncated Stream Recovery:** If an XML stream cuts off prematurely (e.g. at line 1000 of 1200), it safely captures data up to EOF without failing.
|
|
47
|
+
- **Tag Inspector & Hierarchy:** Built-in `check_tags()` discovers all tags, reports unclosed/truncated tags, and renders visual ASCII trees.
|
|
48
|
+
- **Zero Heavy Dependencies:** Built 100% on Python standard library.
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## 📦 Installation
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
pip install regexml
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
---
|
|
59
|
+
|
|
60
|
+
## ⚡ Quick Start
|
|
61
|
+
|
|
62
|
+
### 1. Fluent Dot-Syntax (`XMLText`)
|
|
63
|
+
|
|
64
|
+
Wrap any string with `XMLText` to unlock dot-methods while retaining 100% string compatibility:
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
from regexml import XMLText
|
|
68
|
+
|
|
69
|
+
raw_data = """
|
|
70
|
+
<article id="101">
|
|
71
|
+
<title>Resilient XML Extraction</title>
|
|
72
|
+
<author>
|
|
73
|
+
<first_name>Ishan</first_name>
|
|
74
|
+
<last_name>Jain</last_name>
|
|
75
|
+
</author>
|
|
76
|
+
</article>
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
doc = XMLText(raw_data)
|
|
80
|
+
|
|
81
|
+
# Extract single tag value:
|
|
82
|
+
print(doc.tagvalue("title"))
|
|
83
|
+
# Output: Resilient XML Extraction
|
|
84
|
+
|
|
85
|
+
# Method Chaining:
|
|
86
|
+
print(doc.tagvalue("author").tagvalue("first_name"))
|
|
87
|
+
# Output: Ishan
|
|
88
|
+
|
|
89
|
+
# Path Shorthand:
|
|
90
|
+
print(doc.tagvalue("author/last_name"))
|
|
91
|
+
# Output: Jain
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
### 2. Functional API (Without Wrapping)
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
from regexml import tagvalue, tagvalues
|
|
98
|
+
|
|
99
|
+
data = "<user><name>Alice</name><name>Bob</name></user>"
|
|
100
|
+
|
|
101
|
+
print(tagvalue(data, "name")) # Alice
|
|
102
|
+
print(tagvalues(data, "name")) # ['Alice', 'Bob']
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
---
|
|
106
|
+
|
|
107
|
+
## 🎯 Advanced Features
|
|
108
|
+
|
|
109
|
+
### Balanced Nested Tags of the Same Name
|
|
110
|
+
Standard regex chops off at the first closing tag. `regexml` tracks depth counter to return the true balanced outermost tag:
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
xml = """
|
|
114
|
+
<div id="outer">
|
|
115
|
+
Hello
|
|
116
|
+
<div id="inner">World</div>
|
|
117
|
+
Welcome!
|
|
118
|
+
</div>
|
|
119
|
+
"""
|
|
120
|
+
doc = XMLText(xml)
|
|
121
|
+
print(doc.tagvalue("div"))
|
|
122
|
+
# Output:
|
|
123
|
+
# Hello
|
|
124
|
+
# <div id="inner">World</div>
|
|
125
|
+
# Welcome!
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
### Truncated / Broken XML Recovery
|
|
129
|
+
If your XML was cut off mid-stream (e.g., from network drop or LLM token limit):
|
|
130
|
+
|
|
131
|
+
```python
|
|
132
|
+
truncated_xml = "<user><id>123</id><bio>Software engineer who loves Python... [CUT OFF]"
|
|
133
|
+
doc = XMLText(truncated_xml)
|
|
134
|
+
|
|
135
|
+
bio = doc.tagvalue("bio")
|
|
136
|
+
print(bio) # "Software engineer who loves Python... [CUT OFF]"
|
|
137
|
+
print(bio.is_truncated) # True
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
### Tag Inspection & Broken Markup Detection (`check_tags`)
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
report = doc.check_tags()
|
|
144
|
+
|
|
145
|
+
# View all tags found:
|
|
146
|
+
print(report.tags) # ['user', 'id', 'bio']
|
|
147
|
+
|
|
148
|
+
# Pinpoint unclosed tags (truncated points):
|
|
149
|
+
print(report.unclosed_tags) # ['user', 'bio']
|
|
150
|
+
|
|
151
|
+
# Print terminal ASCII tree:
|
|
152
|
+
print(report.tree())
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
**Output:**
|
|
156
|
+
```text
|
|
157
|
+
└── user (unclosed)
|
|
158
|
+
├── id
|
|
159
|
+
└── bio (unclosed)
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
The tree instantly visualizes the markup hierarchy and flags unclosed tags, making it easy to pinpoint where the XML stream cut off.
|
|
163
|
+
|
|
164
|
+
---
|
|
165
|
+
|
|
166
|
+
## 📄 License
|
|
167
|
+
|
|
168
|
+
MIT License. Developed by Ishan.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
regexml/__init__.py,sha256=2_CkT4CNkX912FGzFUbLwHVdajRgE2tQyQShh4yda2Y,491
|
|
2
|
+
regexml/attributes.py,sha256=yB6xJJi6O5s4oeLn87yEpALH-1VGsvrLS0Du5aaPmls,1737
|
|
3
|
+
regexml/extractor.py,sha256=UqAwpB3WhfXbfLtFE2LqIFk8clFDKTd_v8fh4jyf1dU,3088
|
|
4
|
+
regexml/inspector.py,sha256=mtDWsXM9qvKLtEVJJqIWkU4Zaj5N5YrvL0S9c5I1RbY,2746
|
|
5
|
+
regexml/models.py,sha256=qtGGIJOxX7wdXMq731OmCp1gbRZEPWEaz4g6MCsaCw0,5910
|
|
6
|
+
regexml/nested.py,sha256=dm-nhGCoHLbXEZ4IISHd0N4bySbyd35IS3gkFXZSVhE,5372
|
|
7
|
+
regexml/utils.py,sha256=kb7bl9tJyWscFrxhWwBD4h934kXp5C93Op9orUYcClc,849
|
|
8
|
+
regexml-0.1.0.dist-info/METADATA,sha256=iaT2ucifhxMDyEDCROb4WuTRYJrPKl9c-VyH79gwZ34,5150
|
|
9
|
+
regexml-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
10
|
+
regexml-0.1.0.dist-info/licenses/LICENSE,sha256=aQ9bAE2ZfPPX0z_1Qy1dHN-uKRZZcVRzsM0gFf7JEtM,1062
|
|
11
|
+
regexml-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ishan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|