datamule 0.416__cp311-cp311-win_amd64.whl → 0.417__cp311-cp311-win_amd64.whl

datamule/parser/document_parsing/basic_10k_parser.py

@@ -0,0 +1,82 @@
+from pathlib import Path
+import re
+from .helper import load_file_content, clean_title
+
+PART_PATTERN = re.compile(r'\n\s*part[.:)?\s]+([IVX]+|\d+)', re.I)
+ITEM_PATTERN = re.compile(r'\n\s*item[.:)?\s]+(\d+[A-Z]?)', re.I)
+IS_10K_PATTERN = re.compile(r'item[.:)?\s]+14', re.I)
+TOC_END_PATTERN = re.compile(r'(?:item[.:)?\s]+14).*?(?=\n\s*item[.:)?\s]+1\b)', re.I | re.DOTALL)
+
+ROMAN_TO_NUM = {'I': '1', 'II': '2', 'III': '3', 'IV': '4'}
+
+ITEM_TO_PART = {
+   '1': 'I', '1A': 'I', '1B': 'I', '1C': 'I', '2': 'I', '3': 'I', '4': 'I',
+   '5': 'II', '6': 'II', '7': 'II', '7A': 'II', '8': 'II', '9': 'II', '9A': 'II', '9B': 'II', '9C': 'II',
+   '10': 'III', '11': 'III', '12': 'III', '13': 'III', '14': 'III',
+   '15': 'IV', '16': 'IV', '16A': 'IV'
+}
+
+def find_content_start(content):
+   toc_match = TOC_END_PATTERN.search(content)
+   if toc_match:
+       item_1_pattern = re.compile(r'\n\s*item\s*1\b', re.I)
+       item_1_match = item_1_pattern.search(content, toc_match.end())
+       if item_1_match:
+           return item_1_match.start()
+   return 0
+
+def find_anchors(content):
+   start_pos = find_content_start(content)
+   content = '\n' + content[start_pos:]
+   
+   anchors = []
+   for part_match in PART_PATTERN.finditer(content):
+       anchors.append(('part', part_match.group(1), part_match.start() + start_pos, part_match.group()))
+       
+   for item_match in ITEM_PATTERN.finditer(content):
+       anchors.append(('item', item_match.group(1), item_match.start() + start_pos, item_match.group()))
+   
+   return sorted(anchors, key=lambda x: x[2])
+
+def extract_sections(content, anchors, filename):
+    if not anchors:
+        return {}
+        
+    result = {
+        "metadata": {"document_name": Path(filename).stem},
+        "document": {
+            "part1": {}, "part2": {}, "part3": {}, "part4": {}
+        }
+    }
+    
+    last_item = None
+    current_text = None
+    
+    for i, current in enumerate(anchors):
+        if current[0] == 'item':
+            next_pos = anchors[i+1][2] if i < len(anchors)-1 else len(content)
+            text = content[current[2]:next_pos].strip()
+            
+            if current[1] == last_item:
+                current_text += "\n\n" + text
+            else:
+                if last_item and last_item in ITEM_TO_PART:
+                    part_num = ROMAN_TO_NUM[ITEM_TO_PART[last_item]]
+                    result["document"][f"part{part_num}"][f"item{last_item.lower()}"] = current_text
+                current_text = text
+                last_item = current[1]
+    
+    if last_item and last_item in ITEM_TO_PART:
+        part_num = ROMAN_TO_NUM[ITEM_TO_PART[last_item]]
+        result["document"][f"part{part_num}"][f"item{last_item.lower()}"] = current_text
+    
+    # Only keep non-empty parts
+    result["document"] = {k:v for k,v in result["document"].items() if v}
+    return result
+
+def parse_10k(filename):
+   content = load_file_content(filename)
+   if not IS_10K_PATTERN.search(content):
+       return {}
+   anchors = find_anchors(content)
+   return extract_sections(content, anchors, filename)

datamule/parser/document_parsing/basic_10q_parser.py

@@ -0,0 +1,73 @@
+from pathlib import Path
+from .helper import load_file_content, clean_title
+import re
+
+PART_II_PATTERN = re.compile(r'\n\s*part\s+II\.?(?:[:\s\.]|$)', re.I)
+ITEM_PATTERN = re.compile(r'\n\s*item\s+(\d+[A-Z]?)\.?(?:[:\s\.]|$)', re.I)
+TOC_END_PATTERN = re.compile(r'(?:item\s*6\.?).*?(?=\n\s*item\s*1\.?\b)', re.I | re.DOTALL)
+
+def find_content_start(content):
+    toc_match = TOC_END_PATTERN.search(content)
+    if toc_match:
+        item_1_pattern = re.compile(r'\n\s*item\s*1\b', re.I)
+        item_1_match = item_1_pattern.search(content, toc_match.end())
+        if item_1_match:
+            return item_1_match.start()
+    return 0
+
+def find_anchors(content):
+    start_pos = find_content_start(content)
+    content = '\n' + content[start_pos:]
+    
+    part_ii_match = PART_II_PATTERN.search(content)
+    part_ii_pos = part_ii_match.start() + start_pos if part_ii_match else None
+
+    anchors = []
+    for item_match in ITEM_PATTERN.finditer(content):
+        anchors.append(('item', item_match.group(1), item_match.start() + start_pos, item_match.group()))
+    
+    return sorted(anchors, key=lambda x: x[2]), part_ii_pos
+
+def extract_sections(content, anchors_and_part2, filename):
+    anchors, part2_pos = anchors_and_part2
+    if not anchors:
+        return {}
+        
+    result = {
+        "metadata": {"document_name": Path(filename).stem},
+        "document": {
+            "part1": {},
+            "part2": {}
+        }
+    }
+    
+    last_item = None
+    current_text = None
+    last_pos = None
+    
+    for i, current in enumerate(anchors):
+        next_pos = anchors[i+1][2] if i < len(anchors)-1 else len(content)
+        
+        if current[1] == last_item:
+            current_text += "\n\n" + content[current[2]:next_pos].strip()
+        else:
+            if last_item is not None:
+                part_key = "part2" if (part2_pos and last_pos >= part2_pos) else "part1"
+                result["document"][part_key][f"item{last_item.lower()}"] = current_text
+            
+            current_text = content[current[2]:next_pos].strip()
+            last_item = current[1]
+            last_pos = current[2]
+    
+    if last_item is not None:
+        part_key = "part2" if (part2_pos and last_pos >= part2_pos) else "part1"
+        result["document"][part_key][f"item{last_item.lower()}"] = current_text
+    
+    # Clean empty parts
+    result["document"] = {k:v for k,v in result["document"].items() if v}
+    return result
+
+def parse_10q(filename):
+    content = load_file_content(filename)
+    anchors_and_part2 = find_anchors(content)
+    return extract_sections(content, anchors_and_part2, filename)

datamule/parser/document_parsing/basic_13d_parser.py

@@ -0,0 +1,58 @@
+import re
+from pathlib import Path
+from .helper import load_file_content, clean_title
+
+ITEM_PATTERN = re.compile(
+    r"(?:^[ \t]*)"
+    r"(?:"
+    r"(?:Item|ITEM)\s*"
+    r"(?:"
+    r"1|"
+    r"2|"
+    r"3|"
+    r"4|"
+    r"5|"
+    r"6|"
+    r"7|"
+    r"8|"
+    r"9"
+    r")"
+    r"|"
+    r"SIGNATURES?"
+    r")",
+    re.IGNORECASE | re.MULTILINE
+)
+
+def parse_13d(filename: Path) -> dict:
+    text = load_file_content(filename)
+    matches = [(clean_title(m.group().strip()), m.start()) for m in ITEM_PATTERN.finditer(text)]
+    
+    result = {
+        "metadata": {"document_name": Path(filename).stem},
+        "document": {}
+    }
+    
+    if not matches:
+        return result
+    
+    for i, (current_match, start_pos) in enumerate(matches[:-1]):
+        section_text = WHITESPACE_PATTERN.sub(' ', text[start_pos:matches[i + 1][1]]).strip()
+        if section_text:
+            if "signature" in current_match.lower():
+                key = "signatures" 
+            else:
+                key = f"item{current_match.lower().replace('item', '').strip()}"
+            result["document"][key] = section_text
+    
+    last_match, last_pos = matches[-1]
+    section_text = WHITESPACE_PATTERN.sub(' ', text[last_pos:len(text)]).strip()
+    if section_text:
+       if "signature" in last_match.lower():
+           key = "signatures" 
+       else:
+           key = f"item{last_match.lower().replace('item', '').strip()}"
+       result["document"][key] = section_text
+    
+    return result
+
+WHITESPACE_PATTERN = re.compile(r'\s+')

datamule/parser/document_parsing/basic_13g_parser.py

@@ -0,0 +1,61 @@
+import re
+from pathlib import Path
+from .helper import load_file_content, clean_title
+
+ITEM_PATTERN_13G = re.compile(
+    r"(?:^[ \t]*)"
+    r"(?:"
+    r"(?:Item|ITEM)\s*"
+    r"(?:"
+    r"10|"  # Move 10 to the start so it's matched before 1
+    r"11|"  # Similarly with 11 and 12
+    r"12|"
+    r"1|"
+    r"2|"
+    r"3|"
+    r"4|"
+    r"5|"
+    r"6|"
+    r"7|"
+    r"8|"
+    r"9"
+    r")"
+    r"|"
+    r"SIGNATURES?"
+    r")",
+    re.IGNORECASE | re.MULTILINE
+)
+
+def parse_13g(filename: Path) -> dict:
+   text = load_file_content(filename)
+   matches = [(clean_title(m.group().strip()), m.start()) for m in ITEM_PATTERN_13G.finditer(text)]
+   
+   result = {
+       "metadata": {"document_name": Path(filename).stem},
+       "document": {}
+   }
+   
+   if not matches:
+       return result
+   
+   for i, (current_match, start_pos) in enumerate(matches[:-1]):
+       section_text = WHITESPACE_PATTERN.sub(' ', text[start_pos:matches[i + 1][1]]).strip()
+       if section_text:
+            if "signature" in current_match.lower():
+                key = "signatures" 
+            else:
+                key = f"item{current_match.lower().replace('item', '').strip()}"
+            result["document"][key] = section_text
+   
+   last_match, last_pos = matches[-1]
+   section_text = WHITESPACE_PATTERN.sub(' ', text[last_pos:len(text)]).strip()
+   if section_text:
+        if "signature" in last_match.lower():
+            key = "signatures" 
+        else:
+            key = f"item{last_match.lower().replace('item', '').strip()}"
+        result["document"][key] = section_text
+   
+   return result
+
+WHITESPACE_PATTERN = re.compile(r'\s+')

datamule/parser/document_parsing/basic_8k_parser.py

@@ -0,0 +1,84 @@
+import re
+from pathlib import Path
+from .helper import load_file_content, clean_title
+
+ITEM_PATTERN = re.compile(
+   r"(?:^[ \t]*)"
+   r"(?:"
+   r"(?:Item|ITEM)\s*"
+   r"(?:"
+   r"1\.0[1-4]|"
+   r"2\.0[1-6]|"
+   r"3\.0[1-3]|"
+   r"4\.0[1-2]|"
+   r"5\.0[1-8]|"
+   r"6\.0[1-5]|"
+   r"7\.01|"
+   r"8\.01|"
+   r"9\.01"
+   r")"
+   r"|"
+   r"SIGNATURES?"
+   r")",
+   re.IGNORECASE | re.MULTILINE
+)
+
+WHITESPACE_PATTERN = re.compile(r'\s+')
+
+def parse_section(text: str, start: int, end: int) -> str:
+   return WHITESPACE_PATTERN.sub(' ', text[start:end].strip())
+
+def validate_section_sequence(matches: list) -> None:
+   current_base = None
+   
+   for match, _ in matches:
+       base_section = re.match(r'(?:Item|ITEM)\s*(?:\d+\.\d+|\bSIGNATURES?\b)', match)
+       if base_section:
+           base_section = base_section.group().upper()
+           
+           if current_base is None:
+               current_base = base_section
+           elif base_section != current_base:
+               current_base = base_section
+           else:
+               raise DuplicateSectionError(f"Section {base_section} appears multiple times before a different section")
+
+def parse_8k(filename: Path) -> dict:
+   text = load_file_content(filename)
+   matches = [(clean_title(m.group().strip()), m.start()) for m in ITEM_PATTERN.finditer(text)]
+   
+   result = {
+       "metadata": {"document_name": Path(filename).stem},
+       "document": {}
+   }
+   
+   if not matches:
+       return result
+   
+   validate_section_sequence(matches)
+   
+   # Process all sections except last
+   for i, (current_match, start_pos) in enumerate(matches[:-1]):
+       section_text = parse_section(text, start_pos, matches[i + 1][1])
+       if section_text:
+            if "signature" in current_match.lower():
+                key = "signatures" 
+            else:
+                key = f"item{current_match.lower().replace('item', '').strip()}"
+            result["document"][key] = section_text
+   
+   # Process last section
+   last_match, last_pos = matches[-1]
+   section_text = parse_section(text, last_pos, len(text))
+   if section_text:
+        if "signature" in last_match.lower():
+            key = "signatures" 
+        else:
+            key = f"item{last_match.lower().replace('item', '').strip()}"
+        result["document"][key] = section_text
+    
+   return result
+
+class DuplicateSectionError(Exception):
+   """Raised when a section appears multiple times before a different section."""
+   pass

datamule/parser/document_parsing/form_d_parser.py

@@ -0,0 +1,70 @@
+from xml.etree import ElementTree as ET
+
+def element_to_dict(elem):
+    """Convert an XML element to dict preserving structure."""
+    result = {}
+    
+    # Add attributes directly to result
+    if elem.attrib:
+        result.update(elem.attrib)
+        
+    # Add text content if present and no children
+    if elem.text and elem.text.strip():
+        text = elem.text.strip()
+        if not len(elem):  # No children
+            return text
+        else:
+            result['text'] = text
+            
+    # Process children
+    for child in elem:
+        child_data = element_to_dict(child)
+        child_tag = child.tag.split('}')[-1]  # Remove namespace
+        
+        if child_tag in result:
+            # Convert to list if multiple elements
+            if not isinstance(result[child_tag], list):
+                result[child_tag] = [result[child_tag]]
+            result[child_tag].append(child_data)
+        else:
+            result[child_tag] = child_data
+            
+    return result
+
+def parse_form_d(filepath):
+    """Parse Form D XML file into metadata and document sections."""
+    # Parse XML
+    tree = ET.parse(filepath)
+    root = tree.getroot()
+    
+    # Remove namespaces for cleaner processing
+    for elem in root.iter():
+        if '}' in elem.tag:
+            elem.tag = elem.tag.split('}')[-1]
+    
+    # Convert entire document to dict
+    full_dict = element_to_dict(root)
+    
+    # Separate metadata and document content
+    result = {
+        'metadata': {},
+        'document': {}
+    }
+    
+    # Extract metadata
+    metadata_fields = {
+        'schemaVersion',
+        'submissionType',
+        'testOrLive',
+        'primaryIssuer'  # Including all issuer information in metadata
+    }
+    
+    for field in metadata_fields:
+        if field in full_dict:
+            result['metadata'][field] = full_dict[field]
+            del full_dict[field]  # Remove from full_dict to avoid duplication
+    
+    # Everything else goes to document
+    result['document'] = full_dict
+    
+    return result

datamule/parser/document_parsing/generalized_item_parser.py

@@ -0,0 +1,78 @@
+# Parses e.g. 10-K, 10-Q,..... any form with items and/or parts
+from .helper import load_file_content, clean_title, clean_text
+from pathlib import Path
+import re
+
+# OK figured out general pattern
+# find toc
+# figure out mapping. we do need it
+# just do mapping tonight
+
+pattern = re.compile(r'^\s*(?:item\s+\d+(?:\.\d+)?(?:[a-z])?|signature(?:\.?s)?)\s*', re.I | re.M)
+
+def find_anchors(content):
+   anchors = []
+   prev_title = None
+   
+   for part_match in pattern.finditer(content):
+       title = clean_title(part_match.group())
+       # Skip duplicates, e.g. "item 1" and "item1 continued"
+       if prev_title == title:
+           continue
+       prev_title = title
+       anchors.append((title, part_match.start()))
+   
+   return anchors
+
+# I think this works, but I haven't tested it extensively.
+def map_sections(content, anchors):
+    positions = anchors + [('end', len(content))]
+    
+    result = {}
+    for i, (title, start) in enumerate(positions[:-1]):
+        _, next_start = positions[i + 1]
+        section_text = content[start:next_start].strip()
+        result[title.lower()] = clean_text(section_text)
+    
+    def sort_key(x):
+        match = re.search(r'item\s+(\d+)(?:[\.a-z])?', x[0], re.I)
+        if not match:
+            return float('inf')
+        num = match.group(0).lower()
+        # This will sort 1, 1a, 1b, 2, 2a etc
+        return float(re.findall(r'\d+', num)[0]) + (ord(num[-1]) - ord('a') + 1) / 100 if num[-1].isalpha() else float(re.findall(r'\d+', num)[0])
+        
+    return dict(sorted(result.items(), key=sort_key))
+
+# def find_content_start(anchors):
+#     def find_first_non_repeating(seq):
+#         for i in range(len(seq)):
+#             remaining = seq[i:]
+#             # Get same length subsequence from the next position
+#             next_seq = seq[i + 1:i + 1 + len(remaining)]
+#             if remaining != next_seq and len(next_seq) > 0:
+#                 return i
+#         return 0  # Default to start if no pattern found
+    
+#     return find_first_non_repeating([title for title, _ in anchors])
+
+def generalized_parser(filename):
+    # load content
+    content = load_file_content(filename)
+
+    # find anchors
+    anchors = find_anchors(content)
+
+    # Skip tables of contents. Not implemented yet, since we overwrite the keys anyway.
+    # content_start = find_content_start(anchors)
+    # print(content_start)
+
+    result = {}
+    # assign metadata
+    result["metadata"] = {"document_name": Path(filename).stem}
+
+    # extract sections, assign text based on mapping_dict
+    result['document'] = map_sections(content, anchors)
+
+    return result
+

datamule/parser/document_parsing/helper.py

@@ -0,0 +1,75 @@
+from selectolax.parser import HTMLParser
+import re
+
+# This will be modified in the future to remove SEC specific code such as <PAGE> tags
+def load_text_content(filename):
+    with open(filename) as f:
+        return f.read().translate(str.maketrans({
+            '\xa0': ' ', '\u2003': ' ',
+            '\u2018': "'", '\u2019': "'",
+            '\u201c': '"', '\u201d': '"'
+        }))
+    
+def load_html_content(filename):
+    parser = HTMLParser(open(filename).read())
+    
+    # Remove hidden elements first
+    hidden_nodes = parser.css('[style*="display: none"], [style*="display:none"], .hidden, .hide, .d-none')
+    for node in hidden_nodes:
+        node.decompose()
+    
+    blocks = {'p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'article', 'section', 'li', 'td'}
+    lines = []
+    current_line = []
+    
+    def flush_line():
+        if current_line:
+            lines.append(' '.join(current_line))
+            current_line.clear()
+    
+    for node in parser.root.traverse(include_text=True):
+        if node.tag in ('script', 'style', 'css'):
+            continue
+            
+        if node.tag in blocks:
+            flush_line()
+            lines.append('')
+            
+        if node.text_content:
+            text = node.text_content.strip()
+            if text:
+                if node.tag in blocks:
+                    flush_line()
+                    lines.append(text)
+                    lines.append('')
+                else:
+                    current_line.append(text)
+    
+    flush_line()
+    
+    text = '\n'.join(lines)
+    while '\n\n\n' in text:
+        text = text.replace('\n\n\n', '\n\n')
+    
+    return text.translate(str.maketrans({
+        '\xa0': ' ', '\u2003': ' ',
+        '\u2018': "'", '\u2019': "'",
+        '\u201c': '"', '\u201d': '"'
+    }))
+def load_file_content(filename):
+    if filename.suffix =='.txt':
+        return load_text_content(filename)
+    elif filename.suffix in ['.html','.htm']:
+        return load_html_content(filename)
+    else:
+        raise ValueError(f"Unsupported file type: {filename}")
+
+def clean_title(title: str) -> str:
+    """Clean up section title by removing newlines, periods, and all whitespace, converting to lowercase."""
+    return ''.join(title.replace('\n', '').replace('.', '').split()).lower()
+
+# This is a bit hacky, removes PART IV, PART V etc from the end of the text
+# we do this to avoid having to map for general cases
+def clean_text(text):
+    text = text.strip()
+    return re.sub(r'\s*PART\s+[IVX]+\s*$', '', text, flags=re.I)

datamule/parser/document_parsing/information_table_parser_13fhr.py

@@ -0,0 +1,41 @@
+from xml.etree import ElementTree as ET
+
+def parse_13f_hr_information_table_xml(xml_file):
+    # Parse the XML file
+    tree = ET.parse(xml_file)
+    root = tree.getroot()
+
+    data = []
+    
+    # Iterate through each infoTable
+    for info_table in root.findall('.//{*}infoTable'):
+        row = {
+            'NAMEOFISSUER': info_table.findtext('.//{*}nameOfIssuer') or '',
+            'TITLEOFCLASS': info_table.findtext('.//{*}titleOfClass') or '',
+            'CUSIP': info_table.findtext('.//{*}cusip') or '',
+            'FIGI': info_table.findtext('.//{*}figi') or '',
+            'VALUE': info_table.findtext('.//{*}value') or '',
+            'SSHPRNAMT': '',
+            'SSHPRNAMTTYPE': '',
+            'PUTCALL': info_table.findtext('.//{*}putCall') or '',
+            'INVESTMENTDISCRETION': info_table.findtext('.//{*}investmentDiscretion') or '',
+            'OTHERMANAGER': info_table.findtext('.//{*}otherManager') or '',
+            'VOTING_AUTH_SOLE': '',
+            'VOTING_AUTH_SHARED': '',
+            'VOTING_AUTH_NONE': ''
+        }
+        
+        shrs_or_prn_amt = info_table.find('.//{*}shrsOrPrnAmt')
+        if shrs_or_prn_amt is not None:
+            row['SSHPRNAMT'] = shrs_or_prn_amt.findtext('.//{*}sshPrnamt') or ''
+            row['SSHPRNAMTTYPE'] = shrs_or_prn_amt.findtext('.//{*}sshPrnamtType') or ''
+        
+        voting_authority = info_table.find('.//{*}votingAuthority')
+        if voting_authority is not None:
+            row['VOTING_AUTH_SOLE'] = voting_authority.findtext('.//{*}Sole') or ''
+            row['VOTING_AUTH_SHARED'] = voting_authority.findtext('.//{*}Shared') or ''
+            row['VOTING_AUTH_NONE'] = voting_authority.findtext('.//{*}None') or ''
+        
+        data.append(row)
+
+    return data