datamule 0.381__py3-none-any.whl → 1.0.2__py3-none-any.whl

datamule/parser/basic_13d_parser.py

@@ -1,58 +0,0 @@
-import re
-from pathlib import Path
-from .helper import load_file_content, clean_title
-
-ITEM_PATTERN = re.compile(
-    r"(?:^[ \t]*)"
-    r"(?:"
-    r"(?:Item|ITEM)\s*"
-    r"(?:"
-    r"1|"
-    r"2|"
-    r"3|"
-    r"4|"
-    r"5|"
-    r"6|"
-    r"7|"
-    r"8|"
-    r"9"
-    r")"
-    r"|"
-    r"SIGNATURES?"
-    r")",
-    re.IGNORECASE | re.MULTILINE
-)
-
-def parse_13d(filename: Path) -> dict:
-    text = load_file_content(filename)
-    matches = [(clean_title(m.group().strip()), m.start()) for m in ITEM_PATTERN.finditer(text)]
-    
-    result = {
-        "metadata": {"document_name": Path(filename).stem},
-        "document": {}
-    }
-    
-    if not matches:
-        return result
-    
-    for i, (current_match, start_pos) in enumerate(matches[:-1]):
-        section_text = WHITESPACE_PATTERN.sub(' ', text[start_pos:matches[i + 1][1]]).strip()
-        if section_text:
-            if "signature" in current_match.lower():
-                key = "signatures" 
-            else:
-                key = f"item{current_match.lower().replace('item', '').strip()}"
-            result["document"][key] = section_text
-    
-    last_match, last_pos = matches[-1]
-    section_text = WHITESPACE_PATTERN.sub(' ', text[last_pos:len(text)]).strip()
-    if section_text:
-       if "signature" in last_match.lower():
-           key = "signatures" 
-       else:
-           key = f"item{last_match.lower().replace('item', '').strip()}"
-       result["document"][key] = section_text
-    
-    return result
-
-WHITESPACE_PATTERN = re.compile(r'\s+')

datamule/parser/basic_13g_parser.py

@@ -1,61 +0,0 @@
-import re
-from pathlib import Path
-from .helper import load_file_content, clean_title
-
-ITEM_PATTERN_13G = re.compile(
-    r"(?:^[ \t]*)"
-    r"(?:"
-    r"(?:Item|ITEM)\s*"
-    r"(?:"
-    r"10|"  # Move 10 to the start so it's matched before 1
-    r"11|"  # Similarly with 11 and 12
-    r"12|"
-    r"1|"
-    r"2|"
-    r"3|"
-    r"4|"
-    r"5|"
-    r"6|"
-    r"7|"
-    r"8|"
-    r"9"
-    r")"
-    r"|"
-    r"SIGNATURES?"
-    r")",
-    re.IGNORECASE | re.MULTILINE
-)
-
-def parse_13g(filename: Path) -> dict:
-   text = load_file_content(filename)
-   matches = [(clean_title(m.group().strip()), m.start()) for m in ITEM_PATTERN_13G.finditer(text)]
-   
-   result = {
-       "metadata": {"document_name": Path(filename).stem},
-       "document": {}
-   }
-   
-   if not matches:
-       return result
-   
-   for i, (current_match, start_pos) in enumerate(matches[:-1]):
-       section_text = WHITESPACE_PATTERN.sub(' ', text[start_pos:matches[i + 1][1]]).strip()
-       if section_text:
-            if "signature" in current_match.lower():
-                key = "signatures" 
-            else:
-                key = f"item{current_match.lower().replace('item', '').strip()}"
-            result["document"][key] = section_text
-   
-   last_match, last_pos = matches[-1]
-   section_text = WHITESPACE_PATTERN.sub(' ', text[last_pos:len(text)]).strip()
-   if section_text:
-        if "signature" in last_match.lower():
-            key = "signatures" 
-        else:
-            key = f"item{last_match.lower().replace('item', '').strip()}"
-        result["document"][key] = section_text
-   
-   return result
-
-WHITESPACE_PATTERN = re.compile(r'\s+')

datamule/parser/basic_8k_parser.py

@@ -1,84 +0,0 @@
-import re
-from pathlib import Path
-from .helper import load_file_content, clean_title
-
-ITEM_PATTERN = re.compile(
-   r"(?:^[ \t]*)"
-   r"(?:"
-   r"(?:Item|ITEM)\s*"
-   r"(?:"
-   r"1\.0[1-4]|"
-   r"2\.0[1-6]|"
-   r"3\.0[1-3]|"
-   r"4\.0[1-2]|"
-   r"5\.0[1-8]|"
-   r"6\.0[1-5]|"
-   r"7\.01|"
-   r"8\.01|"
-   r"9\.01"
-   r")"
-   r"|"
-   r"SIGNATURES?"
-   r")",
-   re.IGNORECASE | re.MULTILINE
-)
-
-WHITESPACE_PATTERN = re.compile(r'\s+')
-
-def parse_section(text: str, start: int, end: int) -> str:
-   return WHITESPACE_PATTERN.sub(' ', text[start:end].strip())
-
-def validate_section_sequence(matches: list) -> None:
-   current_base = None
-   
-   for match, _ in matches:
-       base_section = re.match(r'(?:Item|ITEM)\s*(?:\d+\.\d+|\bSIGNATURES?\b)', match)
-       if base_section:
-           base_section = base_section.group().upper()
-           
-           if current_base is None:
-               current_base = base_section
-           elif base_section != current_base:
-               current_base = base_section
-           else:
-               raise DuplicateSectionError(f"Section {base_section} appears multiple times before a different section")
-
-def parse_8k(filename: Path) -> dict:
-   text = load_file_content(filename)
-   matches = [(clean_title(m.group().strip()), m.start()) for m in ITEM_PATTERN.finditer(text)]
-   
-   result = {
-       "metadata": {"document_name": Path(filename).stem},
-       "document": {}
-   }
-   
-   if not matches:
-       return result
-   
-   validate_section_sequence(matches)
-   
-   # Process all sections except last
-   for i, (current_match, start_pos) in enumerate(matches[:-1]):
-       section_text = parse_section(text, start_pos, matches[i + 1][1])
-       if section_text:
-            if "signature" in current_match.lower():
-                key = "signatures" 
-            else:
-                key = f"item{current_match.lower().replace('item', '').strip()}"
-            result["document"][key] = section_text
-   
-   # Process last section
-   last_match, last_pos = matches[-1]
-   section_text = parse_section(text, last_pos, len(text))
-   if section_text:
-        if "signature" in last_match.lower():
-            key = "signatures" 
-        else:
-            key = f"item{last_match.lower().replace('item', '').strip()}"
-        result["document"][key] = section_text
-    
-   return result
-
-class DuplicateSectionError(Exception):
-   """Raised when a section appears multiple times before a different section."""
-   pass

datamule/parser/form_d_parser.py

@@ -1,70 +0,0 @@
-from xml.etree import ElementTree as ET
-
-def element_to_dict(elem):
-    """Convert an XML element to dict preserving structure."""
-    result = {}
-    
-    # Add attributes directly to result
-    if elem.attrib:
-        result.update(elem.attrib)
-        
-    # Add text content if present and no children
-    if elem.text and elem.text.strip():
-        text = elem.text.strip()
-        if not len(elem):  # No children
-            return text
-        else:
-            result['text'] = text
-            
-    # Process children
-    for child in elem:
-        child_data = element_to_dict(child)
-        child_tag = child.tag.split('}')[-1]  # Remove namespace
-        
-        if child_tag in result:
-            # Convert to list if multiple elements
-            if not isinstance(result[child_tag], list):
-                result[child_tag] = [result[child_tag]]
-            result[child_tag].append(child_data)
-        else:
-            result[child_tag] = child_data
-            
-    return result
-
-def parse_form_d(filepath):
-    """Parse Form D XML file into metadata and document sections."""
-    # Parse XML
-    tree = ET.parse(filepath)
-    root = tree.getroot()
-    
-    # Remove namespaces for cleaner processing
-    for elem in root.iter():
-        if '}' in elem.tag:
-            elem.tag = elem.tag.split('}')[-1]
-    
-    # Convert entire document to dict
-    full_dict = element_to_dict(root)
-    
-    # Separate metadata and document content
-    result = {
-        'metadata': {},
-        'document': {}
-    }
-    
-    # Extract metadata
-    metadata_fields = {
-        'schemaVersion',
-        'submissionType',
-        'testOrLive',
-        'primaryIssuer'  # Including all issuer information in metadata
-    }
-    
-    for field in metadata_fields:
-        if field in full_dict:
-            result['metadata'][field] = full_dict[field]
-            del full_dict[field]  # Remove from full_dict to avoid duplication
-    
-    # Everything else goes to document
-    result['document'] = full_dict
-    
-    return result

datamule/parser/generalized_item_parser.py

@@ -1,78 +0,0 @@
-# Parses e.g. 10-K, 10-Q,..... any form with items and/or parts
-from .helper import load_file_content, clean_title, clean_text
-from pathlib import Path
-import re
-
-# OK figured out general pattern
-# find toc
-# figure out mapping. we do need it
-# just do mapping tonight
-
-pattern = re.compile(r'^\s*(?:item\s+\d+(?:\.\d+)?(?:[a-z])?|signature(?:\.?s)?)\s*', re.I | re.M)
-
-def find_anchors(content):
-   anchors = []
-   prev_title = None
-   
-   for part_match in pattern.finditer(content):
-       title = clean_title(part_match.group())
-       # Skip duplicates, e.g. "item 1" and "item1 continued"
-       if prev_title == title:
-           continue
-       prev_title = title
-       anchors.append((title, part_match.start()))
-   
-   return anchors
-
-# I think this works, but I haven't tested it extensively.
-def map_sections(content, anchors):
-    positions = anchors + [('end', len(content))]
-    
-    result = {}
-    for i, (title, start) in enumerate(positions[:-1]):
-        _, next_start = positions[i + 1]
-        section_text = content[start:next_start].strip()
-        result[title.lower()] = clean_text(section_text)
-    
-    def sort_key(x):
-        match = re.search(r'item\s+(\d+)(?:[\.a-z])?', x[0], re.I)
-        if not match:
-            return float('inf')
-        num = match.group(0).lower()
-        # This will sort 1, 1a, 1b, 2, 2a etc
-        return float(re.findall(r'\d+', num)[0]) + (ord(num[-1]) - ord('a') + 1) / 100 if num[-1].isalpha() else float(re.findall(r'\d+', num)[0])
-        
-    return dict(sorted(result.items(), key=sort_key))
-
-# def find_content_start(anchors):
-#     def find_first_non_repeating(seq):
-#         for i in range(len(seq)):
-#             remaining = seq[i:]
-#             # Get same length subsequence from the next position
-#             next_seq = seq[i + 1:i + 1 + len(remaining)]
-#             if remaining != next_seq and len(next_seq) > 0:
-#                 return i
-#         return 0  # Default to start if no pattern found
-    
-#     return find_first_non_repeating([title for title, _ in anchors])
-
-def generalized_parser(filename):
-    # load content
-    content = load_file_content(filename)
-
-    # find anchors
-    anchors = find_anchors(content)
-
-    # Skip tables of contents. Not implemented yet, since we overwrite the keys anyway.
-    # content_start = find_content_start(anchors)
-    # print(content_start)
-
-    result = {}
-    # assign metadata
-    result["metadata"] = {"document_name": Path(filename).stem}
-
-    # extract sections, assign text based on mapping_dict
-    result['document'] = map_sections(content, anchors)
-
-    return result
-

datamule/parser/helper.py

@@ -1,75 +0,0 @@
-from selectolax.parser import HTMLParser
-import re
-
-# This will be modified in the future to remove SEC specific code such as <PAGE> tags
-def load_text_content(filename):
-    with open(filename) as f:
-        return f.read().translate(str.maketrans({
-            '\xa0': ' ', '\u2003': ' ',
-            '\u2018': "'", '\u2019': "'",
-            '\u201c': '"', '\u201d': '"'
-        }))
-    
-def load_html_content(filename):
-    parser = HTMLParser(open(filename).read())
-    
-    # Remove hidden elements first
-    hidden_nodes = parser.css('[style*="display: none"], [style*="display:none"], .hidden, .hide, .d-none')
-    for node in hidden_nodes:
-        node.decompose()
-    
-    blocks = {'p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'article', 'section', 'li', 'td'}
-    lines = []
-    current_line = []
-    
-    def flush_line():
-        if current_line:
-            lines.append(' '.join(current_line))
-            current_line.clear()
-    
-    for node in parser.root.traverse(include_text=True):
-        if node.tag in ('script', 'style', 'css'):
-            continue
-            
-        if node.tag in blocks:
-            flush_line()
-            lines.append('')
-            
-        if node.text_content:
-            text = node.text_content.strip()
-            if text:
-                if node.tag in blocks:
-                    flush_line()
-                    lines.append(text)
-                    lines.append('')
-                else:
-                    current_line.append(text)
-    
-    flush_line()
-    
-    text = '\n'.join(lines)
-    while '\n\n\n' in text:
-        text = text.replace('\n\n\n', '\n\n')
-    
-    return text.translate(str.maketrans({
-        '\xa0': ' ', '\u2003': ' ',
-        '\u2018': "'", '\u2019': "'",
-        '\u201c': '"', '\u201d': '"'
-    }))
-def load_file_content(filename):
-    if filename.endswith('.txt'):
-        return load_text_content(filename)
-    elif filename.endswith('.html') or filename.endswith('.htm'):
-        return load_html_content(filename)
-    else:
-        raise ValueError(f"Unsupported file type: {filename}")
-
-def clean_title(title: str) -> str:
-    """Clean up section title by removing newlines, periods, and all whitespace, converting to lowercase."""
-    return ''.join(title.replace('\n', '').replace('.', '').split()).lower()
-
-# This is a bit hacky, removes PART IV, PART V etc from the end of the text
-# we do this to avoid having to map for general cases
-def clean_text(text):
-    text = text.strip()
-    return re.sub(r'\s*PART\s+[IVX]+\s*$', '', text, flags=re.I)

datamule/parser/information_table_parser_13fhr.py

@@ -1,41 +0,0 @@
-from xml.etree import ElementTree as ET
-
-def parse_13f_hr_information_table_xml(xml_file):
-    # Parse the XML file
-    tree = ET.parse(xml_file)
-    root = tree.getroot()
-
-    data = []
-    
-    # Iterate through each infoTable
-    for info_table in root.findall('.//{*}infoTable'):
-        row = {
-            'NAMEOFISSUER': info_table.findtext('.//{*}nameOfIssuer') or '',
-            'TITLEOFCLASS': info_table.findtext('.//{*}titleOfClass') or '',
-            'CUSIP': info_table.findtext('.//{*}cusip') or '',
-            'FIGI': info_table.findtext('.//{*}figi') or '',
-            'VALUE': info_table.findtext('.//{*}value') or '',
-            'SSHPRNAMT': '',
-            'SSHPRNAMTTYPE': '',
-            'PUTCALL': info_table.findtext('.//{*}putCall') or '',
-            'INVESTMENTDISCRETION': info_table.findtext('.//{*}investmentDiscretion') or '',
-            'OTHERMANAGER': info_table.findtext('.//{*}otherManager') or '',
-            'VOTING_AUTH_SOLE': '',
-            'VOTING_AUTH_SHARED': '',
-            'VOTING_AUTH_NONE': ''
-        }
-        
-        shrs_or_prn_amt = info_table.find('.//{*}shrsOrPrnAmt')
-        if shrs_or_prn_amt is not None:
-            row['SSHPRNAMT'] = shrs_or_prn_amt.findtext('.//{*}sshPrnamt') or ''
-            row['SSHPRNAMTTYPE'] = shrs_or_prn_amt.findtext('.//{*}sshPrnamtType') or ''
-        
-        voting_authority = info_table.find('.//{*}votingAuthority')
-        if voting_authority is not None:
-            row['VOTING_AUTH_SOLE'] = voting_authority.findtext('.//{*}Sole') or ''
-            row['VOTING_AUTH_SHARED'] = voting_authority.findtext('.//{*}Shared') or ''
-            row['VOTING_AUTH_NONE'] = voting_authority.findtext('.//{*}None') or ''
-        
-        data.append(row)
-
-    return data

datamule/parser/insider_trading_parser.py

@@ -1,158 +0,0 @@
-from xml.etree import ElementTree as ET
-from typing import Dict, Any, Optional
-
-def get_footnotes(doc) -> dict:
-    """Extract footnotes into a lookup dictionary."""
-    return {
-        f.attrib.get('id', ''): f.text.strip() 
-        for f in doc.findall('.//footnotes/footnote')
-    }
-
-def get_value_and_footnote(elem, footnotes: dict) -> dict:
-    """Get value and footnote for a field."""
-    result = {'value': ''}
-    
-    if elem is None:
-        return result
-        
-    # Get value
-    value_elem = elem.find('.//value')
-    if value_elem is not None:
-        result['value'] = value_elem.text or ''
-    
-    # Get footnote if exists
-    footnote_elem = elem.find('.//footnoteId')
-    if footnote_elem is not None:
-        footnote_id = footnote_elem.attrib.get('id', '')
-        if footnote_id in footnotes:
-            result['footnote'] = footnotes[footnote_id]
-    
-    return result
-
-def parse_form345(filepath) -> Dict[str, Any]:
-    """Parse SEC Form XML with enhanced data extraction."""
-    doc = ET.parse(filepath).getroot()
-    if doc is None:
-        return {"error": "No ownershipDocument found"}
-        
-    footnotes = get_footnotes(doc)
-    
-    result = {
-        'metadata': {
-            'schemaVersion': doc.findtext('schemaVersion', ''),
-            'documentType': doc.findtext('documentType', ''),
-            'periodOfReport': doc.findtext('periodOfReport', ''),
-            'dateOfOriginalSubmission': doc.findtext('dateOfOriginalSubmission', ''),
-            'form3HoldingsReported': doc.findtext('form3HoldingsReported', ''),
-            'form4TransactionsReported': doc.findtext('form4TransactionsReported', ''),
-            'issuer': {
-                'cik': doc.findtext('.//issuerCik', ''),
-                'name': doc.findtext('.//issuerName', ''),
-                'tradingSymbol': doc.findtext('.//issuerTradingSymbol', '')
-            },
-            'reportingOwner': {
-                'cik': doc.findtext('.//rptOwnerCik', ''),
-                'name': doc.findtext('.//rptOwnerName', ''),
-                'address': {
-                    'street1': doc.findtext('.//rptOwnerStreet1', ''),
-                    'street2': doc.findtext('.//rptOwnerStreet2', ''),
-                    'city': doc.findtext('.//rptOwnerCity', ''),
-                    'state': doc.findtext('.//rptOwnerState', ''),
-                    'zip': doc.findtext('.//rptOwnerZipCode', ''),
-                    'stateDescription': doc.findtext('.//rptOwnerStateDescription', '')
-                },
-                'relationship': {
-                    'isDirector': doc.findtext('.//isDirector', ''),
-                    'isOfficer': doc.findtext('.//isOfficer', ''),
-                    'isTenPercentOwner': doc.findtext('.//isTenPercentOwner', ''),
-                    'isOther': doc.findtext('.//isOther', ''),
-                    'officerTitle': doc.findtext('.//officerTitle', '')
-                }
-            },
-            'signature': {
-                'name': doc.findtext('.//signatureName', ''),
-                'date': doc.findtext('.//signatureDate', '')
-            }
-        },
-        'holdings': []
-    }
-
-    # Parse non-derivative holdings/transactions
-    for entry in doc.findall('.//nonDerivativeTable/*'):
-        holding = {
-            'type': 'non-derivative',
-            'securityTitle': get_value_and_footnote(entry.find('.//securityTitle'), footnotes),
-            'postTransactionAmounts': {
-                'sharesOwned': get_value_and_footnote(entry.find('.//sharesOwnedFollowingTransaction'), footnotes)
-            },
-            'ownershipNature': {
-                'directOrIndirect': entry.findtext('.//directOrIndirectOwnership/value', '')
-            }
-        }
-
-        # Add transaction fields if present
-        if 'Transaction' in entry.tag:
-            transactionCoding = {
-                'formType': entry.findtext('.//transactionFormType', ''),
-                'code': entry.findtext('.//transactionCode', ''),
-                'equitySwapInvolved': entry.findtext('.//equitySwapInvolved', '')
-            }
-            
-            transactionAmounts = {
-                'shares': get_value_and_footnote(entry.find('.//transactionShares'), footnotes),
-                'pricePerShare': get_value_and_footnote(entry.find('.//transactionPricePerShare'), footnotes),
-                'acquiredDisposedCode': get_value_and_footnote(entry.find('.//transactionAcquiredDisposedCode'), footnotes)
-            }
-            
-            holding.update({
-                'transactionDate': get_value_and_footnote(entry.find('.//transactionDate'), footnotes),
-                'transactionCoding': transactionCoding,
-                'transactionAmounts': transactionAmounts
-            })
-            
-        result['holdings'].append(holding)
-
-    # Parse derivative holdings/transactions
-    for entry in doc.findall('.//derivativeTable/*'):
-        holding = {
-            'type': 'derivative',
-            'securityTitle': get_value_and_footnote(entry.find('.//securityTitle'), footnotes),
-            'conversionOrExercisePrice': get_value_and_footnote(entry.find('.//conversionOrExercisePrice'), footnotes),
-            'exerciseDate': get_value_and_footnote(entry.find('.//exerciseDate'), footnotes),
-            'expirationDate': get_value_and_footnote(entry.find('.//expirationDate'), footnotes),
-            'underlyingSecurity': {
-                'title': get_value_and_footnote(entry.find('.//underlyingSecurityTitle'), footnotes),
-                'shares': get_value_and_footnote(entry.find('.//underlyingSecurityShares'), footnotes)
-            },
-            'postTransactionAmounts': {
-                'sharesOwned': get_value_and_footnote(entry.find('.//sharesOwnedFollowingTransaction'), footnotes)
-            },
-            'ownershipNature': {
-                'directOrIndirect': entry.findtext('.//directOrIndirectOwnership/value', ''),
-                'nature': entry.findtext('.//natureOfOwnership/value', '')
-            }
-        }
-
-        # Add transaction-specific fields
-        if 'Transaction' in entry.tag:
-            transactionCoding = {
-                'formType': entry.findtext('.//transactionFormType', ''),
-                'code': entry.findtext('.//transactionCode', ''),
-                'equitySwapInvolved': entry.findtext('.//equitySwapInvolved', '')
-            }
-            
-            transactionAmounts = {
-                'shares': get_value_and_footnote(entry.find('.//transactionShares'), footnotes),
-                'pricePerShare': get_value_and_footnote(entry.find('.//transactionPricePerShare'), footnotes),
-                'acquiredDisposedCode': get_value_and_footnote(entry.find('.//transactionAcquiredDisposedCode'), footnotes)
-            }
-            
-            holding.update({
-                'transactionDate': get_value_and_footnote(entry.find('.//transactionDate'), footnotes),
-                'transactionCoding': transactionCoding,
-                'transactionAmounts': transactionAmounts
-            })
-            
-        result['holdings'].append(holding)
-
-    return result