datamule 0.422__cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

datamule/parser/document_parsing/mappings.py

@@ -0,0 +1,95 @@
+# I will liekely move this file to a more appropriate location in the future
+
+mapping_dict_10k = {
+    'filing_summary': 'Annual report providing comprehensive overview of company business, financial performance, risks, and operations. Contains audited financial statements, business description, risk analysis, and detailed operational metrics.',
+    
+    'structure': {
+        'part1': {
+            'summary': 'Overview of company operations, risks, and material business information. Contains key business strategy, market position, competitive landscape, and significant challenges.',
+            'item1': {
+                'summary': 'Detailed description of business operations including primary products/services, markets served, distribution methods, competitive conditions, regulatory environment, and business segments'
+            },
+            'item1a': {
+                'summary': 'Comprehensive list and explanation of significant risks and uncertainties that could affect business performance, financial condition, and stock value'
+            },
+            'item1b': {
+                'summary': 'Disclosure of any unresolved comments or issues raised by SEC staff regarding company filings'
+            },
+            'item1c': {
+                'summary': 'Information about cybersecurity risks, incidents, risk management, governance, and strategy'
+            },
+            'item2': {
+                'summary': 'Description of principal physical properties, including manufacturing facilities, offices, warehouses, and other significant real estate'
+            },
+            'item3': {
+                'summary': 'Description of material pending legal proceedings, including potential impacts on business'
+            },
+            'item4': {
+                'summary': 'Disclosure of mine safety violations, citations, and orders received under the Mine Act'
+            }
+        },
+        'part2': {
+            'summary': 'Detailed financial performance analysis, including management insights, market risks, and complete audited financial statements.',
+            'item5': {
+                'summary': 'Information about company stock, including market data, price history, dividends, share repurchases, and securities offerings'
+            },
+            'item6': {
+                'summary': 'Selected historical financial data showing trends in financial condition and results over past 5 years'
+            },
+            'item7': {
+                'summary': 'Management\'s analysis of financial condition, operations results, liquidity, capital resources, and future outlook'
+            },
+            'item7a': {
+                'summary': 'Discussion of exposure to market risk including interest rates, foreign exchange, commodities, and hedging activities'
+            },
+            'item8': {
+                'summary': 'Audited financial statements, including balance sheets, income statements, cash flows, and comprehensive notes'
+            },
+            'item9': {
+                'summary': 'Information about changes in independent auditors and any disagreements with them'
+            },
+            'item9a': {
+                'summary': 'Management\'s assessment of internal control effectiveness over financial reporting'
+            },
+            'item9b': {
+                'summary': 'Other significant information not reported elsewhere in the filing'
+            }
+        },
+        'part3': {
+            'summary': 'Information about company leadership, compensation structures, and corporate governance practices.',
+            'item10': {
+                'summary': 'Information about directors and executive officers, including their experience, qualifications, and corporate governance practices'
+            },
+            'item11': {
+                'summary': 'Detailed information about executive compensation, including salary, bonuses, stock awards, and compensation policies'
+            },
+            'item12': {
+                'summary': 'Information about beneficial ownership of securities by management and major shareholders, equity compensation plans'
+            },
+            'item13': {
+                'summary': 'Description of transactions with related parties and potential conflicts of interest'
+            },
+            'item14': {
+                'summary': 'Disclosure of fees paid for audit and non-audit services provided by independent accountants'
+            }
+        },
+        'part4': {
+            'summary': 'Supporting documentation and additional required disclosures.',
+            'item15': {
+                'summary': 'List of all exhibits, including material contracts, corporate documents, and supplementary financial information'
+            },
+            'item16': {
+                'summary': 'Optional summary of key information from the entire Form 10-K filing'
+            }
+        }
+    },
+    
+    'search_hints': {
+        'financial_metrics': ['item6', 'item7', 'item8'],
+        'risk_assessment': ['item1a', 'item1c', 'item7a'],
+        'business_overview': ['item1', 'item2'],
+        'leadership_info': ['item10', 'item11'],
+        'material_events': ['item3', 'item9', 'item13'],
+        'operational_data': ['item1', 'item7', 'item2']
+    }
+}

datamule/parser/document_parsing/n_port_p_parser.py

@@ -0,0 +1,70 @@
+from xml.etree import ElementTree as ET
+
+def element_to_dict(elem):
+    """Convert an XML element to dict preserving structure."""
+    result = {}
+    
+    # Add attributes directly to result
+    if elem.attrib:
+        result.update(elem.attrib)
+        
+    # Add text content if present and no children
+    if elem.text and elem.text.strip():
+        text = elem.text.strip()
+        if not len(elem):  # No children
+            return text
+        else:
+            result['text'] = text
+            
+    # Process children
+    for child in elem:
+        child_data = element_to_dict(child)
+        child_tag = child.tag.split('}')[-1]  # Remove namespace
+        
+        if child_tag in result:
+            # Convert to list if multiple elements
+            if not isinstance(result[child_tag], list):
+                result[child_tag] = [result[child_tag]]
+            result[child_tag].append(child_data)
+        else:
+            result[child_tag] = child_data
+            
+    return result
+
+def parse_nport_p(filepath):
+    """Parse NPORT XML file into metadata and document sections."""
+    # Parse XML
+    tree = ET.parse(filepath)
+    root = tree.getroot()
+    
+    # Remove namespaces for cleaner processing
+    for elem in root.iter():
+        if '}' in elem.tag:
+            elem.tag = elem.tag.split('}')[-1]
+    
+    # Convert entire document to dict
+    full_dict = element_to_dict(root)
+    
+    # Separate metadata and document content
+    result = {
+        'metadata': {},
+        'document': {}
+    }
+    
+    # Extract metadata sections
+    if 'headerData' in full_dict:
+        result['metadata']['headerData'] = full_dict['headerData']
+        
+    if 'formData' in full_dict and 'genInfo' in full_dict['formData']:
+        result['metadata']['genInfo'] = full_dict['formData']['genInfo']
+    
+    # Everything else goes to document
+    result['document'] = full_dict
+    
+    # Remove metadata sections from document to avoid duplication
+    if 'headerData' in result['document']:
+        del result['document']['headerData']
+    if 'formData' in result['document'] and 'genInfo' in result['document']['formData']:
+        del result['document']['formData']['genInfo']
+        
+    return result

datamule/parser/document_parsing/sec_parser.py

@@ -0,0 +1,73 @@
+import xml.etree.ElementTree as ET
+from .basic_8k_parser import parse_8k
+from .basic_10k_parser import parse_10k
+from .basic_10q_parser import parse_10q
+from .information_table_parser_13fhr import parse_13f_hr_information_table_xml
+from .insider_trading_parser import parse_form345
+from .form_d_parser import parse_form_d
+from .n_port_p_parser import parse_nport_p
+from .basic_13d_parser import parse_13d
+from .basic_13g_parser import parse_13g
+from .generalized_item_parser import generalized_parser
+from .mappings import *
+
+class Parser:
+
+    def __init__(self):
+        pass
+
+    def parse_filing(self, filename, filing_type):
+        if filing_type == 'INFORMATION TABLE':
+            return parse_13f_hr_information_table_xml(filename)
+        elif filing_type == '8-K':
+            return parse_8k(filename)
+        elif filing_type == '10-K':
+            return parse_10k(filename)
+        elif filing_type == '10-Q':
+            return parse_10q(filename)
+        elif filing_type in ['3', '4', '5']:
+            return parse_form345(filename)
+        elif filing_type == 'D':
+            return parse_form_d(filename)
+        elif filing_type == 'NPORT-P':
+            return parse_nport_p(filename)
+        elif filing_type == 'SC 13D':
+            return parse_13d(filename)
+        elif filing_type == 'SC 13G':
+            return parse_13g(filename)
+        else:
+            raise ValueError(f'Filing type {filing_type} not supported')
+
+
+    def parse_company_concepts(self, data):
+    
+        # get cik
+        cik = data['cik']
+        # get categories
+        categories = list(data['facts'].keys())
+
+        table_dict_list = []
+        for category in categories:
+            for fact in data['facts'][category]:
+                label = data['facts'][category][fact]['label']
+                description = data['facts'][category][fact]['description']
+                units = list(data['facts'][category][fact]['units'].keys())
+
+                for unit in units:
+                    table = data['facts'][category][fact]['units'][unit]
+
+                    # Find all unique keys across all rows
+                    all_keys = set()
+                    for row in table:
+                        all_keys.update(row.keys())
+
+                    # Ensure all rows have all keys
+                    for row in table:
+                        for key in all_keys:
+                            if key not in row:
+                                row[key] = None
+
+                    table_dict = {'cik':cik, 'category': category, 'fact': fact, 'label': label, 'description': description, 'unit': unit, 'table': table}
+                    table_dict_list.append(table_dict)
+
+        return table_dict_list

datamule/parser/document_parsing/sgml_parser.py

@@ -0,0 +1,94 @@
+import os
+import json
+import uu
+from io import BytesIO
+
+class SimpleSGMLParser:
+    def _extract_tag_content(self, line: str) -> tuple[str, str] | None:
+        if not (line.startswith('<') and '>' in line):
+            return None
+            
+        tag_end = line.index('>')
+        tag = line[1:tag_end]
+        
+        if tag.startswith('/'):
+            return None
+        
+        content = line[tag_end + 1:].strip()
+        return (tag, content)
+
+    def _write_document(self, content: str, document_info: dict, output_dir: str) -> None:
+        if not content:
+            return
+
+        output_path = os.path.join(output_dir, document_info.get('FILENAME', f"{document_info.get('SEQUENCE', 'unknown')}.txt"))
+        
+        first_line = content.partition('\n')[0].strip()
+        if first_line.startswith('begin '):
+            with BytesIO(content.encode()) as input_file:
+                uu.decode(input_file, output_path,quiet=True)
+        else:
+            with open(output_path, 'w', encoding='utf-8') as f:
+                f.write(content)
+
+    def parse_file(self, filepath: str, output_dir: str) -> None:
+        os.makedirs(output_dir, exist_ok=True)
+        
+        submission_data = {}
+        documents = []
+        current_document = {}
+        text_buffer = []
+        
+        in_document = False
+        in_text = False
+        in_submission = True
+        
+        with open(filepath, 'r', encoding='utf-8') as file:
+            for line in file:
+                stripped = line.strip()
+                
+                if stripped == '<DOCUMENT>':
+                    in_document = True
+                    in_submission = False
+                    current_document = {}
+                    
+                elif stripped == '</DOCUMENT>':
+                    documents.append(current_document)
+                    self._write_document(''.join(text_buffer), current_document, output_dir)
+                    text_buffer = []
+                    in_document = False
+                    
+                elif stripped == '<TEXT>':
+                    in_text = True
+                    text_buffer = []
+                    
+                elif stripped == '</TEXT>':
+                    in_text = False
+                    
+                elif in_text:
+                    if stripped not in ['<PDF>', '</PDF>']:
+                        text_buffer.append(line)
+                        
+                else:
+                    tag_content = self._extract_tag_content(stripped)
+                    if tag_content:
+                        key, value = tag_content
+                        if in_submission:
+                            submission_data[key] = value
+                        elif in_document:
+                            current_document[key] = value
+        
+        metadata = {
+            'submission': submission_data,
+            'documents': documents
+        }
+        
+        with open(os.path.join(output_dir, 'metadata.json'), 'w', encoding='utf-8') as f:
+            json.dump(metadata, f, indent=4)
+
+def parse_sgml_submission(filepath: str, output_dir: str | None = None) -> None:
+    if output_dir is None:
+        output_dir = os.path.splitext(filepath)[0] + '_output'
+        
+    parser = SimpleSGMLParser()
+    parser.parse_file(filepath, output_dir)