datamule 0.416__cp310-cp310-win_amd64.whl → 0.418__cp310-cp310-win_amd64.whl

datamule/parser/document_parsing/form_d_parser.py

@@ -0,0 +1,70 @@
+from xml.etree import ElementTree as ET
+
+def element_to_dict(elem):
+    """Convert an XML element to dict preserving structure."""
+    result = {}
+    
+    # Add attributes directly to result
+    if elem.attrib:
+        result.update(elem.attrib)
+        
+    # Add text content if present and no children
+    if elem.text and elem.text.strip():
+        text = elem.text.strip()
+        if not len(elem):  # No children
+            return text
+        else:
+            result['text'] = text
+            
+    # Process children
+    for child in elem:
+        child_data = element_to_dict(child)
+        child_tag = child.tag.split('}')[-1]  # Remove namespace
+        
+        if child_tag in result:
+            # Convert to list if multiple elements
+            if not isinstance(result[child_tag], list):
+                result[child_tag] = [result[child_tag]]
+            result[child_tag].append(child_data)
+        else:
+            result[child_tag] = child_data
+            
+    return result
+
+def parse_form_d(filepath):
+    """Parse Form D XML file into metadata and document sections."""
+    # Parse XML
+    tree = ET.parse(filepath)
+    root = tree.getroot()
+    
+    # Remove namespaces for cleaner processing
+    for elem in root.iter():
+        if '}' in elem.tag:
+            elem.tag = elem.tag.split('}')[-1]
+    
+    # Convert entire document to dict
+    full_dict = element_to_dict(root)
+    
+    # Separate metadata and document content
+    result = {
+        'metadata': {},
+        'document': {}
+    }
+    
+    # Extract metadata
+    metadata_fields = {
+        'schemaVersion',
+        'submissionType',
+        'testOrLive',
+        'primaryIssuer'  # Including all issuer information in metadata
+    }
+    
+    for field in metadata_fields:
+        if field in full_dict:
+            result['metadata'][field] = full_dict[field]
+            del full_dict[field]  # Remove from full_dict to avoid duplication
+    
+    # Everything else goes to document
+    result['document'] = full_dict
+    
+    return result

datamule/parser/document_parsing/generalized_item_parser.py

@@ -0,0 +1,78 @@
+# Parses e.g. 10-K, 10-Q,..... any form with items and/or parts
+from .helper import load_file_content, clean_title, clean_text
+from pathlib import Path
+import re
+
+# OK figured out general pattern
+# find toc
+# figure out mapping. we do need it
+# just do mapping tonight
+
+pattern = re.compile(r'^\s*(?:item\s+\d+(?:\.\d+)?(?:[a-z])?|signature(?:\.?s)?)\s*', re.I | re.M)
+
+def find_anchors(content):
+   anchors = []
+   prev_title = None
+   
+   for part_match in pattern.finditer(content):
+       title = clean_title(part_match.group())
+       # Skip duplicates, e.g. "item 1" and "item1 continued"
+       if prev_title == title:
+           continue
+       prev_title = title
+       anchors.append((title, part_match.start()))
+   
+   return anchors
+
+# I think this works, but I haven't tested it extensively.
+def map_sections(content, anchors):
+    positions = anchors + [('end', len(content))]
+    
+    result = {}
+    for i, (title, start) in enumerate(positions[:-1]):
+        _, next_start = positions[i + 1]
+        section_text = content[start:next_start].strip()
+        result[title.lower()] = clean_text(section_text)
+    
+    def sort_key(x):
+        match = re.search(r'item\s+(\d+)(?:[\.a-z])?', x[0], re.I)
+        if not match:
+            return float('inf')
+        num = match.group(0).lower()
+        # This will sort 1, 1a, 1b, 2, 2a etc
+        return float(re.findall(r'\d+', num)[0]) + (ord(num[-1]) - ord('a') + 1) / 100 if num[-1].isalpha() else float(re.findall(r'\d+', num)[0])
+        
+    return dict(sorted(result.items(), key=sort_key))
+
+# def find_content_start(anchors):
+#     def find_first_non_repeating(seq):
+#         for i in range(len(seq)):
+#             remaining = seq[i:]
+#             # Get same length subsequence from the next position
+#             next_seq = seq[i + 1:i + 1 + len(remaining)]
+#             if remaining != next_seq and len(next_seq) > 0:
+#                 return i
+#         return 0  # Default to start if no pattern found
+    
+#     return find_first_non_repeating([title for title, _ in anchors])
+
+def generalized_parser(filename):
+    # load content
+    content = load_file_content(filename)
+
+    # find anchors
+    anchors = find_anchors(content)
+
+    # Skip tables of contents. Not implemented yet, since we overwrite the keys anyway.
+    # content_start = find_content_start(anchors)
+    # print(content_start)
+
+    result = {}
+    # assign metadata
+    result["metadata"] = {"document_name": Path(filename).stem}
+
+    # extract sections, assign text based on mapping_dict
+    result['document'] = map_sections(content, anchors)
+
+    return result
+

datamule/parser/document_parsing/helper.py

@@ -0,0 +1,75 @@
+from selectolax.parser import HTMLParser
+import re
+
+# This will be modified in the future to remove SEC specific code such as <PAGE> tags
+def load_text_content(filename):
+    with open(filename) as f:
+        return f.read().translate(str.maketrans({
+            '\xa0': ' ', '\u2003': ' ',
+            '\u2018': "'", '\u2019': "'",
+            '\u201c': '"', '\u201d': '"'
+        }))
+    
+def load_html_content(filename):
+    parser = HTMLParser(open(filename).read())
+    
+    # Remove hidden elements first
+    hidden_nodes = parser.css('[style*="display: none"], [style*="display:none"], .hidden, .hide, .d-none')
+    for node in hidden_nodes:
+        node.decompose()
+    
+    blocks = {'p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'article', 'section', 'li', 'td'}
+    lines = []
+    current_line = []
+    
+    def flush_line():
+        if current_line:
+            lines.append(' '.join(current_line))
+            current_line.clear()
+    
+    for node in parser.root.traverse(include_text=True):
+        if node.tag in ('script', 'style', 'css'):
+            continue
+            
+        if node.tag in blocks:
+            flush_line()
+            lines.append('')
+            
+        if node.text_content:
+            text = node.text_content.strip()
+            if text:
+                if node.tag in blocks:
+                    flush_line()
+                    lines.append(text)
+                    lines.append('')
+                else:
+                    current_line.append(text)
+    
+    flush_line()
+    
+    text = '\n'.join(lines)
+    while '\n\n\n' in text:
+        text = text.replace('\n\n\n', '\n\n')
+    
+    return text.translate(str.maketrans({
+        '\xa0': ' ', '\u2003': ' ',
+        '\u2018': "'", '\u2019': "'",
+        '\u201c': '"', '\u201d': '"'
+    }))
+def load_file_content(filename):
+    if filename.suffix =='.txt':
+        return load_text_content(filename)
+    elif filename.suffix in ['.html','.htm']:
+        return load_html_content(filename)
+    else:
+        raise ValueError(f"Unsupported file type: {filename}")
+
+def clean_title(title: str) -> str:
+    """Clean up section title by removing newlines, periods, and all whitespace, converting to lowercase."""
+    return ''.join(title.replace('\n', '').replace('.', '').split()).lower()
+
+# This is a bit hacky, removes PART IV, PART V etc from the end of the text
+# we do this to avoid having to map for general cases
+def clean_text(text):
+    text = text.strip()
+    return re.sub(r'\s*PART\s+[IVX]+\s*$', '', text, flags=re.I)

datamule/parser/document_parsing/information_table_parser_13fhr.py

@@ -0,0 +1,41 @@
+from xml.etree import ElementTree as ET
+
+def parse_13f_hr_information_table_xml(xml_file):
+    # Parse the XML file
+    tree = ET.parse(xml_file)
+    root = tree.getroot()
+
+    data = []
+    
+    # Iterate through each infoTable
+    for info_table in root.findall('.//{*}infoTable'):
+        row = {
+            'NAMEOFISSUER': info_table.findtext('.//{*}nameOfIssuer') or '',
+            'TITLEOFCLASS': info_table.findtext('.//{*}titleOfClass') or '',
+            'CUSIP': info_table.findtext('.//{*}cusip') or '',
+            'FIGI': info_table.findtext('.//{*}figi') or '',
+            'VALUE': info_table.findtext('.//{*}value') or '',
+            'SSHPRNAMT': '',
+            'SSHPRNAMTTYPE': '',
+            'PUTCALL': info_table.findtext('.//{*}putCall') or '',
+            'INVESTMENTDISCRETION': info_table.findtext('.//{*}investmentDiscretion') or '',
+            'OTHERMANAGER': info_table.findtext('.//{*}otherManager') or '',
+            'VOTING_AUTH_SOLE': '',
+            'VOTING_AUTH_SHARED': '',
+            'VOTING_AUTH_NONE': ''
+        }
+        
+        shrs_or_prn_amt = info_table.find('.//{*}shrsOrPrnAmt')
+        if shrs_or_prn_amt is not None:
+            row['SSHPRNAMT'] = shrs_or_prn_amt.findtext('.//{*}sshPrnamt') or ''
+            row['SSHPRNAMTTYPE'] = shrs_or_prn_amt.findtext('.//{*}sshPrnamtType') or ''
+        
+        voting_authority = info_table.find('.//{*}votingAuthority')
+        if voting_authority is not None:
+            row['VOTING_AUTH_SOLE'] = voting_authority.findtext('.//{*}Sole') or ''
+            row['VOTING_AUTH_SHARED'] = voting_authority.findtext('.//{*}Shared') or ''
+            row['VOTING_AUTH_NONE'] = voting_authority.findtext('.//{*}None') or ''
+        
+        data.append(row)
+
+    return data

datamule/parser/document_parsing/insider_trading_parser.py

@@ -0,0 +1,158 @@
+from xml.etree import ElementTree as ET
+from typing import Dict, Any, Optional
+
+def get_footnotes(doc) -> dict:
+    """Extract footnotes into a lookup dictionary."""
+    return {
+        f.attrib.get('id', ''): f.text.strip() 
+        for f in doc.findall('.//footnotes/footnote')
+    }
+
+def get_value_and_footnote(elem, footnotes: dict) -> dict:
+    """Get value and footnote for a field."""
+    result = {'value': ''}
+    
+    if elem is None:
+        return result
+        
+    # Get value
+    value_elem = elem.find('.//value')
+    if value_elem is not None:
+        result['value'] = value_elem.text or ''
+    
+    # Get footnote if exists
+    footnote_elem = elem.find('.//footnoteId')
+    if footnote_elem is not None:
+        footnote_id = footnote_elem.attrib.get('id', '')
+        if footnote_id in footnotes:
+            result['footnote'] = footnotes[footnote_id]
+    
+    return result
+
+def parse_form345(filepath) -> Dict[str, Any]:
+    """Parse SEC Form XML with enhanced data extraction."""
+    doc = ET.parse(filepath).getroot()
+    if doc is None:
+        return {"error": "No ownershipDocument found"}
+        
+    footnotes = get_footnotes(doc)
+    
+    result = {
+        'metadata': {
+            'schemaVersion': doc.findtext('schemaVersion', ''),
+            'documentType': doc.findtext('documentType', ''),
+            'periodOfReport': doc.findtext('periodOfReport', ''),
+            'dateOfOriginalSubmission': doc.findtext('dateOfOriginalSubmission', ''),
+            'form3HoldingsReported': doc.findtext('form3HoldingsReported', ''),
+            'form4TransactionsReported': doc.findtext('form4TransactionsReported', ''),
+            'issuer': {
+                'cik': doc.findtext('.//issuerCik', ''),
+                'name': doc.findtext('.//issuerName', ''),
+                'tradingSymbol': doc.findtext('.//issuerTradingSymbol', '')
+            },
+            'reportingOwner': {
+                'cik': doc.findtext('.//rptOwnerCik', ''),
+                'name': doc.findtext('.//rptOwnerName', ''),
+                'address': {
+                    'street1': doc.findtext('.//rptOwnerStreet1', ''),
+                    'street2': doc.findtext('.//rptOwnerStreet2', ''),
+                    'city': doc.findtext('.//rptOwnerCity', ''),
+                    'state': doc.findtext('.//rptOwnerState', ''),
+                    'zip': doc.findtext('.//rptOwnerZipCode', ''),
+                    'stateDescription': doc.findtext('.//rptOwnerStateDescription', '')
+                },
+                'relationship': {
+                    'isDirector': doc.findtext('.//isDirector', ''),
+                    'isOfficer': doc.findtext('.//isOfficer', ''),
+                    'isTenPercentOwner': doc.findtext('.//isTenPercentOwner', ''),
+                    'isOther': doc.findtext('.//isOther', ''),
+                    'officerTitle': doc.findtext('.//officerTitle', '')
+                }
+            },
+            'signature': {
+                'name': doc.findtext('.//signatureName', ''),
+                'date': doc.findtext('.//signatureDate', '')
+            }
+        },
+        'holdings': []
+    }
+
+    # Parse non-derivative holdings/transactions
+    for entry in doc.findall('.//nonDerivativeTable/*'):
+        holding = {
+            'type': 'non-derivative',
+            'securityTitle': get_value_and_footnote(entry.find('.//securityTitle'), footnotes),
+            'postTransactionAmounts': {
+                'sharesOwned': get_value_and_footnote(entry.find('.//sharesOwnedFollowingTransaction'), footnotes)
+            },
+            'ownershipNature': {
+                'directOrIndirect': entry.findtext('.//directOrIndirectOwnership/value', '')
+            }
+        }
+
+        # Add transaction fields if present
+        if 'Transaction' in entry.tag:
+            transactionCoding = {
+                'formType': entry.findtext('.//transactionFormType', ''),
+                'code': entry.findtext('.//transactionCode', ''),
+                'equitySwapInvolved': entry.findtext('.//equitySwapInvolved', '')
+            }
+            
+            transactionAmounts = {
+                'shares': get_value_and_footnote(entry.find('.//transactionShares'), footnotes),
+                'pricePerShare': get_value_and_footnote(entry.find('.//transactionPricePerShare'), footnotes),
+                'acquiredDisposedCode': get_value_and_footnote(entry.find('.//transactionAcquiredDisposedCode'), footnotes)
+            }
+            
+            holding.update({
+                'transactionDate': get_value_and_footnote(entry.find('.//transactionDate'), footnotes),
+                'transactionCoding': transactionCoding,
+                'transactionAmounts': transactionAmounts
+            })
+            
+        result['holdings'].append(holding)
+
+    # Parse derivative holdings/transactions
+    for entry in doc.findall('.//derivativeTable/*'):
+        holding = {
+            'type': 'derivative',
+            'securityTitle': get_value_and_footnote(entry.find('.//securityTitle'), footnotes),
+            'conversionOrExercisePrice': get_value_and_footnote(entry.find('.//conversionOrExercisePrice'), footnotes),
+            'exerciseDate': get_value_and_footnote(entry.find('.//exerciseDate'), footnotes),
+            'expirationDate': get_value_and_footnote(entry.find('.//expirationDate'), footnotes),
+            'underlyingSecurity': {
+                'title': get_value_and_footnote(entry.find('.//underlyingSecurityTitle'), footnotes),
+                'shares': get_value_and_footnote(entry.find('.//underlyingSecurityShares'), footnotes)
+            },
+            'postTransactionAmounts': {
+                'sharesOwned': get_value_and_footnote(entry.find('.//sharesOwnedFollowingTransaction'), footnotes)
+            },
+            'ownershipNature': {
+                'directOrIndirect': entry.findtext('.//directOrIndirectOwnership/value', ''),
+                'nature': entry.findtext('.//natureOfOwnership/value', '')
+            }
+        }
+
+        # Add transaction-specific fields
+        if 'Transaction' in entry.tag:
+            transactionCoding = {
+                'formType': entry.findtext('.//transactionFormType', ''),
+                'code': entry.findtext('.//transactionCode', ''),
+                'equitySwapInvolved': entry.findtext('.//equitySwapInvolved', '')
+            }
+            
+            transactionAmounts = {
+                'shares': get_value_and_footnote(entry.find('.//transactionShares'), footnotes),
+                'pricePerShare': get_value_and_footnote(entry.find('.//transactionPricePerShare'), footnotes),
+                'acquiredDisposedCode': get_value_and_footnote(entry.find('.//transactionAcquiredDisposedCode'), footnotes)
+            }
+            
+            holding.update({
+                'transactionDate': get_value_and_footnote(entry.find('.//transactionDate'), footnotes),
+                'transactionCoding': transactionCoding,
+                'transactionAmounts': transactionAmounts
+            })
+            
+        result['holdings'].append(holding)
+
+    return result

datamule/parser/document_parsing/mappings.py

@@ -0,0 +1,95 @@
+# I will liekely move this file to a more appropriate location in the future
+
+mapping_dict_10k = {
+    'filing_summary': 'Annual report providing comprehensive overview of company business, financial performance, risks, and operations. Contains audited financial statements, business description, risk analysis, and detailed operational metrics.',
+    
+    'structure': {
+        'part1': {
+            'summary': 'Overview of company operations, risks, and material business information. Contains key business strategy, market position, competitive landscape, and significant challenges.',
+            'item1': {
+                'summary': 'Detailed description of business operations including primary products/services, markets served, distribution methods, competitive conditions, regulatory environment, and business segments'
+            },
+            'item1a': {
+                'summary': 'Comprehensive list and explanation of significant risks and uncertainties that could affect business performance, financial condition, and stock value'
+            },
+            'item1b': {
+                'summary': 'Disclosure of any unresolved comments or issues raised by SEC staff regarding company filings'
+            },
+            'item1c': {
+                'summary': 'Information about cybersecurity risks, incidents, risk management, governance, and strategy'
+            },
+            'item2': {
+                'summary': 'Description of principal physical properties, including manufacturing facilities, offices, warehouses, and other significant real estate'
+            },
+            'item3': {
+                'summary': 'Description of material pending legal proceedings, including potential impacts on business'
+            },
+            'item4': {
+                'summary': 'Disclosure of mine safety violations, citations, and orders received under the Mine Act'
+            }
+        },
+        'part2': {
+            'summary': 'Detailed financial performance analysis, including management insights, market risks, and complete audited financial statements.',
+            'item5': {
+                'summary': 'Information about company stock, including market data, price history, dividends, share repurchases, and securities offerings'
+            },
+            'item6': {
+                'summary': 'Selected historical financial data showing trends in financial condition and results over past 5 years'
+            },
+            'item7': {
+                'summary': 'Management\'s analysis of financial condition, operations results, liquidity, capital resources, and future outlook'
+            },
+            'item7a': {
+                'summary': 'Discussion of exposure to market risk including interest rates, foreign exchange, commodities, and hedging activities'
+            },
+            'item8': {
+                'summary': 'Audited financial statements, including balance sheets, income statements, cash flows, and comprehensive notes'
+            },
+            'item9': {
+                'summary': 'Information about changes in independent auditors and any disagreements with them'
+            },
+            'item9a': {
+                'summary': 'Management\'s assessment of internal control effectiveness over financial reporting'
+            },
+            'item9b': {
+                'summary': 'Other significant information not reported elsewhere in the filing'
+            }
+        },
+        'part3': {
+            'summary': 'Information about company leadership, compensation structures, and corporate governance practices.',
+            'item10': {
+                'summary': 'Information about directors and executive officers, including their experience, qualifications, and corporate governance practices'
+            },
+            'item11': {
+                'summary': 'Detailed information about executive compensation, including salary, bonuses, stock awards, and compensation policies'
+            },
+            'item12': {
+                'summary': 'Information about beneficial ownership of securities by management and major shareholders, equity compensation plans'
+            },
+            'item13': {
+                'summary': 'Description of transactions with related parties and potential conflicts of interest'
+            },
+            'item14': {
+                'summary': 'Disclosure of fees paid for audit and non-audit services provided by independent accountants'
+            }
+        },
+        'part4': {
+            'summary': 'Supporting documentation and additional required disclosures.',
+            'item15': {
+                'summary': 'List of all exhibits, including material contracts, corporate documents, and supplementary financial information'
+            },
+            'item16': {
+                'summary': 'Optional summary of key information from the entire Form 10-K filing'
+            }
+        }
+    },
+    
+    'search_hints': {
+        'financial_metrics': ['item6', 'item7', 'item8'],
+        'risk_assessment': ['item1a', 'item1c', 'item7a'],
+        'business_overview': ['item1', 'item2'],
+        'leadership_info': ['item10', 'item11'],
+        'material_events': ['item3', 'item9', 'item13'],
+        'operational_data': ['item1', 'item7', 'item2']
+    }
+}

datamule/parser/document_parsing/n_port_p_parser.py

@@ -0,0 +1,70 @@
+from xml.etree import ElementTree as ET
+
+def element_to_dict(elem):
+    """Convert an XML element to dict preserving structure."""
+    result = {}
+    
+    # Add attributes directly to result
+    if elem.attrib:
+        result.update(elem.attrib)
+        
+    # Add text content if present and no children
+    if elem.text and elem.text.strip():
+        text = elem.text.strip()
+        if not len(elem):  # No children
+            return text
+        else:
+            result['text'] = text
+            
+    # Process children
+    for child in elem:
+        child_data = element_to_dict(child)
+        child_tag = child.tag.split('}')[-1]  # Remove namespace
+        
+        if child_tag in result:
+            # Convert to list if multiple elements
+            if not isinstance(result[child_tag], list):
+                result[child_tag] = [result[child_tag]]
+            result[child_tag].append(child_data)
+        else:
+            result[child_tag] = child_data
+            
+    return result
+
+def parse_nport_p(filepath):
+    """Parse NPORT XML file into metadata and document sections."""
+    # Parse XML
+    tree = ET.parse(filepath)
+    root = tree.getroot()
+    
+    # Remove namespaces for cleaner processing
+    for elem in root.iter():
+        if '}' in elem.tag:
+            elem.tag = elem.tag.split('}')[-1]
+    
+    # Convert entire document to dict
+    full_dict = element_to_dict(root)
+    
+    # Separate metadata and document content
+    result = {
+        'metadata': {},
+        'document': {}
+    }
+    
+    # Extract metadata sections
+    if 'headerData' in full_dict:
+        result['metadata']['headerData'] = full_dict['headerData']
+        
+    if 'formData' in full_dict and 'genInfo' in full_dict['formData']:
+        result['metadata']['genInfo'] = full_dict['formData']['genInfo']
+    
+    # Everything else goes to document
+    result['document'] = full_dict
+    
+    # Remove metadata sections from document to avoid duplication
+    if 'headerData' in result['document']:
+        del result['document']['headerData']
+    if 'formData' in result['document'] and 'genInfo' in result['document']['formData']:
+        del result['document']['formData']['genInfo']
+        
+    return result

datamule/parser/document_parsing/sec_parser.py

@@ -0,0 +1,73 @@
+import xml.etree.ElementTree as ET
+from .basic_8k_parser import parse_8k
+from .basic_10k_parser import parse_10k
+from .basic_10q_parser import parse_10q
+from .information_table_parser_13fhr import parse_13f_hr_information_table_xml
+from .insider_trading_parser import parse_form345
+from .form_d_parser import parse_form_d
+from .n_port_p_parser import parse_nport_p
+from .basic_13d_parser import parse_13d
+from .basic_13g_parser import parse_13g
+from .generalized_item_parser import generalized_parser
+from .mappings import *
+
+class Parser:
+
+    def __init__(self):
+        pass
+
+    def parse_filing(self, filename, filing_type):
+        if filing_type == 'INFORMATION TABLE':
+            return parse_13f_hr_information_table_xml(filename)
+        elif filing_type == '8-K':
+            return parse_8k(filename)
+        elif filing_type == '10-K':
+            return parse_10k(filename)
+        elif filing_type == '10-Q':
+            return parse_10q(filename)
+        elif filing_type in ['3', '4', '5']:
+            return parse_form345(filename)
+        elif filing_type == 'D':
+            return parse_form_d(filename)
+        elif filing_type == 'NPORT-P':
+            return parse_nport_p(filename)
+        elif filing_type == 'SC 13D':
+            return parse_13d(filename)
+        elif filing_type == 'SC 13G':
+            return parse_13g(filename)
+        else:
+            raise ValueError(f'Filing type {filing_type} not supported')
+
+
+    def parse_company_concepts(self, data):
+    
+        # get cik
+        cik = data['cik']
+        # get categories
+        categories = list(data['facts'].keys())
+
+        table_dict_list = []
+        for category in categories:
+            for fact in data['facts'][category]:
+                label = data['facts'][category][fact]['label']
+                description = data['facts'][category][fact]['description']
+                units = list(data['facts'][category][fact]['units'].keys())
+
+                for unit in units:
+                    table = data['facts'][category][fact]['units'][unit]
+
+                    # Find all unique keys across all rows
+                    all_keys = set()
+                    for row in table:
+                        all_keys.update(row.keys())
+
+                    # Ensure all rows have all keys
+                    for row in table:
+                        for key in all_keys:
+                            if key not in row:
+                                row[key] = None
+
+                    table_dict = {'cik':cik, 'category': category, 'fact': fact, 'label': label, 'description': description, 'unit': unit, 'table': table}
+                    table_dict_list.append(table_dict)
+
+        return table_dict_list