datamule 0.380__py3-none-any.whl → 1.0.0__py3-none-any.whl

datamule/parser/mappings.py

@@ -1,95 +0,0 @@
-# I will liekely move this file to a more appropriate location in the future
-
-mapping_dict_10k = {
-    'filing_summary': 'Annual report providing comprehensive overview of company business, financial performance, risks, and operations. Contains audited financial statements, business description, risk analysis, and detailed operational metrics.',
-    
-    'structure': {
-        'part1': {
-            'summary': 'Overview of company operations, risks, and material business information. Contains key business strategy, market position, competitive landscape, and significant challenges.',
-            'item1': {
-                'summary': 'Detailed description of business operations including primary products/services, markets served, distribution methods, competitive conditions, regulatory environment, and business segments'
-            },
-            'item1a': {
-                'summary': 'Comprehensive list and explanation of significant risks and uncertainties that could affect business performance, financial condition, and stock value'
-            },
-            'item1b': {
-                'summary': 'Disclosure of any unresolved comments or issues raised by SEC staff regarding company filings'
-            },
-            'item1c': {
-                'summary': 'Information about cybersecurity risks, incidents, risk management, governance, and strategy'
-            },
-            'item2': {
-                'summary': 'Description of principal physical properties, including manufacturing facilities, offices, warehouses, and other significant real estate'
-            },
-            'item3': {
-                'summary': 'Description of material pending legal proceedings, including potential impacts on business'
-            },
-            'item4': {
-                'summary': 'Disclosure of mine safety violations, citations, and orders received under the Mine Act'
-            }
-        },
-        'part2': {
-            'summary': 'Detailed financial performance analysis, including management insights, market risks, and complete audited financial statements.',
-            'item5': {
-                'summary': 'Information about company stock, including market data, price history, dividends, share repurchases, and securities offerings'
-            },
-            'item6': {
-                'summary': 'Selected historical financial data showing trends in financial condition and results over past 5 years'
-            },
-            'item7': {
-                'summary': 'Management\'s analysis of financial condition, operations results, liquidity, capital resources, and future outlook'
-            },
-            'item7a': {
-                'summary': 'Discussion of exposure to market risk including interest rates, foreign exchange, commodities, and hedging activities'
-            },
-            'item8': {
-                'summary': 'Audited financial statements, including balance sheets, income statements, cash flows, and comprehensive notes'
-            },
-            'item9': {
-                'summary': 'Information about changes in independent auditors and any disagreements with them'
-            },
-            'item9a': {
-                'summary': 'Management\'s assessment of internal control effectiveness over financial reporting'
-            },
-            'item9b': {
-                'summary': 'Other significant information not reported elsewhere in the filing'
-            }
-        },
-        'part3': {
-            'summary': 'Information about company leadership, compensation structures, and corporate governance practices.',
-            'item10': {
-                'summary': 'Information about directors and executive officers, including their experience, qualifications, and corporate governance practices'
-            },
-            'item11': {
-                'summary': 'Detailed information about executive compensation, including salary, bonuses, stock awards, and compensation policies'
-            },
-            'item12': {
-                'summary': 'Information about beneficial ownership of securities by management and major shareholders, equity compensation plans'
-            },
-            'item13': {
-                'summary': 'Description of transactions with related parties and potential conflicts of interest'
-            },
-            'item14': {
-                'summary': 'Disclosure of fees paid for audit and non-audit services provided by independent accountants'
-            }
-        },
-        'part4': {
-            'summary': 'Supporting documentation and additional required disclosures.',
-            'item15': {
-                'summary': 'List of all exhibits, including material contracts, corporate documents, and supplementary financial information'
-            },
-            'item16': {
-                'summary': 'Optional summary of key information from the entire Form 10-K filing'
-            }
-        }
-    },
-    
-    'search_hints': {
-        'financial_metrics': ['item6', 'item7', 'item8'],
-        'risk_assessment': ['item1a', 'item1c', 'item7a'],
-        'business_overview': ['item1', 'item2'],
-        'leadership_info': ['item10', 'item11'],
-        'material_events': ['item3', 'item9', 'item13'],
-        'operational_data': ['item1', 'item7', 'item2']
-    }
-}

datamule/parser/n_port_p_parser.py

@@ -1,70 +0,0 @@
-from xml.etree import ElementTree as ET
-
-def element_to_dict(elem):
-    """Convert an XML element to dict preserving structure."""
-    result = {}
-    
-    # Add attributes directly to result
-    if elem.attrib:
-        result.update(elem.attrib)
-        
-    # Add text content if present and no children
-    if elem.text and elem.text.strip():
-        text = elem.text.strip()
-        if not len(elem):  # No children
-            return text
-        else:
-            result['text'] = text
-            
-    # Process children
-    for child in elem:
-        child_data = element_to_dict(child)
-        child_tag = child.tag.split('}')[-1]  # Remove namespace
-        
-        if child_tag in result:
-            # Convert to list if multiple elements
-            if not isinstance(result[child_tag], list):
-                result[child_tag] = [result[child_tag]]
-            result[child_tag].append(child_data)
-        else:
-            result[child_tag] = child_data
-            
-    return result
-
-def parse_nport_p(filepath):
-    """Parse NPORT XML file into metadata and document sections."""
-    # Parse XML
-    tree = ET.parse(filepath)
-    root = tree.getroot()
-    
-    # Remove namespaces for cleaner processing
-    for elem in root.iter():
-        if '}' in elem.tag:
-            elem.tag = elem.tag.split('}')[-1]
-    
-    # Convert entire document to dict
-    full_dict = element_to_dict(root)
-    
-    # Separate metadata and document content
-    result = {
-        'metadata': {},
-        'document': {}
-    }
-    
-    # Extract metadata sections
-    if 'headerData' in full_dict:
-        result['metadata']['headerData'] = full_dict['headerData']
-        
-    if 'formData' in full_dict and 'genInfo' in full_dict['formData']:
-        result['metadata']['genInfo'] = full_dict['formData']['genInfo']
-    
-    # Everything else goes to document
-    result['document'] = full_dict
-    
-    # Remove metadata sections from document to avoid duplication
-    if 'headerData' in result['document']:
-        del result['document']['headerData']
-    if 'formData' in result['document'] and 'genInfo' in result['document']['formData']:
-        del result['document']['formData']['genInfo']
-        
-    return result

datamule/parser/sec_parser.py

@@ -1,79 +0,0 @@
-import xml.etree.ElementTree as ET
-from ..datamule_api import parse_textual_filing
-from .basic_8k_parser import parse_8k
-from .basic_10k_parser import parse_10k
-from .basic_10q_parser import parse_10q
-from .information_table_parser_13fhr import parse_13f_hr_information_table_xml
-from .insider_trading_parser import parse_form345
-from .form_d_parser import parse_form_d
-from .n_port_p_parser import parse_nport_p
-from .basic_13d_parser import parse_13d
-from .basic_13g_parser import parse_13g
-from .generalized_item_parser import generalized_parser
-from .mappings import *
-
-class Parser:
-
-    def __init__(self):
-        pass
-
-    def parse_filing(self, filename, filing_type):
-        # add handling for url vs file
-        # api will handle filing type detection
-        if filing_type == '13F-HR-INFORMATIONTABLE':
-            return parse_13f_hr_information_table_xml(filename)
-        # elif filing_type in ['10-K','10KSB','8-K']:
-        #     return generalized_parser(filename)
-        elif filing_type == '8-K':
-            return parse_8k(filename)
-        elif filing_type == '10-K':
-            return parse_10k(filename)
-        elif filing_type == '10-Q':
-            return parse_10q(filename)
-        elif filing_type in ['3', '4', '5']:
-            return parse_form345(filename)
-        elif filing_type == 'D':
-            return parse_form_d(filename)
-        elif filing_type == 'NPORT-P':
-            return parse_nport_p(filename)
-        elif filing_type == 'SC 13D':
-            return parse_13d(filename)
-        elif filing_type == 'SC 13G':
-            return parse_13g(filename)
-        else:
-            data = parse_textual_filing(url=filename, return_type='json')
-        return data 
-
-
-    def parse_company_concepts(self, data):
-    
-        # get cik
-        cik = data['cik']
-        # get categories
-        categories = list(data['facts'].keys())
-
-        table_dict_list = []
-        for category in categories:
-            for fact in data['facts'][category]:
-                label = data['facts'][category][fact]['label']
-                description = data['facts'][category][fact]['description']
-                units = list(data['facts'][category][fact]['units'].keys())
-
-                for unit in units:
-                    table = data['facts'][category][fact]['units'][unit]
-
-                    # Find all unique keys across all rows
-                    all_keys = set()
-                    for row in table:
-                        all_keys.update(row.keys())
-
-                    # Ensure all rows have all keys
-                    for row in table:
-                        for key in all_keys:
-                            if key not in row:
-                                row[key] = None
-
-                    table_dict = {'cik':cik, 'category': category, 'fact': fact, 'label': label, 'description': description, 'unit': unit, 'table': table}
-                    table_dict_list.append(table_dict)
-
-        return table_dict_list

datamule/parser/sgml_parser.py

@@ -1,180 +0,0 @@
-import shutil
-import os
-import json
-
-class UUEncodeError(Exception):
-    pass
-
-def UUdecoder(text):
-    text = text.split('\n')
-    result = bytearray()
-    
-    for line in text:
-        if not line or line in ['end', '`']: 
-            continue
-            
-        length = (ord(line[0]) - 32) & 63
-        chars = line[1:]
-        
-        for i in range(0, len(chars), 4):
-            group = chars[i:i+4]
-            if len(group) < 4:
-                break
-                
-            n = 0
-            for c in group:
-                n = n * 64 + ((ord(c) - 32) & 63)
-            
-            result.append((n >> 16) & 0xFF)
-            if length > 1:
-                result.append((n >> 8) & 0xFF)
-            if length > 2:
-                result.append(n & 0xFF)
-            
-            length -= 3
-            if length <= 0:
-                break
-                
-    return bytes(result)
-
-def read_line(line):
-    try:
-        key = line.split('<')[1].split('>')[0]
-        value = ''
-        if key.startswith('/'):
-            return None
-        if line.endswith('>'):
-            return {key: {}}
-        value = line.split('>', 1)[1].strip()
-        return {key: value}
-    except:
-        raise ValueError(f"Could not parse line: {line}")
-
-def parse_submission(filepath, output_dir):
-    shutil.rmtree(output_dir, ignore_errors=True)
-    os.makedirs(output_dir, exist_ok=True)
-
-    metadata = {
-        'submission': {},
-        'documents': []
-    }
-    
-    tag_stack = []
-    path_stack = [metadata['submission']]
-    current_document = None
-    text_content = []
-    last_key = None
-
-    in_text = False
-    is_uuencoded = False
-    collecting_binary = False
-    lines_since_text_start = 0  # Track lines since <TEXT>
-    
-    with open(filepath, 'r') as file:
-        for line in file:
-            line = line.rstrip('\n')  # Preserve spaces but remove newline
-            
-            if line == '<SUBMISSION>':
-                tag_stack.append('SUBMISSION')
-                
-            elif line == '</SUBMISSION>':
-                tag_stack.pop()
-                
-            elif line == '<DOCUMENT>':
-                current_document = {}
-                metadata['documents'].append(current_document)
-                path_stack = [current_document]
-                tag_stack.append('DOCUMENT')
-                last_key = None
-                
-            elif line == '</DOCUMENT>':
-                if current_document and text_content:
-                    if 'FILENAME' in current_document:
-                        output_path = os.path.join(output_dir, current_document['FILENAME'])
-                    elif 'SEQUENCE' in current_document:
-                        output_path = os.path.join(output_dir, f'{current_document["SEQUENCE"]}.txt')
-                    else:
-                        raise ValueError("Document does not have a FILENAME or SEQUENCE")
-                    
-                    if is_uuencoded:
-                        content = UUdecoder('\n'.join(text_content))
-                        with open(output_path, 'wb') as f:
-                            f.write(content)
-                    else:
-                        with open(output_path, 'w', encoding='utf-8') as f:
-                            f.write('\n'.join(text_content))
-                            
-                text_content = []
-                current_document = None
-                path_stack = [metadata['submission']]
-                tag_stack.pop()
-                last_key = None
-                is_uuencoded = False
-                collecting_binary = False
-                lines_since_text_start = 0
-                
-            elif line == '<TEXT>':
-                in_text = True
-                text_content = []
-                tag_stack.append('TEXT')
-                last_key = None
-                lines_since_text_start = 0
-                
-            elif line == '</TEXT>':
-                in_text = False
-                tag_stack.pop()
-                
-            elif in_text:
-                if collecting_binary:
-                    if line in ['end', '`']:
-                        collecting_binary = False
-                    else:
-                        text_content.append(line)
-                else:
-                    stripped_line = line.strip()
-                    if stripped_line == '<PDF>':
-                        lines_since_text_start = 0  # Reset counter after PDF marker
-                        continue
-                        
-                    if lines_since_text_start == 0 and not stripped_line:
-                        # Skip empty lines right after <TEXT> or <PDF>
-                        continue
-                        
-                    if lines_since_text_start == 0 and stripped_line.startswith('begin 644'):
-                        is_uuencoded = True
-                        collecting_binary = True
-                    else:
-                        if not is_uuencoded:
-                            text_content.append(line)
-                        lines_since_text_start += 1
-                
-            else:
-                if line.startswith('<'):
-                    parsed = read_line(line)
-                    if parsed is None:
-                        if tag_stack:
-                            tag_stack.pop()
-                            if len(path_stack) > 1:
-                                path_stack.pop()
-                            last_key = None
-                    else:
-                        key = list(parsed.keys())[0]
-                        value = parsed[key]
-                        
-                        if isinstance(value, dict):
-                            current_dict = path_stack[-1]
-                            current_dict[key] = {}
-                            path_stack.append(current_dict[key])
-                            tag_stack.append(key)
-                            last_key = None
-                        else:
-                            current_dict = path_stack[-1]
-                            current_dict[key] = value
-                            last_key = key
-                elif last_key:
-                    current_dict = path_stack[-1]
-                    current_dict[last_key] += ' ' + line.strip()
-    
-    metadata_path = os.path.join(output_dir, 'metadata.json')
-    with open(metadata_path, 'w') as f:
-        json.dump(metadata, f, indent=4)

datamule/sec_filing.py

@@ -1,126 +0,0 @@
-import json
-import csv
-from .parser.sec_parser import Parser
-from .helper import convert_to_dashed_accession
-
-class Filing:
-    def __init__(self, filename, filing_type):
-        self.filename = filename
-        self.parser = Parser()
-        self.data = None
-        self.filing_type = filing_type
-
-    def parse_filing(self):
-        self.data = self.parser.parse_filing(self.filename, self.filing_type)
-        return self.data
-    
-    def write_json(self, output_filename=None):
-        if not self.data:
-            raise ValueError("No data to write. Parse filing first.")
-            
-        if output_filename is None:
-            output_filename = f"{self.filename.rsplit('.', 1)[0]}.json"
-            
-        with open(output_filename, 'w') as f:
-            json.dump(self.data, f, indent=2)
-
-    def write_csv(self, output_filename=None, accession_number=None):
-        if self.data is None:
-            raise ValueError("No data available. Please call parse_filing() first.")
-
-        if output_filename is None:
-            output_filename = f"{self.filename.rsplit('.', 1)[0]}.csv"
-
-        with open(output_filename, 'w', newline='') as csvfile:
-            if not self.data:
-                return output_filename
-
-            has_document = any('document' in item for item in self.data)
-            
-            if has_document and 'document' in self.data:
-                writer = csv.DictWriter(csvfile, ['section', 'text'], quoting=csv.QUOTE_ALL)
-                writer.writeheader()
-                flattened = self._flatten_dict(self.data['document'])
-                for section, text in flattened.items():
-                    writer.writerow({'section': section, 'text': text})
-            else:
-                fieldnames = list(self.data[0].keys())
-                if accession_number:
-                    fieldnames.append('Accession Number')
-                writer = csv.DictWriter(csvfile, fieldnames, quoting=csv.QUOTE_ALL)
-                writer.writeheader()
-                for row in self.data:
-                    if accession_number:
-                        row['Accession Number'] = convert_to_dashed_accession(accession_number)
-                    writer.writerow(row)
-
-        return output_filename
-    
-    def _document_to_section_text(self, document_data, parent_key=''):
-        items = []
-        
-        if isinstance(document_data, dict):
-            for key, value in document_data.items():
-                # Build the section name
-                section = f"{parent_key}_{key}" if parent_key else key
-                
-                # If the value is a dict, recurse
-                if isinstance(value, dict):
-                    items.extend(self._document_to_section_text(value, section))
-                # If it's a list, handle each item
-                elif isinstance(value, list):
-                    for i, item in enumerate(value):
-                        if isinstance(item, dict):
-                            items.extend(self._document_to_section_text(item, f"{section}_{i+1}"))
-                        else:
-                            items.append({
-                                'section': f"{section}_{i+1}",
-                                'text': str(item)
-                            })
-                # Base case - add the item
-                else:
-                    items.append({
-                        'section': section,
-                        'text': str(value)
-                    })
-        
-        return items
-
-    def _flatten_dict(self, d, parent_key=''):
-        items = {}
-        
-        if isinstance(d, list):
-            return [self._flatten_dict(item) for item in d]
-                
-        for k, v in d.items():
-            new_key = f"{parent_key}_{k}" if parent_key else k
-            
-            if isinstance(v, dict):
-                items.update(self._flatten_dict(v, new_key))
-            else:
-                items[new_key] = str(v)
-                    
-        return items
-   
-    def __iter__(self):
-        if not self.data:
-            self.parse_filing()
-
-        if self.filing_type == '13F-HR-INFORMATIONTABLE':
-            return iter(self.data)
-        elif self.filing_type == '8-K':
-            return iter(self._document_to_section_text(self.data['document']))
-        elif self.filing_type == '10-K':
-            return iter(self._document_to_section_text(self.data['document']))
-        elif self.filing_type == '10-Q':
-            return iter(self._document_to_section_text(self.data['document']))
-        elif self.filing_type in ['3', '4', '5']:
-            return iter(self._flatten_dict(self.data['holdings']))
-        elif self.filing_type == 'D':
-            return iter(self._flatten_dict(self.data['document']['relatedPersonsList']['relatedPersonInfo']))
-        elif self.filing_type == 'NPORT-P':
-            return iter(self._flatten_dict(self.data['document']['formData']['invstOrSecs']['invstOrSec']))
-        elif self.filing_type == 'SC 13D':
-            return iter(self._document_to_section_text(self.data['document']))
-        elif self.filing_type == 'SC 13G':
-            return iter(self._document_to_section_text(self.data['document']))

datamule/sec_search.py

@@ -1,20 +0,0 @@
-import os
-import re
-
-def search(directory, keyword):
-    results = []
-    keyword = keyword.lower()  # Convert keyword to lowercase
-    
-    for root, dirs, files in os.walk(directory):
-        for file in files:
-            if file.endswith(('.xml', '.html', '.txt','.htm')):  # Add more extensions if needed
-                file_path = os.path.join(root, file)
-                try:
-                    with open(file_path, 'r', encoding='utf-8') as f:
-                        content = f.read().lower()  # Convert content to lowercase
-                        if keyword in content:  # Simple string search instead of regex
-                            results.append(file_path)
-                except Exception as e:
-                    print(f"Error reading {file_path}: {str(e)}")
-    
-    return results

datamule-0.380.dist-info/METADATA

@@ -1,110 +0,0 @@
-Metadata-Version: 2.1
-Name: datamule
-Version: 0.380
-Summary: Making it easier to use SEC filings.
-Home-page: https://github.com/john-friedman/datamule-python
-Author: John Friedman
-Description-Content-Type: text/markdown
-Requires-Dist: aiohttp
-Requires-Dist: aiolimiter
-Requires-Dist: tqdm
-Requires-Dist: requests
-Requires-Dist: nest-asyncio
-Requires-Dist: aiofiles
-Requires-Dist: polars
-Requires-Dist: setuptools
-Requires-Dist: selectolax
-Provides-Extra: all
-Requires-Dist: openai ; extra == 'all'
-Requires-Dist: flask ; extra == 'all'
-Requires-Dist: lxml ; extra == 'all'
-Requires-Dist: google-generativeai ; extra == 'all'
-Requires-Dist: pandas ; extra == 'all'
-Requires-Dist: psutil ; extra == 'all'
-Provides-Extra: dataset_builder
-Requires-Dist: pandas ; extra == 'dataset_builder'
-Requires-Dist: google-generativeai ; extra == 'dataset_builder'
-Requires-Dist: psutil ; extra == 'dataset_builder'
-Provides-Extra: filing_viewer
-Requires-Dist: lxml ; extra == 'filing_viewer'
-Provides-Extra: mulebot
-Requires-Dist: openai ; extra == 'mulebot'
-Provides-Extra: mulebot_server
-Requires-Dist: flask ; extra == 'mulebot_server'
-
-# datamule
-
-[![Downloads](https://static.pepy.tech/badge/datamule)](https://pepy.tech/project/datamule)
-[![Hits](https://hits.seeyoufarm.com/api/count/incr/badge.svg?url=https%3A%2F%2Fgithub.com%2Fjohn-friedman%2Fdatamule-python&count_bg=%2379C83D&title_bg=%23555555&icon=&icon_color=%23E7E7E7&title=hits&edge_flat=false)](https://hits.seeyoufarm.com)
-![GitHub](https://img.shields.io/github/stars/john-friedman/datamule-python)
-
-A Python package for working with SEC filings at scale. Includes [Mulebot](https://chat.datamule.xyz/), an open-source chatbot for SEC data with no storage requirements. 📚 [Full Documentation](https://john-friedman.github.io/datamule-python/) | 🌐 [Website](https://datamule.xyz/)
-
-Articles:
-- [Article: Creating Structured Datasets from SEC filings](https://medium.com/@jgfriedman99/how-to-create-alternative-datasets-using-datamule-d3a0192da8f6)
-- [Article: Deploy a Financial Chatbot in 5 Minutes](https://medium.com/@jgfriedman99/how-to-deploy-a-financial-chatbot-in-5-minutes-ef5eec973d4c)
-
-## Key Features
-
-- 📥 [Download SEC filings quickly and efficiently](https://john-friedman.github.io/datamule-python/usage/downloader.html#)
-- 🔍 [Monitor EDGAR for new filings in real-time](https://john-friedman.github.io/datamule-python/usage/downloader.html#monitoring-new-filings)
-- 📊 [Parse filings at scale](https://john-friedman.github.io/datamule-python/usage/parsing.html#filing-parser)
-- 💾 [Access comprehensive datasets (10-Ks, SIC codes, etc.)](https://john-friedman.github.io/datamule-python/datasets.html)
-- 🤖 [Interact with SEC data using MuleBot](https://john-friedman.github.io/datamule-python/usage/mulebot.html)
-
-## Quick Start
-
-```bash
-# Basic installation
-pip install datamule
-
-# Install with all features
-pip install datamule[all]
-```
-
-```python
-import datamule as dm
-
-# Download filings
-downloader = dm.Downloader()
-downloader.download(form='10-K', ticker='AAPL')
-
-# Download filing attachments such as information tables
-downloader.download(form='13F-HR',file_types=['INFORMATION TABLE'],date=('2024-09-14','2024-09-16'))
-
-# Download every 10Q from 2023. Should take 2 minutes
-downloader.download_dataset(dataset='10q_2023')
-```
-
-## Available Extras
-
-- `filing_viewer`: Filing viewer module
-- `mulebot`: SEC data interaction chatbot
-- `mulebot_server`: Flask server for MuleBot
-- `all`: All available features
-
-## Resources
-
-- 📊 [SEC Filing Glossary](https://datamule.xyz/sec_glossary)
-- 📈 [XBRL Fact Glossary](https://datamule.xyz/xbrl_fact_glossary)
-- 🤖 [Try MuleBot](https://chat.datamule.xyz/)
-
-## Datasets
-
-Access comprehensive SEC datasets including:
-- Historical FTD data (since 2004)
-- 10-K and 10-Q filings (since 2001)
-- 13F-HR Information Tables (since 2013)
-- MD&A collection (100,000+ since 2001, requires free API key)
-
-## Contributing
-
-Contributions are welcome! Please feel free to submit a Pull Request.
-
-## License
-
-[MIT License](LICENSE)
-
----
-
-For detailed usage examples, API reference, and advanced features, please visit our [documentation](https://john-friedman.github.io/datamule-python/).