datamule 0.428__tar.gz → 1.0.3__tar.gz

{datamule-0.428 → datamule-1.0.3}/PKG-INFO

@@ -1,10 +1,9 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 0.428
+Version: 1.0.3
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
 Provides-Extra: mulebot
 Provides-Extra: mulebot_server
-Provides-Extra: dataset_builder
 Provides-Extra: all

datamule-1.0.3/datamule/__init__.py

@@ -0,0 +1,47 @@
+from .downloader.downloader import Downloader
+from .downloader.premiumdownloader import PremiumDownloader
+from .monitor import Monitor
+from .packageupdater import PackageUpdater
+from .submission import Submission
+from .portfolio import Portfolio
+from .document import Document
+from secsgml import parse_sgml_submission
+from .helper import load_package_csv, load_package_dataset
+from .config import Config
+
+
+# Keep the notebook environment setup
+def _is_notebook_env():
+    """Check if the code is running in a Jupyter or Colab environment."""
+    try:
+        shell = get_ipython().__class__.__name__
+        return shell in ('ZMQInteractiveShell', 'Shell', 'Google.Colab')
+    except NameError:
+        return False
+
+from functools import lru_cache
+
+@lru_cache(maxsize=1)
+def _setup_notebook_env():
+    """Setup Jupyter/Colab-specific configurations if needed."""
+    if _is_notebook_env():
+        import nest_asyncio
+        nest_asyncio.apply()
+
+# Set up notebook environment
+_setup_notebook_env()
+
+__all__ = [
+    'Downloader',
+    'PremiumDownloader',
+    'load_package_csv',
+    'load_package_dataset',
+    'Filing',
+    'Portfolio',
+    'Monitor',
+    'PackageUpdater',
+    'Submission',
+    'Document',
+    'parse_sgml_submission',
+    'Config'
+]

datamule-1.0.3/datamule/book/book.py

@@ -0,0 +1,34 @@
+# Streams data rather than downloading it. 
+# additional functionality such as query by xbrl, and other db
+# also this is basically our experimental rework of portfolio w/o disturbing existing users
+# this is highly experimental and may not work as expected
+# only for datamule source
+# likely new bottleneck will be local parsing() - will be bypassed in future when we have parsed archive
+# wow parsed archive is going to be crazy fast - like every 10k in 1 minute.
+
+# example queries filter by sic = 7372, xbrl query = dei:operatingprofit > 0 in date range 2018-2019
+
+# hmm do we go for sql esq or not.
+# I think we do.
+# i think we remove cik, ticker, sic, etc and just have a query object
+# should be sql esq so users can use it easily w/o learnign new syntax
+
+# WHERE submission_type = '10-K'
+# AND us-gaap:ResearchAndDevelopmentExpense > 0 
+# AND dei:debt_to_equity < 2
+# AND filing_date BETWEEN '2023-01-01' AND '2023-12-31'
+# AND CIK in (123, 456, 789)
+# AND SIC in (123, 456, 789)
+# AND ticker in ('AAPL', 'GOOGL', 'AMZN')
+# AND document_type = 'EX-99.1' # to select attachments
+
+from .eftsquery import EFTSQuery
+
+
+class Book():
+    def process_submissions(self,cik,ticker,sic,submission_type,document_type,date,
+                            xbrl_query={},
+                            metadata_callback=None,
+                            document_callback=None,):
+        # grabs data and processes it
+        pass 

datamule-1.0.3/datamule/book/eftsquery.py

@@ -0,0 +1,127 @@
+import asyncio
+import aiohttp
+from tqdm import tqdm
+from datetime import datetime
+from urllib.parse import urlencode
+import time
+
+class PreciseRateLimiter:
+    def __init__(self, rate=10, interval=1.0):
+        self.rate = rate  # requests per interval
+        self.interval = interval  # in seconds
+        self.token_time = self.interval / self.rate  # time per token
+        self.last_time = time.time()
+        self.lock = asyncio.Lock()
+
+    async def acquire(self):
+        async with self.lock:
+            now = time.time()
+            wait_time = self.last_time + self.token_time - now
+            if wait_time > 0:
+                await asyncio.sleep(wait_time)
+            self.last_time = time.time()
+            return True
+
+class EFTSQuery:
+    def __init__(self):
+        self.headers = {
+            'User-Agent': 'Your Name yourname@email.com',
+            'Accept-Encoding': 'gzip, deflate',
+            'Host': 'efts.sec.gov'
+        }
+        self.session = None
+        self.limiter = PreciseRateLimiter(10)
+
+    async def __aenter__(self):
+        if not self.session:
+            self.session = aiohttp.ClientSession(headers=self.headers)
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self.session:
+            await self.session.close()
+            self.session = None
+
+    async def _fetch_json(self, url):
+        await self.limiter.acquire()
+        try:
+            async with self.session.get(url) as response:
+                if response.status == 429:
+                    await asyncio.sleep(61)
+                    return await self._fetch_json(url)
+                return await response.json()
+        except Exception as e:
+            print(f"Error fetching {url}: {str(e)}")
+            return None
+
+    async def _get_accession_numbers(self, base_url):
+        data = await self._fetch_json(f"{base_url}&from=0&size=1")
+        if not data or 'hits' not in data:
+            return []
+            
+        total_hits = data['hits']['total']['value']
+        if not total_hits:
+            return []
+
+        accession_numbers = []
+        start = 0
+        page_size = 100
+        batch_size = 10  # Number of concurrent requests
+        
+        with tqdm(total=total_hits) as pbar:
+            while start < total_hits:
+                tasks = []
+                for i in range(batch_size):
+                    if start + i * page_size >= total_hits:
+                        break
+                    url = f"{base_url}&from={start + i * page_size}&size={page_size}"
+                    tasks.append(self._fetch_json(url))
+                
+                if not tasks:
+                    break
+                    
+                results = await asyncio.gather(*tasks)
+                
+                for data in results:
+                    if data and 'hits' in data:
+                        hits = data['hits']['hits']
+                        batch_numbers = [
+                            f"{hit['_source']['ciks'][0]}/{hit['_id'].split(':')[0]}"
+                            for hit in hits
+                        ]
+                        accession_numbers.extend(batch_numbers)
+                        pbar.update(len(hits))
+                
+                start += batch_size * page_size
+
+        return accession_numbers
+
+    def query_efts(self, cik=None, ticker=None, submission_type=None, filing_date=None, search_text=None):
+        async def _download():
+            async with self as downloader:
+                params = {}
+                
+                if cik:
+                    params['ciks'] = str(cik).zfill(10)
+
+                if submission_type:
+                    params['forms'] = ','.join(submission_type) if isinstance(submission_type, list) else submission_type
+
+                if isinstance(filing_date, list):
+                    dates = [(d, d) for d in filing_date]
+                elif isinstance(filing_date, tuple):
+                    dates = [filing_date]
+                else:
+                    date_str = filing_date if filing_date else f"2001-01-01,{datetime.now().strftime('%Y-%m-%d')}"
+                    start, end = date_str.split(',')
+                    dates = [(start, end)]
+                
+                params['startdt'], params['enddt'] = dates[0]
+
+                if search_text:
+                    params['q'] = f'"{search_text}"'
+
+                base_url = f"https://efts.sec.gov/LATEST/search-index?{urlencode(params, doseq=True)}"
+                return await self._get_accession_numbers(base_url)
+
+        return asyncio.run(_download())

datamule-1.0.3/datamule/book/xbrl_retriever.py

@@ -0,0 +1,88 @@
+import asyncio
+import aiohttp
+import time
+
+class PreciseRateLimiter:
+    def __init__(self, rate=10, interval=1.0):
+        self.rate = rate
+        self.interval = interval
+        self.token_time = self.interval / self.rate
+        self.last_time = time.time()
+        self.lock = asyncio.Lock()
+
+    async def acquire(self):
+        async with self.lock:
+            now = time.time()
+            wait_time = self.last_time + self.token_time - now
+            if wait_time > 0:
+                await asyncio.sleep(wait_time)
+            self.last_time = time.time()
+            return True
+
+class XBRLRetriever:
+    def __init__(self):
+        self.base_url = "https://data.sec.gov/api/xbrl/frames"
+        self.headers = {
+            'User-Agent': 'Your Name yourname@email.com',
+            'Accept-Encoding': 'gzip, deflate',
+            'Host': 'data.sec.gov'
+        }
+        self.session = None
+        self.limiter = PreciseRateLimiter(10)
+
+    async def __aenter__(self):
+        if not self.session:
+            self.session = aiohttp.ClientSession(headers=self.headers)
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self.session:
+            await self.session.close()
+            self.session = None
+
+    async def _fetch_json(self, url):
+        await self.limiter.acquire()
+        try:
+            async with self.session.get(url) as response:
+                if response.status == 429:
+                    await asyncio.sleep(61)
+                    return await self._fetch_json(url)
+                elif response.status == 200:
+                    return await response.json()
+                else:
+                    print(f"Error {response.status} for URL: {url}")
+                    return None
+        except Exception as e:
+            print(f"Error fetching {url}: {str(e)}")
+            return None
+
+    def _build_url(self, params):
+        taxonomy = params.get('taxonomy')
+        concept = params.get('concept')
+        unit = params.get('unit')
+        period = params.get('period')
+        
+        if not all([taxonomy, concept, unit, period]):
+            raise ValueError("Missing required parameters")
+            
+        return f"{self.base_url}/{taxonomy}/{concept}/{unit}/{period}.json"
+
+    async def _get_xbrl_data(self, params_list):
+        tasks = []
+        urls = {}
+        
+        for params in params_list:
+            url = self._build_url(params)
+            urls[url] = params
+            tasks.append(self._fetch_json(url))
+        
+        results = await asyncio.gather(*tasks)
+        
+        return {url: result for url, result in zip(urls.keys(), results) if result is not None}
+
+    def get_xbrl_frames(self, params_list):
+        async def _download():
+            async with self as downloader:
+                return await self._get_xbrl_data(params_list)
+
+        return asyncio.run(_download())

datamule-1.0.3/datamule/document.py

@@ -0,0 +1,279 @@
+import json
+import csv
+from .helper import convert_to_dashed_accession
+import re
+from doc2dict import xml2dict, txt2dict, dict2dict
+from doc2dict.mapping import flatten_hierarchy
+from .mapping_dicts import txt_mapping_dicts
+from .mapping_dicts import xml_mapping_dicts
+from selectolax.parser import HTMLParser
+
+class Document:
+    def __init__(self, type, filename):
+        self.type = type
+        self.path = filename
+
+        self.data = None
+        self.content = None
+
+
+    def load_content(self,encoding='utf-8'):
+        with open(self.path, 'r',encoding=encoding) as f:
+            self.content = f.read()
+
+    def _load_text_content(self):
+        with open(self.path) as f:
+            return f.read().translate(str.maketrans({
+                '\xa0': ' ', '\u2003': ' ',
+                '\u2018': "'", '\u2019': "'",
+                '\u201c': '"', '\u201d': '"'
+            }))
+
+    # will deprecate this when we add html2dict
+    def _load_html_content(self):
+        with open(self.path,'rb') as f:
+            parser = HTMLParser(f.read(),detect_encoding=True,decode_errors='ignore')
+        
+        # Remove hidden elements first
+        hidden_nodes = parser.css('[style*="display: none"], [style*="display:none"], .hidden, .hide, .d-none')
+        for node in hidden_nodes:
+            node.decompose()
+        
+        blocks = {'p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'article', 'section', 'li', 'td'}
+        lines = []
+        current_line = []
+        
+        def flush_line():
+            if current_line:
+                # Don't add spaces between adjacent spans
+                lines.append(''.join(current_line))
+                current_line.clear()
+        
+        for node in parser.root.traverse(include_text=True):
+            if node.tag in ('script', 'style', 'css'):
+                continue
+                
+            if node.tag in blocks:
+                flush_line()
+                lines.append('')
+                
+            if node.text_content:
+                text = node.text_content.strip()
+                if text:
+                    if node.tag in blocks:
+                        flush_line()
+                        lines.append(text)
+                        lines.append('')
+                    else:
+                        # Only add space if nodes aren't directly adjacent
+                        if current_line and not current_line[-1].endswith(' '):
+                            if node.prev and node.prev.text_content:
+                                if node.parent != node.prev.parent or node.prev.next != node:
+                                    current_line.append(' ')
+                        current_line.append(text)
+        
+        flush_line()
+        
+        text = '\n'.join(lines)
+        while '\n\n\n' in text:
+            text = text.replace('\n\n\n', '\n\n')
+        
+        return text.translate(str.maketrans({
+            '\xa0': ' ', '\u2003': ' ',
+            '\u2018': "'", '\u2019': "'",
+            '\u201c': '"', '\u201d': '"'
+        }))
+
+    def _load_file_content(self):
+        if self.path.suffix =='.txt':
+            self.content = self._load_text_content()
+        elif self.path.suffix in ['.html','.htm']:
+            self.content =  self._load_html_content()
+        else:
+            raise ValueError(f"Unsupported file type: {self.path.suffix}")
+
+
+    def contains_string(self, pattern):
+        """Currently only works for .htm, .html, and .txt files"""
+        if self.path.suffix in ['.htm', '.html', '.txt']:
+            if self.content is None:
+                self.content = self._load_file_content(self.path)
+            return bool(re.search(pattern, self.content))
+        return False
+
+    # Note: this method will be heavily modified in the future
+    def parse(self):
+        mapping_dict = None
+
+        if self.path.suffix == '.xml':
+            if self.type in ['3', '4', '5']:
+                mapping_dict = xml_mapping_dicts.dict_345
+
+            self.load_content()
+            self.data = xml2dict(content=self.content, mapping_dict=mapping_dict)
+        # will deprecate this when we add html2dict
+        elif self.path.suffix in ['.htm', '.html','.txt']:
+            self._load_file_content()
+
+            if self.type == '10-K':
+                mapping_dict = txt_mapping_dicts.dict_10k
+            elif self.type == '10-Q':
+                mapping_dict = txt_mapping_dicts.dict_10q
+            elif self.type == '8-K':
+                mapping_dict = txt_mapping_dicts.dict_8k
+            elif self.type == 'SC 13D':
+                mapping_dict = txt_mapping_dicts.dict_13d
+            elif self.type == 'SC 13G':
+                mapping_dict = txt_mapping_dicts.dict_13g
+            
+            self.data = {}
+            self.data['document'] = dict2dict(txt2dict(content=self.content, mapping_dict=mapping_dict))
+        return self.data
+    
+    def write_json(self, output_filename=None):
+        if not self.data:
+            self.parse()
+            
+        if output_filename is None:
+            output_filename = f"{self.path.rsplit('.', 1)[0]}.json"
+            
+        with open(output_filename, 'w',encoding='utf-8') as f:
+            json.dump(self.data, f, indent=2)
+
+    def write_csv(self, output_filename=None, accession_number=None):
+        self.parse()
+
+        if output_filename is None:
+            output_filename = f"{self.path.rsplit('.', 1)[0]}.csv"
+
+        with open(output_filename, 'w', newline='') as csvfile:
+            if not self.data:
+                return output_filename
+
+            has_document = any('document' in item for item in self.data)
+            
+            if has_document and 'document' in self.data:
+                writer = csv.DictWriter(csvfile, ['section', 'text'], quoting=csv.QUOTE_ALL)
+                writer.writeheader()
+                flattened = self._flatten_dict(self.data['document'])
+                for section, text in flattened.items():
+                    writer.writerow({'section': section, 'text': text})
+            else:
+                fieldnames = list(self.data[0].keys())
+                if accession_number:
+                    fieldnames.append('Accession Number')
+                writer = csv.DictWriter(csvfile, fieldnames, quoting=csv.QUOTE_ALL)
+                writer.writeheader()
+                for row in self.data:
+                    if accession_number:
+                        row['Accession Number'] = convert_to_dashed_accession(accession_number)
+                    writer.writerow(row)
+
+        return output_filename
+    
+    def _document_to_section_text(self, document_data, parent_key=''):
+        items = []
+        
+        if isinstance(document_data, dict):
+            for key, value in document_data.items():
+                # Build the section name
+                section = f"{parent_key}_{key}" if parent_key else key
+                
+                # If the value is a dict, recurse
+                if isinstance(value, dict):
+                    items.extend(self._document_to_section_text(value, section))
+                # If it's a list, handle each item
+                elif isinstance(value, list):
+                    for i, item in enumerate(value):
+                        if isinstance(item, dict):
+                            items.extend(self._document_to_section_text(item, f"{section}_{i+1}"))
+                        else:
+                            items.append({
+                                'section': f"{section}_{i+1}",
+                                'text': str(item)
+                            })
+                # Base case - add the item
+                else:
+                    items.append({
+                        'section': section,
+                        'text': str(value)
+                    })
+        
+        return items
+
+    # we'll modify this for every dict
+    def _flatten_dict(self, d, parent_key=''):
+        items = {}
+        
+        if isinstance(d, list):
+            return [self._flatten_dict(item) for item in d]
+                
+        for k, v in d.items():
+            new_key = f"{parent_key}_{k}" if parent_key else k
+            
+            if isinstance(v, dict):
+                items.update(self._flatten_dict(v, new_key))
+            else:
+                items[new_key] = str(v)
+                    
+        return items
+   
+   # this will all have to be changed. default will be to flatten everything
+    def __iter__(self):
+        if not self.data:
+            self.parse()
+
+        # Let's remove XML iterable for now
+
+        # Handle text-based documents
+        if self.path.suffix in ['.txt', '.htm', '.html']:
+            document_data = self.data
+            if not document_data:
+                return iter([])
+                
+            # Find highest hierarchy level from mapping dict
+            highest_hierarchy = float('inf')
+            section_type = None
+            
+            if self.type in ['10-K', '10-Q']:
+                mapping_dict = txt_mapping_dicts.dict_10k if self.type == '10-K' else txt_mapping_dicts.dict_10q
+            elif self.type == '8-K':
+                mapping_dict = txt_mapping_dicts.dict_8k
+            elif self.type == 'SC 13D':
+                mapping_dict = txt_mapping_dicts.dict_13d
+            elif self.type == 'SC 13G':
+                mapping_dict = txt_mapping_dicts.dict_13g
+            else:
+                return iter([])
+                
+            # Find section type with highest hierarchy number
+            highest_hierarchy = -1  # Start at -1 to find highest
+            for mapping in mapping_dict['rules']['mappings']:
+                if mapping.get('hierarchy') is not None:
+                    if mapping['hierarchy'] > highest_hierarchy:
+                        highest_hierarchy = mapping['hierarchy']
+                        section_type = mapping['name']
+                        
+            if not section_type:
+                return iter([])
+                
+            # Extract sections of the identified type
+            def find_sections(data, target_type):
+                sections = []
+                if isinstance(data, dict):
+                    if data.get('type') == target_type:
+                        sections.append({
+                            'item': data.get('text', ''),
+                            'text': flatten_hierarchy(data.get('content', []))
+                        })
+                    for value in data.values():
+                        if isinstance(value, (dict, list)):
+                            sections.extend(find_sections(value, target_type))
+                elif isinstance(data, list):
+                    for item in data:
+                        sections.extend(find_sections(item, target_type))
+                return sections
+                
+            return iter(find_sections(document_data, section_type))
+            
+        return iter([])

{datamule-0.428 → datamule-1.0.3}/datamule/downloader/downloader.py

@@ -10,7 +10,7 @@ import time
 from collections import deque
 
 from ..helper import identifier_to_cik, load_package_csv, fix_filing_url, headers
-from ..parser.sgml_parsing.sgml_parser_cy import parse_sgml_submission
+from secsgml import parse_sgml_submission
 
 class RetryException(Exception):
     def __init__(self, url, retry_after=601):
@@ -122,8 +122,8 @@ class Downloader:
                     raise RetryException(url)
                 raise
 
-    async def _get_filing_urls_from_efts(self, base_url):
-        """Fetch filing URLs from EFTS in batches."""
+    async def _get_filing_urls_from_efts(self, base_url, submission_type=None):
+        """Fetch filing URLs from EFTS in batches with form type filtering."""
         start = 0
         page_size = 100
         urls = []
@@ -152,12 +152,22 @@ class Downloader:
                     if data and 'hits' in data:
                         hits = data['hits']['hits']
                         if hits:
+                            # Filter hits based on exact form match
+                            if not submission_type or submission_type == "-0":
+                                filtered_hits = hits
+                            else:
+                                requested_forms = [submission_type] if isinstance(submission_type, str) else submission_type
+                                filtered_hits = [
+                                    hit for hit in hits
+                                    if hit['_source'].get('form', '') in requested_forms
+                                ]
+                            
                             batch_urls = [
                                 f"https://www.sec.gov/Archives/edgar/data/{hit['_source']['ciks'][0]}/{hit['_id'].split(':')[0]}.txt" 
-                                for hit in hits
+                                for hit in filtered_hits
                             ]
                             urls.extend(batch_urls)
-                            pbar.update(len(hits))
+                            pbar.update(len(hits))  # Update progress based on total hits processed
                             self.update_progress_description()
                 
                 start += 10 * page_size
@@ -173,7 +183,7 @@ class Downloader:
         pbar.close()
         self.current_pbar = None
         return urls
-
+        
     async def _download_file(self, url, filepath):
         """Download single file with precise rate limiting."""
         async with self.connection_semaphore:
@@ -197,7 +207,7 @@ class Downloader:
 
                                 parsed_data = parse_sgml_submission(
                                     content=content.decode(), 
-                                    output_dir=os.path.dirname(filepath) + f'/{url.split("/")[-1].split(".")[0].replace("-", "")}'
+                                    output_dir=os.path.dirname(filepath)
                                 )
                                 
                                 try:
@@ -306,7 +316,7 @@ class Downloader:
                     base_url = "https://efts.sec.gov/LATEST/search-index"
                     efts_url = f"{base_url}?{urlencode(params, doseq=True)}"
                     
-                    urls = await self._get_filing_urls_from_efts(efts_url)
+                    urls = await self._get_filing_urls_from_efts(efts_url,submission_type)
                     if urls:
                         filepaths, parsed_data = await self._download_and_process(urls, output_dir)
                         all_filepaths.extend(filepaths)