datamule 1.2.5__py3-none-any.whl → 1.2.9__py3-none-any.whl

datamule/document/table.py

@@ -18,7 +18,12 @@ from .mappings.thirteenfhr import *
 from .mappings.twentyfivense import *
 from .mappings.twentyfourf2nt import *
 from .mappings.information_table import *
+from .mappings.submission_metadata import *
+from .mappings.ex102_abs import *
+from .mappings.d import *
 
+from pathlib import Path
+import csv
 # need to check if mappings correctly create new columns
 class Table():
     def __init__(self, data, type,accession):
@@ -27,11 +32,18 @@ class Table():
         self.type = type
         self.data = data
         self.accession = accession
-        self.columns = self.determine_columns()
+        self.columns = self.determine_columns_complete()
+
+    def determine_columns_complete(self):
+        if not self.data:
+            return []
+        return list(set().union(*(row.keys() for row in self.data)))
+
 
     def determine_columns(self):
         if len(self.data) == 0:
             return []
+        
         return self.data[0].keys()
 
     def add_column(self,column_name,value):
@@ -190,6 +202,17 @@ class Table():
         elif self.type == 'signature_schedule_13':
             mapping_dict = signature_schedule_13_dict
 
+        # D 
+        elif self.type == 'issuer_list_d':
+            mapping_dict = issuer_list_d_dict
+        elif self.type == 'metadata_d':
+            mapping_dict = metadata_d_dict
+        elif self.type == 'offering_data_d':
+            mapping_dict = offering_data_d_dict
+        elif self.type == 'primary_issuer_d':
+            mapping_dict = primary_issuer_d_dict
+        elif self.type == 'related_persons_list_d':
+            mapping_dict = related_persons_d_dict
         # SDR
         elif self.type == 'sdr':
             mapping_dict = sdr_dict
@@ -227,6 +250,15 @@ class Table():
             mapping_dict = item_9_24f2nt_dict
         elif self.type == 'signature_info_schedule_a':
             mapping_dict = signature_24f2nt_dict
+        # ABS
+        elif self.type == 'assets_ex102_absee':
+            mapping_dict = assets_dict_ex102_abs
+        elif self.type =='properties_ex102_absee':
+            mapping_dict = properties_dict_ex102_abs
+        # submission metadata
+        elif self.type == 'document_submission_metadata':
+            mapping_dict = document_submission_metadata_dict
+        
 
         else:
             mapping_dict = {}
@@ -245,9 +277,6 @@ class Table():
             for old_key, new_key in mapping_dict.items():
                 if old_key in row:
                     ordered_row[new_key] = row.pop(old_key)
-                else:
-                    # if the old key is not present, set the new key to None
-                    ordered_row[new_key] = None
             
             # Then add any remaining keys that weren't in the mapping
             for key, value in row.items():
@@ -257,4 +286,30 @@ class Table():
             row.clear()
             row.update(ordered_row)
 
-        self.determine_columns()
+        # Update the columns after mapping
+        columns = set(self.columns)
+        # remove the old columns that are now in the mapping
+        columns.difference_update(mapping_dict.keys())
+        # add the new columns from the mapping
+        columns.update(mapping_dict.values())
+        # add the accession column to the columns
+        columns.add('accession')
+
+        self.columns = list(columns)
+
+    def write_csv(self, output_file):
+        output_file = Path(output_file)
+        fieldnames = self.columns
+        
+        # Check if the file already exists
+        if output_file.exists():
+            # Append to existing file without writing header
+            with open(output_file, 'a', newline='') as csvfile:
+                writer = csv.DictWriter(csvfile, fieldnames=fieldnames, quoting=csv.QUOTE_ALL)
+                writer.writerows(self.data)
+        else:
+            # Create new file with header
+            with open(output_file, 'w', newline='') as csvfile:
+                writer = csv.DictWriter(csvfile, fieldnames=fieldnames, quoting=csv.QUOTE_ALL)
+                writer.writeheader()
+                writer.writerows(self.data)

datamule/helper.py

@@ -79,7 +79,16 @@ def _process_cik_and_metadata_filters(cik=None, ticker=None, **kwargs):
 
         # Convert ticker to CIK if provided
         if ticker is not None:
-            cik = get_cik_from_dataset('listed_filer_metadata', 'ticker', ticker)
+            if isinstance(ticker, str):
+                ticker = [ticker]
+
+            ciks_from_ticker = []
+            for t in ticker:
+                ciks = get_cik_from_dataset('listed_filer_metadata', 'ticker', t)
+                if ciks:
+                    ciks_from_ticker.extend(ciks)
+
+            cik = ciks
 
         # Normalize CIK format
         if cik is not None:

datamule/index.py

@@ -1,16 +1,16 @@
-from pathlib import Path
+
 from .sec.submissions.textsearch import query
-from .helper import _process_cik_and_metadata_filters, load_package_dataset
+from .helper import _process_cik_and_metadata_filters
+from pathlib import Path
 
 class Index:
-    def __init__(self, path=None):
-        self.path = Path(path) if path else None
+    def __init__(self):
+        pass
         
     def search_submissions(
         self,
         text_query,
-        start_date=None,
-        end_date=None,
+        filing_date=None,
         submission_type=None,
         cik=None,
         ticker=None,
@@ -47,16 +47,14 @@ class Index:
         # Execute the search query
         results = query(
             f'{text_query}',
-            filing_date=(start_date, end_date),
+            filing_date=filing_date,
             requests_per_second=requests_per_second,
             quiet=quiet,
             submission_type=submission_type,
             **kwargs
         )
         
-        # Save results to path if specified
-        if self.path:
-            self._save_results(results, text_query)
+
             
         return results
     

datamule/portfolio.py

@@ -9,22 +9,28 @@ import os
 from .helper import _process_cik_and_metadata_filters
 from .seclibrary.downloader import download as seclibrary_download
 from .sec.xbrl.filter_xbrl import filter_xbrl
-from .sec.submissions.monitor import monitor
-from .sec.xbrl.xbrlmonitor import XBRLMonitor
+from .sec.submissions.monitor import Monitor
+#from .sec.xbrl.xbrlmonitor import XBRLMonitor
 
 
 class Portfolio:
     def __init__(self, path):
         self.path = Path(path)
+        self.api_key = None
         self.submissions = []
         self.submissions_loaded = False
         self.MAX_WORKERS = os.cpu_count() - 1 
+
+        self.monitor = Monitor()
         
         if self.path.exists():
             self._load_submissions()
             self.submissions_loaded = True
         else:
             self.path.mkdir(parents=True, exist_ok=True)
+
+    def set_api_key(self, api_key):
+        self.api_key = api_key
     
     def _load_submissions(self):
         folders = [f for f in self.path.iterdir() if f.is_dir()]
@@ -132,6 +138,7 @@ class Portfolio:
             seclibrary_download(
                 output_dir=self.path,
                 cik=cik,
+                api_key=self.api_key,
                 submission_type=submission_type,
                 filing_date=filing_date,
                 accession_numbers=self.accession_numbers if hasattr(self, 'accession_numbers') else None,
@@ -149,20 +156,18 @@ class Portfolio:
             )
 
         self.submissions_loaded = False
-    def monitor_submissions(self,data_callback=None, poll_callback=None, submission_type=None, cik=None, 
-           polling_interval=200, requests_per_second=5, quiet=False, start_date=None, ticker=None, **kwargs):
+    def monitor_submissions(self, data_callback=None, interval_callback=None,
+                            polling_interval=1000, quiet=True, start_date=None,
+                            validation_interval=600000):
         
-        cik = _process_cik_and_metadata_filters(cik, ticker, **kwargs)
 
-        monitor(
+        self.monitor.monitor_submissions(
             data_callback=data_callback,
-            poll_callback=poll_callback,
-            cik=cik,
-            submission_type=submission_type,
+            interval_callback=interval_callback,
             polling_interval=polling_interval,
-            requests_per_second=requests_per_second,
             quiet=quiet,
-            start_date=start_date
+            start_date=start_date,
+            validation_interval=validation_interval
         )
 
         

datamule/sec/submissions/monitor.py

@@ -1,130 +1,183 @@
+import time
+from collections import deque
+from datetime import datetime
+import xml.etree.ElementTree as ET
+import re
 import asyncio
-from datetime import datetime, timedelta
-from .eftsquery import EFTSQuery  # Import the class directly instead of the function
-from ..rss.monitor import start_monitor  # Import start_monitor directly
-import pytz
+from ..utils import headers, PreciseRateLimiter
+from .eftsquery import EFTSQuery
+import aiohttp
 
 
-async def _process_efts_hits(hits, collected_accession_numbers, data_callback=None,rate_limiter=None):
-    """Process EFTS hits, collect accession numbers, and call data callback."""
-    processed_hits = []
+async def poll_rss(limiter):
+    base_url = 'https://www.sec.gov/cgi-bin/browse-edgar?count=100&action=getcurrent&output=rss'
     
-    for hit in hits:
-        try:
-            source = hit.get('_source', {})
-            
-            # Extract key fields
-            accession_number = source.get('adsh')
-                
-            # Extract submission_type (form) and ciks
-            submission_type = source.get('form')
-            ciks = source.get('ciks', [])
-            ciks = [str(int(cik)) for cik in ciks]
-
-            filing_date = source.get('file_date')
-                
-            # Create standardized filing record
-            filing = {
-                'accession_number': accession_number,
-                'submission_type': submission_type,
-                'ciks': ciks,
-                'filing_date': filing_date,
-
-            }
-            
-            processed_hits.append(filing)
-            collected_accession_numbers.add(accession_number)  # Changed append to add for set operation
-            
-        except Exception as e:
-            print(f"Error processing EFTS hit: {e}")
+    # Create a session specifically for this RSS polling operation
+    async with aiohttp.ClientSession(headers=headers) as session:
+        # Use the rate limiter before making the request
+        async with limiter:
+            # Make the HTTP request with the session
+            async with session.get(base_url) as response:
+                content = await response.read()
     
-    # Call data callback if provided
-    if data_callback and processed_hits:
-        await data_callback(processed_hits, rate_limiter)
+    # Process the content
+    content_str = content.decode('utf-8')
+    root = ET.fromstring(content_str)
+    namespace = {'atom': 'http://www.w3.org/2005/Atom'}
+    entries = root.findall('atom:entry', namespace)
+    grouped = {}
+
+    for entry in entries:
+        url = entry.find('atom:link', namespace).get('href')
+        accession = re.search(r'/(\d{10})-(\d{2})-(\d{6})', url)
+        accession = accession.group(1) + accession.group(2) + accession.group(3)
+        cik = re.search(r'/data/(\d+)/', url).group(1)
         
-    return processed_hits
-
-async def _master_monitor_impl(data_callback=None, poll_callback=None, submission_type=None, cik=None, 
-                              polling_interval=200, requests_per_second=2.0, quiet=True, start_date=None):
-    """Implementation of the master monitor."""
-    # Set default start date to today if not provided (eastern)
-    eastern_tz = pytz.timezone('US/Eastern')
-    current_date = datetime.now(eastern_tz).strftime('%Y-%m-%d')
-    if not start_date:
-        start_date = current_date
+        if accession not in grouped:
+            grouped[accession] = {'submission_type': '', 'ciks': set(), 'filing_date': ''}
         
-    # Changed from list to set for more efficient lookups
-    collected_accession_numbers = set()
-    
-    if not quiet:
-        print(f"Starting SEC monitoring from {start_date}")
-    
-    # Step 1: Query EFTS for all filings from start_date up to current date
-    if not quiet:
-        print(f"Fetching filings from {start_date} to {current_date}...")
-    
-    # Prepare a wrapper callback to collect accession numbers
-    async def process_callback(hits):
-         await _process_efts_hits(hits, collected_accession_numbers, data_callback, efts_query.limiter)
-    
-    # Create an EFTSQuery instance
-    efts_query = EFTSQuery(requests_per_second=requests_per_second)
+        grouped[accession]['ciks'].add(cik)
+        grouped[accession]['submission_type'] = entry.find('atom:category', namespace).get('term')
+        summary_text = entry.find('atom:summary', namespace).text
+        filing_date_match = re.search(r'Filed:</b>\s*(\d{4}-\d{2}-\d{2})', summary_text)
+        if filing_date_match:
+            grouped[accession]['filing_date'] = filing_date_match.group(1)
+
+    results = [{'accession': int(k.replace('-', '')), 'submission_type': v['submission_type'], 'ciks': list(v['ciks']), 'filing_date': v['filing_date']} for k, v in grouped.items()]
+    return results
+
+def clean_efts_hits(hits):
+    # clean hits
+    hits = [{'accession': int(hit['_source']['adsh'].replace('-','')), 'filing_date': hit['_source']['file_date'], 'ciks': hit['_source']['ciks']} for hit in hits]
+    return hits
+
+class Monitor():
+    def __init__(self):
+        self.accessions = deque(maxlen=50000)
+        self.ratelimiters = {'sec.gov': PreciseRateLimiter(rate=5)}
+        self.efts_query = EFTSQuery(quiet=True)
+        self.efts_query.limiter = self.ratelimiters['sec.gov']
+
+    def set_domain_rate_limit(self, domain, rate):
+        self.ratelimiters[domain] = PreciseRateLimiter(rate=rate)
+        if domain == 'sec.gov':
+            self.efts_query.limiter = self.ratelimiters[domain]
     
-    # Run EFTS query for the date range
-    async with efts_query:
-        await efts_query.query(
-            cik=cik,
-            submission_type=submission_type,
-            filing_date=(start_date, current_date),
-            callback=process_callback
+    async def _async_run_efts_query(self, **kwargs):
+        """Async helper method to run EFTS query without creating a new event loop"""
+        # Make sure to set quiet parameter if provided in kwargs
+        self.efts_query.quiet = kwargs.get('quiet', True)
+        return await self.efts_query.query(
+            cik=kwargs.get('cik'), 
+            submission_type=kwargs.get('submission_type'),
+            filing_date=kwargs.get('filing_date'),
+            location=kwargs.get('location'),
+            callback=kwargs.get('callback'),
+            name=kwargs.get('name')
         )
+
+    async def _async_monitor_submissions(self, data_callback=None, interval_callback=None,
+                            polling_interval=1000, quiet=True, start_date=None,
+                            validation_interval=60000):
+        """
+        Async implementation of monitor_submissions.
+        """
+
+        # Backfill if start_date is provided
+        if start_date is not None:
+            today_date = datetime.now().date().strftime('%Y-%m-%d')
+            if not quiet:
+                print(f"Backfilling from {start_date} to {today_date}")
+
+            hits = clean_efts_hits(await self._async_run_efts_query(
+                filing_date=(start_date, today_date),
+                quiet=quiet
+            ))
+
+            new_hits = self._filter_new_accessions(hits)
+            if not quiet:
+                print(f"New submissions found: {len(new_hits)}")
+            if new_hits and data_callback:
+                data_callback(new_hits)
+
+        last_polling_time = time.time()
+        last_validation_time = last_polling_time
+        current_time = last_polling_time
+
+        while True:
+            # RSS polling
+            if not quiet:
+                print(f"Polling RSS feed")
+            results = await poll_rss(self.ratelimiters['sec.gov'])
+            new_results = self._filter_new_accessions(results)
+            if new_results:
+                if not quiet:
+                    print(f"Found {len(new_results)} new submissions via RSS")
+                if data_callback:
+                    data_callback(new_results)
+            
+            # EFTS validation
+            if validation_interval and (current_time - last_validation_time) >= validation_interval/1000:
+                # Get submissions from the last 24 hours for validation
+                today_date = datetime.now().strftime('%Y-%m-%d')
+                if not quiet:
+                    print(f"Validating submissions from {today_date}")
+
+                hits = clean_efts_hits(await self._async_run_efts_query(
+                    filing_date=(today_date, today_date),
+                    quiet=quiet
+                ))
+                
+                new_hits = self._filter_new_accessions(hits)
+                if new_hits:
+                    if not quiet:
+                        print(f"Found {len(new_hits)} new submissions via EFTS validation")
+                    if data_callback:
+                        data_callback(new_hits)
+                last_polling_time = time.time()
+                last_validation_time = current_time
+            
+            # Interval callback
+            if interval_callback:
+                interval_callback()
+
+            next_poll_time = last_polling_time + (polling_interval / 1000)
+            current_time = time.time()
+            time_to_sleep = max(0, next_poll_time - current_time)
+            await asyncio.sleep(time_to_sleep)
+            last_polling_time = next_poll_time
+
+
+    def monitor_submissions(self, data_callback=None, interval_callback=None,
+                            polling_interval=1000, quiet=True, start_date=None,
+                            validation_interval=60000):
+        """
+        Monitor SEC submissions using the EDGAR system.
+        :param data_callback: function to call with the data
+        :param interval_callback: function that executes between polls
+        :param polling_interval: interval between polls in milliseconds
+        :param quiet: if True, suppresses output
+        :param start_date: backfill start date in YYYY-MM-DD format
+        :param validation_interval: interval between validation in milliseconds
+
+        This function combines the speed of the RSS feed (fast, but misses some submissions) with the accuracy of the EFTS system.
+        """
+        # This is now a synchronous wrapper around the async implementation
+        return asyncio.run(self._async_monitor_submissions(
+            data_callback=data_callback,
+            interval_callback=interval_callback,
+            polling_interval=polling_interval,
+            quiet=quiet,
+            start_date=start_date,
+            validation_interval=validation_interval
+        ))
     
-    if not quiet:
-        print(f"Historical query complete. Collected {len(collected_accession_numbers)} accession numbers.")
-    
-    # Step 2: Hand off to RSS monitor with collected accession numbers
-    if not quiet:
-        print("Starting real-time RSS monitoring...")
-    
-    # Start RSS monitor with the set of accession numbers to skip (from EFTS)
-    # and an empty list for ongoing tracking
-    await start_monitor(
-        data_callback=data_callback,
-        poll_callback=poll_callback,
-        submission_type=submission_type,
-        cik=cik,
-        polling_interval=polling_interval,
-        requests_per_second=requests_per_second,
-        quiet=quiet,
-        known_accession_numbers=[],  # Start with an empty list for ongoing tracking
-        skip_initial_accession_numbers=collected_accession_numbers  # Pass the EFTS accession numbers as the skip list
-    )
-
-def monitor(data_callback=None, poll_callback=None, submission_type=None, cik=None, 
-           polling_interval=200, requests_per_second=2.0, quiet=True, start_date=None):
-    """
-    Monitor SEC filings by combining EFTS historical queries with real-time RSS monitoring.
-    
-    Parameters:
-        data_callback (callable): Async function to call when new filings are found.
-                                 Will be called with a list of dicts containing
-                                 'accession_number', 'submission_type', and 'ciks'.
-        poll_callback (callable): Async function to call during RSS polling wait periods.
-        submission_type (str or list): Form type(s) to monitor (e.g., "8-K", "10-Q").
-        cik (str or list): CIK(s) to monitor.
-        polling_interval (int): Polling interval in milliseconds for RSS monitor.
-        requests_per_second (float): Maximum requests per second.
-        quiet (bool): Suppress verbose output.
-        start_date (str): ISO format date (YYYY-MM-DD) from which to start monitoring.
-                        If None, will start from current date. (EASTERN TIME)
-    """
-    return asyncio.run(_master_monitor_impl(
-        data_callback=data_callback,
-        poll_callback=poll_callback,
-        submission_type=submission_type,
-        cik=cik,
-        polling_interval=polling_interval,
-        requests_per_second=requests_per_second,
-        quiet=quiet,
-        start_date=start_date
-    ))
+    def _filter_new_accessions(self, items):
+        """Filter items to only include those with new accession numbers."""
+        new_items = []
+        for item in items:
+            accession = item['accession']
+            if accession not in self.accessions:
+                self.accessions.append(accession)
+                new_items.append(item)
+        return new_items

datamule/sec/submissions/textsearch.py

@@ -1,8 +1,4 @@
 import asyncio
-import aiohttp
-from datetime import datetime
-from urllib.parse import urlencode
-from tqdm import tqdm
 from .eftsquery import EFTSQuery
 
 class TextSearchEFTSQuery(EFTSQuery):

datamule/sec/xbrl/streamcompanyfacts.py

@@ -2,7 +2,7 @@ import asyncio
 import aiohttp
 import json
 from tqdm import tqdm
-from ..utils import PreciseRateLimiter, RateMonitor, RetryException, headers
+from ..utils import PreciseRateLimiter, RateMonitor, headers
 
 async def fetch_company_facts(session, cik, rate_limiter, rate_monitor, pbar):
     # Format CIK with leading zeros to 10 digits

datamule/seclibrary/downloader.py

@@ -1,7 +1,6 @@
 import os
 import asyncio
 import aiohttp
-from pathlib import Path
 from tqdm import tqdm
 import time
 import shutil
@@ -13,11 +12,12 @@ from concurrent.futures import ThreadPoolExecutor
 from functools import partial
 from queue import Queue, Empty
 from threading import Thread
-from secsgml import parse_sgml_submission
 from .query import query
 from os import cpu_count
 from ..submission import Submission
 
+
+
 class Downloader:
     def __init__(self, api_key=None):
         self.BASE_URL = "https://library.datamule.xyz/original/nc/"