datamule 1.5.9__py3-none-any.whl → 1.6.1__py3-none-any.whl

datamule/document/document.py

@@ -12,6 +12,7 @@ from .processing import process_tabular_data
 from pathlib import Path
 import webbrowser
 from secsgml.utils import bytes_to_str
+from secxbrl import parse_inline_xbrl
 
 class Document:
     def __init__(self, type, content, extension,accession,filing_date,path=None):
@@ -33,6 +34,7 @@ class Document:
         self.extension = extension
         # this will be filled by parsed
         self.data = None
+        self.xbrl = None
 
     #_load_text_content
     def _preprocess_txt_content(self):
@@ -101,12 +103,23 @@ class Document:
         if self.extension in ['.htm', '.html', '.txt','.xml']:
             return bool(re.search(pattern, self.content))
         return False
+    
+    def parse_xbrl(self,type='inline'):
+        if self.xbrl:
+            return
+        if type =='inline':
+            if self.extension not in ['.htm','.html']:
+                return
+            
+            self.xbrl = parse_inline_xbrl(self.content)
+        else:
+            raise ValueError("Only inline has been implemented so far.")
 
     # Note: this method will be heavily modified in the future
     def parse(self):
         # check if we have already parsed the content
         if self.data:
-            return self.data
+            return
         
         mapping_dict = None
         

datamule/document/mappings_new/mappings.py

@@ -0,0 +1,13 @@
+import ownership
+
+
+# key is document type
+# note: this assumes XML format.
+table_mappings = {
+    '3' : ownership.mappings,
+    '3/A' : ownership.mappings,
+    '4' : ownership.mappings,
+    '4/A' : ownership.mappings,
+    '5' : ownership.mappings,
+    '5/A' : ownership.mappings,
+}

datamule/document/mappings_new/ownership.py

@@ -0,0 +1,174 @@
+
+
+
+# Non-derivative transaction ownership mapping
+ownership_non_derivative_transactions_dict = {
+    'securityTitle_value': 'securityTitle',
+    'securityTitle_footnote': 'securityTitleFootnote',
+    'transactionDate_value': 'transactionDate',
+    'transactionDate_footnote': 'transactionDateFootnote',
+    'deemedExecutionDate_value': 'deemedExecutionDate',
+    'deemedExecutionDate_footnote': 'deemedExecutionDateFootnote',
+    'transactionCoding_transactionFormType': 'transactionFormType',
+    'transactionCoding_transactionCode': 'transactionCode',
+    'transactionCoding_equitySwapInvolved': 'equitySwapInvolved',
+    'transactionCoding_footnote': 'transactionCodingFootnote',
+    'transactionAmounts_transactionShares_value': 'transactionShares',
+    'transactionAmounts_transactionShares_footnote': 'transactionSharesFootnote',
+    'transactionAmounts_transactionPricePerShare_value': 'transactionPricePerShare',
+    'transactionAmounts_transactionPricePerShare_footnote': 'transactionPricePerShareFootnote',
+    'transactionAmounts_transactionAcquiredDisposedCode_value': 'transactionAcquiredDisposedCode',
+    'transactionAmounts_transactionAcquiredDisposedCode_footnote': 'transactionAcquiredDisposedCodeFootnote',
+    'postTransactionAmounts_sharesOwnedFollowingTransaction_value': 'sharesOwnedFollowingTransaction',
+    'postTransactionAmounts_sharesOwnedFollowingTransaction_footnote': 'sharesOwnedFollowingTransactionFootnote',
+    'ownershipNature_directOrIndirectOwnership_value': 'directOrIndirectOwnership',
+    'ownershipNature_directOrIndirectOwnership_footnote': 'directOrIndirectOwnershipFootnote',
+    'ownershipNature_natureOfOwnership_value': 'natureOfOwnership',
+    'ownershipNature_natureOfOwnership_footnote': 'natureOfOwnershipFootnote',
+    'transactionTimeliness_value': 'transactionTimeliness',
+    'transactionTimeliness_footnote': 'transactionTimelinessFootnote',
+    'postTransactionAmounts_valueOwnedFollowingTransaction_value': 'valueOwnedFollowingTransaction',
+    'postTransactionAmounts_valueOwnedFollowingTransaction_footnote': 'valueOwnedFollowingTransactionFootnote'
+}
+
+# Derivative transaction ownership mapping
+derivative_transaction_ownership_dict = {
+    'securityTitle_value': 'securityTitle',
+    'securityTitle_footnote': 'securityTitleFootnote',
+    'conversionOrExercisePrice_value': 'conversionOrExercisePrice',
+    'conversionOrExercisePrice_footnote': 'conversionOrExercisePriceFootnote',
+    'transactionDate_value': 'transactionDate',
+    'transactionDate_footnote': 'transactionDateFootnote',
+    'deemedExecutionDate_value': 'deemedExecutionDate',
+    'deemedExecutionDate_footnote': 'deemedExecutionDateFootnote',
+    'transactionCoding_transactionFormType': 'transactionFormType',
+    'transactionCoding_transactionCode': 'transactionCode',
+    'transactionCoding_equitySwapInvolved': 'equitySwapInvolved',
+    'transactionCoding_footnote': 'transactionCodingFootnote',
+    'transactionAmounts_transactionShares_value': 'transactionShares',
+    'transactionAmounts_transactionShares_footnote': 'transactionSharesFootnote',
+    'transactionAmounts_transactionPricePerShare_value': 'transactionPricePerShare',
+    'transactionAmounts_transactionPricePerShare_footnote': 'transactionPricePerShareFootnote',
+    'transactionAmounts_transactionAcquiredDisposedCode_value': 'transactionAcquiredDisposedCode',
+    'transactionAmounts_transactionTotalValue_value': 'transactionTotalValue',
+    'transactionAmounts_transactionTotalValue_footnote': 'transactionTotalValueFootnote',
+    'exerciseDate_value': 'exerciseDate',
+    'exerciseDate_footnote': 'exerciseDateFootnote',
+    'expirationDate_value': 'expirationDate',
+    'expirationDate_footnote': 'expirationDateFootnote',
+    'underlyingSecurity_underlyingSecurityTitle_value': 'underlyingSecurityTitle',
+    'underlyingSecurity_underlyingSecurityTitle_footnote': 'underlyingSecurityTitleFootnote',
+    'underlyingSecurity_underlyingSecurityShares_value': 'underlyingSecurityShares',
+    'underlyingSecurity_underlyingSecurityShares_footnote': 'underlyingSecuritySharesFootnote',
+    'underlyingSecurity_underlyingSecurityValue_value': 'underlyingSecurityValue',
+    'postTransactionAmounts_sharesOwnedFollowingTransaction_value': 'sharesOwnedFollowingTransaction',
+    'postTransactionAmounts_sharesOwnedFollowingTransaction_footnote': 'sharesOwnedFollowingTransactionFootnote',
+    'ownershipNature_directOrIndirectOwnership_value': 'directOrIndirectOwnership',
+    'ownershipNature_directOrIndirectOwnership_footnote': 'directOrIndirectOwnershipFootnote',
+    'ownershipNature_natureOfOwnership_value': 'natureOfOwnership',
+    'ownershipNature_natureOfOwnership_footnote': 'natureOfOwnershipFootnote',
+    'transactionTimeliness_value': 'transactionTimeliness',
+    'transactionTimeliness_footnote': 'transactionTimelinessFootnote',
+    'postTransactionAmounts_valueOwnedFollowingTransaction_value': 'valueOwnedFollowingTransaction',
+    'postTransactionAmounts_valueOwnedFollowingTransaction_footnote': 'valueOwnedFollowingTransactionFootnote',
+    'transactionAmounts_transactionAcquiredDisposedCode_footnote': 'transactionAcquiredDisposedCodeFootnote',
+    'underlyingSecurity_underlyingSecurityValue_footnote': 'underlyingSecurityValueFootnote'
+}
+
+# Non-derivative holding ownership mapping
+non_derivative_holding_ownership_dict = {
+    'securityTitle_value': 'securityTitle',
+    'securityTitle_footnote': 'securityTitleFootnote',
+    'postTransactionAmounts_sharesOwnedFollowingTransaction_value': 'sharesOwnedFollowingTransaction',
+    'postTransactionAmounts_sharesOwnedFollowingTransaction_footnote': 'sharesOwnedFollowingTransactionFootnote',
+    'ownershipNature_directOrIndirectOwnership_value': 'directOrIndirectOwnership',
+    'ownershipNature_directOrIndirectOwnership_footnote': 'directOrIndirectOwnershipFootnote',
+    'ownershipNature_natureOfOwnership_value': 'natureOfOwnership',
+    'ownershipNature_natureOfOwnership_footnote': 'natureOfOwnershipFootnote',
+    'postTransactionAmounts_valueOwnedFollowingTransaction_value': 'valueOwnedFollowingTransaction',
+    'transactionCoding_footnote': 'transactionCodingFootnote',
+    'transactionCoding_transactionFormType': 'transactionFormType',
+    'postTransactionAmounts_valueOwnedFollowingTransaction_footnote': 'valueOwnedFollowingTransactionFootnote'
+}
+
+# Derivative holding ownership mapping
+derivative_holding_ownership_dict = {
+    'securityTitle_value': 'securityTitle',
+    'securityTitle_footnote': 'securityTitleFootnote',
+    'conversionOrExercisePrice_value': 'conversionOrExercisePrice',
+    'conversionOrExercisePrice_footnote': 'conversionOrExercisePriceFootnote',
+    'exerciseDate_value': 'exerciseDate',
+    'exerciseDate_footnote': 'exerciseDateFootnote',
+    'expirationDate_value': 'expirationDate',
+    'expirationDate_footnote': 'expirationDateFootnote',
+    'underlyingSecurity_underlyingSecurityTitle_value': 'underlyingSecurityTitle',
+    'underlyingSecurity_underlyingSecurityTitle_footnote': 'underlyingSecurityTitleFootnote',
+    'underlyingSecurity_underlyingSecurityShares_value': 'underlyingSecurityShares',
+    'underlyingSecurity_underlyingSecurityShares_footnote': 'underlyingSecuritySharesFootnote',
+    'underlyingSecurity_underlyingSecurityValue_value': 'underlyingSecurityValue',
+    'underlyingSecurity_underlyingSecurityValue_footnote': 'underlyingSecurityValueFootnote',
+    'ownershipNature_directOrIndirectOwnership_value': 'directOrIndirectOwnership',
+    'ownershipNature_directOrIndirectOwnership_footnote': 'directOrIndirectOwnershipFootnote',
+    'ownershipNature_natureOfOwnership_value': 'natureOfOwnership',
+    'ownershipNature_natureOfOwnership_footnote': 'natureOfOwnershipFootnote',
+    'postTransactionAmounts_sharesOwnedFollowingTransaction_value': 'sharesOwnedFollowingTransaction',
+    'postTransactionAmounts_sharesOwnedFollowingTransaction_footnote': 'sharesOwnedFollowingTransactionFootnote',
+    'postTransactionAmounts_valueOwnedFollowingTransaction_value': 'valueOwnedFollowingTransaction',
+    'postTransactionAmounts_valueOwnedFollowingTransaction_footnote': 'valueOwnedFollowingTransactionFootnote',
+    'transactionCoding_transactionFormType': 'transactionFormType',
+    'transactionCoding_footnote': 'transactionCodingFootnote'
+}
+
+# Reporting owner ownership mapping
+reporting_owner_ownership_dict = {
+    'reportingOwnerAddress_rptOwnerCity': 'rptOwnerCity',
+    'reportingOwnerAddress_rptOwnerState': 'rptOwnerState',
+    'reportingOwnerAddress_rptOwnerStateDescription': 'rptOwnerStateDescription',
+    'reportingOwnerAddress_rptOwnerStreet1': 'rptOwnerStreet1',
+    'reportingOwnerAddress_rptOwnerStreet2': 'rptOwnerStreet2',
+    'reportingOwnerAddress_rptOwnerZipCode': 'rptOwnerZipCode',
+    'reportingOwnerId_rptOwnerCik': 'rptOwnerCik',
+    'reportingOwnerId_rptOwnerName': 'rptOwnerName',
+    'reportingOwnerRelationship_isDirector': 'rptOwnerIsDirector',
+    'reportingOwnerRelationship_isOfficer': 'rptOwnerIsOfficer',
+    'reportingOwnerRelationship_isTenPercentOwner': 'rptOwnerIsTenPercentOwner',
+    'reportingOwnerRelationship_isOther': 'rptOwnerIsOther',
+    'reportingOwnerRelationship_officerTitle': 'rptOwnerOfficerTitle',
+    'reportingOwnerRelationship_otherText': 'rptOwnerOtherText'
+}
+
+# Metadata ownership mapping
+metadata_ownership_dict = {
+    'periodOfReport': 'periodOfReport',
+    'issuer_issuerCik': 'issuerCik',
+    'issuer_issuerName': 'issuerName',
+    'issuer_issuerTradingSymbol': 'issuerTradingSymbol',
+    'documentType': 'documentType',
+    'remarks': 'remarks',
+    'documentDescription': 'documentDescription',
+    'footnotes': 'footnotes',
+    'notSubjectToSection16': 'notSubjectToSection16',
+    'form3HoldingsReported': 'form3HoldingsReported',
+    'form4TransactionsReported': 'form4TransactionsReported',
+    'noSecuritiesOwned': 'noSecuritiesOwned',
+    'aff10b5One': 'aff10b5One',
+    'dateOfOriginalSubmission': 'dateOfOriginalSubmission',
+    'schemaVersion': 'schemaVersion'
+}
+
+# Owner signature ownership mapping
+owner_signature_ownership_dict = {
+    'signatureName': 'signatureName',
+    'signatureDate': 'signatureDate'
+}
+
+
+mappings = {
+    'ownership_non_derivative_transactions' : ownership_non_derivative_transactions_dict,
+    'ownership_derivative_transactions' : derivative_transaction_ownership_dict,
+    'ownership_non_derivative_holdings' : non_derivative_holding_ownership_dict,
+    'ownership_derivative_holdings' : derivative_holding_ownership_dict,
+    'ownership_reporting_owner' : reporting_owner_ownership_dict,
+    'ownership_metadata' : metadata_ownership_dict,
+    'ownership_owner_signature' : owner_signature_ownership_dict
+}

datamule/sec/submissions/monitor.py

@@ -9,16 +9,14 @@ from .eftsquery import EFTSQuery
 import aiohttp
 from zoneinfo import ZoneInfo 
 
-async def poll_rss(limiter):
+async def poll_rss(limiter, session):
     base_url = 'https://www.sec.gov/cgi-bin/browse-edgar?count=100&action=getcurrent&output=rss'
     
-    # Create a session specifically for this RSS polling operation
-    async with aiohttp.ClientSession(headers=headers) as session:
-        # Use the rate limiter before making the request
-        async with limiter:
-            # Make the HTTP request with the session
-            async with session.get(base_url) as response:
-                content = await response.read()
+    # Use the rate limiter before making the request
+    async with limiter:
+        # Use the provided session instead of creating a new one
+        async with session.get(base_url) as response:
+            content = await response.read()
     
     # Process the content
     content_str = content.decode('utf-8')
@@ -70,12 +68,31 @@ class Monitor():
         self.ratelimiters = {'sec.gov': PreciseRateLimiter(rate=5)}
         self.efts_query = EFTSQuery(quiet=True)
         self.efts_query.limiter = self.ratelimiters['sec.gov']
+        self.session = None
+        self.session_created_at = 0
+        self.session_lifetime = 300  # 5 minutes in seconds
 
     def set_domain_rate_limit(self, domain, rate):
         self.ratelimiters[domain] = PreciseRateLimiter(rate=rate)
         if domain == 'sec.gov':
             self.efts_query.limiter = self.ratelimiters[domain]
     
+    async def _ensure_fresh_session(self):
+        """Ensure we have a fresh session, recreating if expired or missing"""
+        current_time = time.time()
+        
+        # Check if we need a new session
+        if (self.session is None or 
+            current_time - self.session_created_at > self.session_lifetime):
+            
+            # Close old session if it exists
+            if self.session:
+                await self.session.close()
+            
+            # Create new session
+            self.session = aiohttp.ClientSession(headers=headers)
+            self.session_created_at = current_time
+    
     async def _async_run_efts_query(self, **kwargs):
         """Async helper method to run EFTS query without creating a new event loop"""
         # Make sure to set quiet parameter if provided in kwargs
@@ -103,83 +120,106 @@ class Monitor():
         if polling_interval is None and validation_interval is None:
             raise ValueError("At least one of polling_interval or validation_interval must be specified")
 
-        # Backfill if start_date is provided
-        if start_date is not None:
-            today_date = datetime.now(ZoneInfo("America/New_York")).strftime('%Y-%m-%d')
-            if not quiet:
-                print(f"Backfilling from {start_date} to {today_date}")
-
-            hits = clean_efts_hits(await self._async_run_efts_query(
-                filing_date=(start_date, today_date),
-                quiet=quiet
-            ))
-
-            new_hits = self._filter_new_accessions(hits)
-            if not quiet:
-                print(f"New submissions found: {len(new_hits)}")
-            if new_hits and data_callback:
-                data_callback(new_hits)
-
-        # Initialize timing variables
-        current_time = time.time()
-        last_polling_time = current_time
-        last_validation_time = current_time
-        
-        # Determine which operations to perform
-        do_polling = polling_interval is not None
-        do_validation = validation_interval is not None
+        # Ensure we have a fresh session
+        await self._ensure_fresh_session()
         
-        while True:
-            current_time = time.time()
-            
-            # RSS polling (if enabled)
-            if do_polling and (current_time - last_polling_time) >= polling_interval/1000:
-                if not quiet:
-                    print(f"Polling RSS feed")
-                results = await poll_rss(self.ratelimiters['sec.gov'])
-                new_results = self._filter_new_accessions(results)
-                if new_results:
-                    if not quiet:
-                        print(f"Found {len(new_results)} new submissions via RSS")
-                    if data_callback:
-                        data_callback(new_results)
-                last_polling_time = current_time
-            
-            # EFTS validation (if enabled)
-            if do_validation and (current_time - last_validation_time) >= validation_interval/1000:
-                # Get submissions from the last 24 hours for validation
+        try:
+            # Backfill if start_date is provided
+            if start_date is not None:
                 today_date = datetime.now(ZoneInfo("America/New_York")).strftime('%Y-%m-%d')
                 if not quiet:
-                    print(f"Validating submissions from {today_date}")
+                    print(f"Backfilling from {start_date} to {today_date}")
 
                 hits = clean_efts_hits(await self._async_run_efts_query(
-                    filing_date=(today_date, today_date),
+                    filing_date=(start_date, today_date),
                     quiet=quiet
                 ))
-                
+
                 new_hits = self._filter_new_accessions(hits)
-                if new_hits:
-                    if not quiet:
-                        print(f"Found {len(new_hits)} new submissions via EFTS validation")
-                    if data_callback:
-                        data_callback(new_hits)
-                last_validation_time = current_time
+                if not quiet:
+                    print(f"New submissions found: {len(new_hits)}")
+                if new_hits and data_callback:
+                    data_callback(new_hits)
+
+            # Initialize timing variables
+            current_time = time.time()
+            last_polling_time = current_time
+            last_validation_time = current_time
             
-            # Interval callback
-            if interval_callback:
-                interval_callback()
-
-            # Calculate next wake-up time
-            next_times = []
-            if do_polling:
-                next_times.append(last_polling_time + (polling_interval / 1000))
-            if do_validation:
-                next_times.append(last_validation_time + (validation_interval / 1000))
+            # Determine which operations to perform
+            do_polling = polling_interval is not None
+            do_validation = validation_interval is not None
             
-            next_wake_time = min(next_times)
-            current_time = time.time()
-            time_to_sleep = max(0, next_wake_time - current_time)
-            await asyncio.sleep(time_to_sleep)
+            while True:
+                current_time = time.time()
+                
+                # RSS polling (if enabled)
+                if do_polling and (current_time - last_polling_time) >= polling_interval/1000:
+                    if not quiet:
+                        print(f"Polling RSS feed")
+                    
+                    # Ensure session is fresh before polling
+                    await self._ensure_fresh_session()
+                    
+                    try:
+                        results = await poll_rss(self.ratelimiters['sec.gov'], self.session)
+                        new_results = self._filter_new_accessions(results)
+                        if new_results:
+                            if not quiet:
+                                print(f"Found {len(new_results)} new submissions via RSS")
+                            if data_callback:
+                                data_callback(new_results)
+                    except Exception as e:
+                        if not quiet:
+                            print(f"RSS polling error: {e}, will recreate session on next poll")
+                        # Force session recreation on next poll
+                        if self.session:
+                            await self.session.close()
+                            self.session = None
+                    
+                    last_polling_time = current_time
+                
+                # EFTS validation (if enabled)
+                if do_validation and (current_time - last_validation_time) >= validation_interval/1000:
+                    # Get submissions from the last 24 hours for validation
+                    today_date = datetime.now(ZoneInfo("America/New_York")).strftime('%Y-%m-%d')
+                    if not quiet:
+                        print(f"Validating submissions from {today_date}")
+
+                    hits = clean_efts_hits(await self._async_run_efts_query(
+                        filing_date=(today_date, today_date),
+                        quiet=quiet
+                    ))
+                    
+                    new_hits = self._filter_new_accessions(hits)
+                    if new_hits:
+                        if not quiet:
+                            print(f"Found {len(new_hits)} new submissions via EFTS validation")
+                        if data_callback:
+                            data_callback(new_hits)
+                    last_validation_time = current_time
+                
+                # Interval callback
+                if interval_callback:
+                    interval_callback()
+
+                # Calculate next wake-up time
+                next_times = []
+                if do_polling:
+                    next_times.append(last_polling_time + (polling_interval / 1000))
+                if do_validation:
+                    next_times.append(last_validation_time + (validation_interval / 1000))
+                
+                next_wake_time = min(next_times)
+                current_time = time.time()
+                time_to_sleep = max(0, next_wake_time - current_time)
+                await asyncio.sleep(time_to_sleep)
+                
+        finally:
+            # Clean up the session when done
+            if self.session:
+                await self.session.close()
+                self.session = None
 
     def monitor_submissions(self, data_callback=None, interval_callback=None,
                             polling_interval=1000, quiet=True, start_date=None,

datamule/submission.py

@@ -251,7 +251,8 @@ class Submission:
                         try: 
                             content = tar.extractfile(filename+'.zst').read()
                         except:
-                            raise ValueError("Something went wrong with tar")
+                            # some of these issues are on SEC data end, will fix when I setup cloud.
+                            raise ValueError(f"Something went wrong with tar: {self.path}")
                 # Decompress if compressed
                 if filename.endswith('.gz'):
                     content = gzip.decompress(content)

datamule/utils/construct_submissions_data.py

@@ -0,0 +1,150 @@
+import zipfile
+import json
+import csv
+import os
+import tempfile
+from concurrent.futures import ThreadPoolExecutor
+import threading
+from tqdm import tqdm
+import urllib.request
+
+headers = {'User-Agent': 'John Smith johnsmith@gmail.com'}
+
+def process_file_batch(zip_file, filenames_batch):
+    """Process a batch of files from the zip archive"""
+    batch_filings = []
+    
+    for filename in filenames_batch:
+        if not filename.startswith('CIK'):
+            continue
+            
+        try:
+            # Extract CIK from filename
+            cik = int(filename.split('.')[0].split('-')[0][3:])
+            
+            # Read raw bytes and parse JSON
+            with zip_file.open(filename) as file:
+                raw_data = file.read()
+                submissions_dct = json.loads(raw_data)
+            
+            # Handle different file types
+            if 'submissions' in filename:
+                filings_data = submissions_dct
+            else:
+                filings_data = submissions_dct['filings']['recent']
+            
+            # Extract required data
+            accession_numbers = filings_data['accessionNumber']
+            filing_dates = filings_data['filingDate']
+            forms = filings_data['form']
+            
+            # Create filing records for this file
+            for j in range(len(accession_numbers)):
+                filing_record = {
+                    'accessionNumber': accession_numbers[j],
+                    'filingDate': filing_dates[j],
+                    'form': forms[j],
+                    'cik': cik
+                }
+                batch_filings.append(filing_record)
+                
+        except Exception as e:
+            print(f"Error processing {filename}: {e}")
+            continue
+    
+    return batch_filings
+
+def write_csv_chunk(output_path, filings_data, is_first_write, write_lock):
+    """Thread-safe CSV writing with lock"""
+    with write_lock:
+        if is_first_write:
+            with open(output_path, 'w', newline='') as csvfile:
+                fieldnames = ['accessionNumber', 'filingDate', 'form', 'cik']
+                writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+                writer.writeheader()
+                writer.writerows(filings_data)
+        else:
+            with open(output_path, 'a', newline='') as csvfile:
+                fieldnames = ['accessionNumber', 'filingDate', 'form', 'cik']
+                writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+                writer.writerows(filings_data)
+
+def construct_submissions_data(output_path, submissions_zip_path=None, max_workers=4, batch_size=100):
+    """Creates a list of dicts of every accession number, with filing date, submission type, and ciks"""
+    
+    if submissions_zip_path is None:
+        url = "https://www.sec.gov/Archives/edgar/daily-index/bulkdata/submissions.zip"
+        
+        temp_dir = tempfile.mkdtemp()
+        zip_path = os.path.join(temp_dir, 'submissions.zip')
+        
+        req = urllib.request.Request(url, headers=headers)
+        
+        with urllib.request.urlopen(req) as response:
+            total_size = int(response.headers.get('Content-Length', 0))
+            
+            with open(zip_path, 'wb') as f, tqdm(
+                desc="Downloading",
+                total=total_size,
+                unit='B',
+                unit_scale=True,
+                unit_divisor=1024,
+            ) as pbar:
+                while True:
+                    chunk = response.read(8192)
+                    if not chunk:
+                        break
+                    f.write(chunk)
+                    pbar.update(len(chunk))
+        
+        submissions_zip_path = zip_path
+    
+    # Keep zip file open throughout processing
+    with zipfile.ZipFile(submissions_zip_path, 'r') as zip_file:
+        # Get all CIK filenames
+        all_filenames = [f for f in zip_file.namelist() if f.startswith('CIK')]
+        
+        print(f"Processing {len(all_filenames)} files with {max_workers} workers...")
+        
+        # Create batches of filenames
+        filename_batches = []
+        for i in range(0, len(all_filenames), batch_size):
+            batch = all_filenames[i:i + batch_size]
+            filename_batches.append(batch)
+        
+        # Setup for threading
+        write_lock = threading.Lock()
+        total_filings = 0
+        is_first_write = True
+        
+        # Process batches with thread pool
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            # Submit all batch jobs
+            future_to_batch = {
+                executor.submit(process_file_batch, zip_file, batch): i 
+                for i, batch in enumerate(filename_batches)
+            }
+            
+            # Process results with progress bar
+            with tqdm(total=len(filename_batches), desc="Processing batches", unit="batch") as pbar:
+                for future in future_to_batch:
+                    try:
+                        batch_filings = future.result()
+                        
+                        if batch_filings:  # Only write if we have data
+                            write_csv_chunk(output_path, batch_filings, is_first_write, write_lock)
+                            is_first_write = False
+                            total_filings += len(batch_filings)
+                        
+                        pbar.update(1)
+                        pbar.set_postfix({
+                            'filings': total_filings,
+                            'files': len(filename_batches[future_to_batch[future]])
+                        })
+                        
+                    except Exception as e:
+                        print(f"Error processing batch: {e}")
+                        pbar.update(1)
+    
+    print(f"Complete! Processed {total_filings} total filings")
+    print(f"Data saved to {output_path}")

{datamule-1.5.9.dist-info → datamule-1.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.5.9
+Version: 1.6.1
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
@@ -15,6 +15,7 @@ Requires-Dist: selectolax
 Requires-Dist: pytz
 Requires-Dist: zstandard
 Requires-Dist: doc2dict
+Requires-Dist: secxbrl
 Requires-Dist: secsgml
 Requires-Dist: websocket-client
 

{datamule-1.5.9.dist-info → datamule-1.6.1.dist-info}/RECORD

@@ -5,12 +5,12 @@ datamule/index.py,sha256=Rrcna9FJV-Oh_K6O2IuUEIDmtay_7UZ4l4jgKCi7A7I,2079
 datamule/package_updater.py,sha256=Z9zaa_y0Z5cknpRn8oPea3gg4kquFHfpfhduKKCZ6NU,958
 datamule/portfolio.py,sha256=eF1eDSwIg-CI8ZmZAHRjCGU0UhuPN4ijxPB0YDT4s2o,8023
 datamule/sheet.py,sha256=TvFqK9eAYuVoJ2uWdAlx5EN6vS9lke-aZf7FqtUiDBc,22304
-datamule/submission.py,sha256=6JIi-ayLL-jENVj6Q4IhmrYlAreJI7xBAHP_NYaDB6k,12918
+datamule/submission.py,sha256=vAiYNas1YrWgm4Grw24peJbfSUVERySEko1zmdtG49s,13033
 datamule/data/listed_filer_metadata.csv,sha256=dT9fQ8AC5P1-Udf_UF0ZkdXJ88jNxJb_tuhi5YYL1rc,2426827
 datamule/datamule/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/datamule/sec_connector.py,sha256=T3edE7I-d4oHysqj7zYlIOxH3Fuauj9tfw39UdFWvB8,2393
 datamule/document/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/document/document.py,sha256=04Rivdphq0D1HEGIBjtl1LelJr-IyQU1qCMi8yNJajw,14038
+datamule/document/document.py,sha256=YGo-Iz_sBXekUeKEAoNJV0BiLDtSOgD9OXFo2FocYq8,14439
 datamule/document/processing.py,sha256=jDCEzBFDSQtq7nQxRScIsbALnFcvMPOkNkMUCa7mFxg,31921
 datamule/document/table.py,sha256=73yUJKY82ap32jhLmZeTti-jQ_lyhcJGlGwyxLtgYOg,12944
 datamule/document/mappings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -37,6 +37,9 @@ datamule/document/mappings/ta.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 datamule/document/mappings/thirteenfhr.py,sha256=XpYRIMPZnGLfEE4TqBI0BPXbyuq0xf3hut1fePOF6kU,4250
 datamule/document/mappings/twentyfivense.py,sha256=lKyj0ZBhkHX9gQJMTUPrQlxYFg3k-aBnWqtoS5bujZM,905
 datamule/document/mappings/twentyfourf2nt.py,sha256=Q7RPT3JgJHjYdjMuaSyAxclt6QPT_LgCQloxp-ByDuI,4118
+datamule/document/mappings_new/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamule/document/mappings_new/mappings.py,sha256=sP94GK3-klMCTD6XFajAP9KxJ7Wq5YMMaXcHx1rQEKA,281
+datamule/document/mappings_new/ownership.py,sha256=GVtyROefvEC_X5l6kayvZv57-kHxj8bHckAru8JtFOQ,10656
 datamule/mapping_dicts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/mapping_dicts/html_mapping_dicts.py,sha256=G2PWB__FNg4VH9iFJFkflM0u-qOEtk67IWtGoqesb0k,5388
 datamule/mapping_dicts/txt_mapping_dicts.py,sha256=DQPrGYbAPQxomRUtt4iiMGrwuF7BHc_LeFBQuYBzU9o,6311
@@ -48,7 +51,7 @@ datamule/sec/infrastructure/submissions_metadata.py,sha256=f1KarzFSryKm0EV8DCDNs
 datamule/sec/submissions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/submissions/downloader.py,sha256=zGS0oJJI8tVF_GnVpZm20MymdYxnjrEjQioSVggw7Ck,1486
 datamule/sec/submissions/eftsquery.py,sha256=mSZon8rlW8dxma7M49ZW5V02Fn-ENOdt9TNO6elBrhE,27983
-datamule/sec/submissions/monitor.py,sha256=ll0nfHzG8FI3bA8zVFrfsfZGnbt5qAD4rRZ4LG2SORY,9567
+datamule/sec/submissions/monitor.py,sha256=1JUMRYsTqtd31hX3UrUA_aXFUmZN6n-V7h0i1gavNOs,11395
 datamule/sec/submissions/streamer.py,sha256=Qydj40CmWB_wsPv2dibefRohmCokegG2pR7iZ9C3xLQ,11584
 datamule/sec/submissions/textsearch.py,sha256=MKDXEz_VI_0ljl73_aw2lx4MVzJW5uDt8KxjvJBwPwM,5794
 datamule/sec/xbrl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -60,7 +63,9 @@ datamule/seclibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 datamule/seclibrary/bq.py,sha256=C8sb_rpXTvchprrFLcbRar4Qi0XWW25tnv1YsHSS5o4,18025
 datamule/seclibrary/downloader.py,sha256=ylv69VF22IVfrdeCkiGr5mVa2GKrPC9zFiDJU1fiBu8,17262
 datamule/seclibrary/query.py,sha256=qGuursTERRbOGfoDcYcpo4oWkW3PCBW6x1Qf1Puiak4,7352
-datamule-1.5.9.dist-info/METADATA,sha256=DkoMbTIImVjWfEkqwfe7BBqCpkvBC8CFRRF5v7PKyco,501
-datamule-1.5.9.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-1.5.9.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-1.5.9.dist-info/RECORD,,
+datamule/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamule/utils/construct_submissions_data.py,sha256=Jn37Ra2_nCIalATCjP_484eUiFP_YeglX_uNdK4Qfu8,5883
+datamule-1.6.1.dist-info/METADATA,sha256=0SEtRwvbaGgU-x_D8u3n0MUPYLssODtQf4GhQrGfl7s,524
+datamule-1.6.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-1.6.1.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-1.6.1.dist-info/RECORD,,