datamule 1.6.0__tar.gz → 1.6.1__tar.gz

{datamule-1.6.0 → datamule-1.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.6.0
+Version: 1.6.1
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.6.0 → datamule-1.6.1}/datamule/sec/submissions/monitor.py

@@ -9,16 +9,14 @@ from .eftsquery import EFTSQuery
 import aiohttp
 from zoneinfo import ZoneInfo 
 
-async def poll_rss(limiter):
+async def poll_rss(limiter, session):
     base_url = 'https://www.sec.gov/cgi-bin/browse-edgar?count=100&action=getcurrent&output=rss'
     
-    # Create a session specifically for this RSS polling operation
-    async with aiohttp.ClientSession(headers=headers) as session:
-        # Use the rate limiter before making the request
-        async with limiter:
-            # Make the HTTP request with the session
-            async with session.get(base_url) as response:
-                content = await response.read()
+    # Use the rate limiter before making the request
+    async with limiter:
+        # Use the provided session instead of creating a new one
+        async with session.get(base_url) as response:
+            content = await response.read()
     
     # Process the content
     content_str = content.decode('utf-8')
@@ -70,12 +68,31 @@ class Monitor():
         self.ratelimiters = {'sec.gov': PreciseRateLimiter(rate=5)}
         self.efts_query = EFTSQuery(quiet=True)
         self.efts_query.limiter = self.ratelimiters['sec.gov']
+        self.session = None
+        self.session_created_at = 0
+        self.session_lifetime = 300  # 5 minutes in seconds
 
     def set_domain_rate_limit(self, domain, rate):
         self.ratelimiters[domain] = PreciseRateLimiter(rate=rate)
         if domain == 'sec.gov':
             self.efts_query.limiter = self.ratelimiters[domain]
     
+    async def _ensure_fresh_session(self):
+        """Ensure we have a fresh session, recreating if expired or missing"""
+        current_time = time.time()
+        
+        # Check if we need a new session
+        if (self.session is None or 
+            current_time - self.session_created_at > self.session_lifetime):
+            
+            # Close old session if it exists
+            if self.session:
+                await self.session.close()
+            
+            # Create new session
+            self.session = aiohttp.ClientSession(headers=headers)
+            self.session_created_at = current_time
+    
     async def _async_run_efts_query(self, **kwargs):
         """Async helper method to run EFTS query without creating a new event loop"""
         # Make sure to set quiet parameter if provided in kwargs
@@ -103,83 +120,106 @@ class Monitor():
         if polling_interval is None and validation_interval is None:
             raise ValueError("At least one of polling_interval or validation_interval must be specified")
 
-        # Backfill if start_date is provided
-        if start_date is not None:
-            today_date = datetime.now(ZoneInfo("America/New_York")).strftime('%Y-%m-%d')
-            if not quiet:
-                print(f"Backfilling from {start_date} to {today_date}")
-
-            hits = clean_efts_hits(await self._async_run_efts_query(
-                filing_date=(start_date, today_date),
-                quiet=quiet
-            ))
-
-            new_hits = self._filter_new_accessions(hits)
-            if not quiet:
-                print(f"New submissions found: {len(new_hits)}")
-            if new_hits and data_callback:
-                data_callback(new_hits)
-
-        # Initialize timing variables
-        current_time = time.time()
-        last_polling_time = current_time
-        last_validation_time = current_time
-        
-        # Determine which operations to perform
-        do_polling = polling_interval is not None
-        do_validation = validation_interval is not None
+        # Ensure we have a fresh session
+        await self._ensure_fresh_session()
         
-        while True:
-            current_time = time.time()
-            
-            # RSS polling (if enabled)
-            if do_polling and (current_time - last_polling_time) >= polling_interval/1000:
-                if not quiet:
-                    print(f"Polling RSS feed")
-                results = await poll_rss(self.ratelimiters['sec.gov'])
-                new_results = self._filter_new_accessions(results)
-                if new_results:
-                    if not quiet:
-                        print(f"Found {len(new_results)} new submissions via RSS")
-                    if data_callback:
-                        data_callback(new_results)
-                last_polling_time = current_time
-            
-            # EFTS validation (if enabled)
-            if do_validation and (current_time - last_validation_time) >= validation_interval/1000:
-                # Get submissions from the last 24 hours for validation
+        try:
+            # Backfill if start_date is provided
+            if start_date is not None:
                 today_date = datetime.now(ZoneInfo("America/New_York")).strftime('%Y-%m-%d')
                 if not quiet:
-                    print(f"Validating submissions from {today_date}")
+                    print(f"Backfilling from {start_date} to {today_date}")
 
                 hits = clean_efts_hits(await self._async_run_efts_query(
-                    filing_date=(today_date, today_date),
+                    filing_date=(start_date, today_date),
                     quiet=quiet
                 ))
-                
+
                 new_hits = self._filter_new_accessions(hits)
-                if new_hits:
-                    if not quiet:
-                        print(f"Found {len(new_hits)} new submissions via EFTS validation")
-                    if data_callback:
-                        data_callback(new_hits)
-                last_validation_time = current_time
+                if not quiet:
+                    print(f"New submissions found: {len(new_hits)}")
+                if new_hits and data_callback:
+                    data_callback(new_hits)
+
+            # Initialize timing variables
+            current_time = time.time()
+            last_polling_time = current_time
+            last_validation_time = current_time
             
-            # Interval callback
-            if interval_callback:
-                interval_callback()
-
-            # Calculate next wake-up time
-            next_times = []
-            if do_polling:
-                next_times.append(last_polling_time + (polling_interval / 1000))
-            if do_validation:
-                next_times.append(last_validation_time + (validation_interval / 1000))
+            # Determine which operations to perform
+            do_polling = polling_interval is not None
+            do_validation = validation_interval is not None
             
-            next_wake_time = min(next_times)
-            current_time = time.time()
-            time_to_sleep = max(0, next_wake_time - current_time)
-            await asyncio.sleep(time_to_sleep)
+            while True:
+                current_time = time.time()
+                
+                # RSS polling (if enabled)
+                if do_polling and (current_time - last_polling_time) >= polling_interval/1000:
+                    if not quiet:
+                        print(f"Polling RSS feed")
+                    
+                    # Ensure session is fresh before polling
+                    await self._ensure_fresh_session()
+                    
+                    try:
+                        results = await poll_rss(self.ratelimiters['sec.gov'], self.session)
+                        new_results = self._filter_new_accessions(results)
+                        if new_results:
+                            if not quiet:
+                                print(f"Found {len(new_results)} new submissions via RSS")
+                            if data_callback:
+                                data_callback(new_results)
+                    except Exception as e:
+                        if not quiet:
+                            print(f"RSS polling error: {e}, will recreate session on next poll")
+                        # Force session recreation on next poll
+                        if self.session:
+                            await self.session.close()
+                            self.session = None
+                    
+                    last_polling_time = current_time
+                
+                # EFTS validation (if enabled)
+                if do_validation and (current_time - last_validation_time) >= validation_interval/1000:
+                    # Get submissions from the last 24 hours for validation
+                    today_date = datetime.now(ZoneInfo("America/New_York")).strftime('%Y-%m-%d')
+                    if not quiet:
+                        print(f"Validating submissions from {today_date}")
+
+                    hits = clean_efts_hits(await self._async_run_efts_query(
+                        filing_date=(today_date, today_date),
+                        quiet=quiet
+                    ))
+                    
+                    new_hits = self._filter_new_accessions(hits)
+                    if new_hits:
+                        if not quiet:
+                            print(f"Found {len(new_hits)} new submissions via EFTS validation")
+                        if data_callback:
+                            data_callback(new_hits)
+                    last_validation_time = current_time
+                
+                # Interval callback
+                if interval_callback:
+                    interval_callback()
+
+                # Calculate next wake-up time
+                next_times = []
+                if do_polling:
+                    next_times.append(last_polling_time + (polling_interval / 1000))
+                if do_validation:
+                    next_times.append(last_validation_time + (validation_interval / 1000))
+                
+                next_wake_time = min(next_times)
+                current_time = time.time()
+                time_to_sleep = max(0, next_wake_time - current_time)
+                await asyncio.sleep(time_to_sleep)
+                
+        finally:
+            # Clean up the session when done
+            if self.session:
+                await self.session.close()
+                self.session = None
 
     def monitor_submissions(self, data_callback=None, interval_callback=None,
                             polling_interval=1000, quiet=True, start_date=None,

datamule-1.6.1/datamule/utils/construct_submissions_data.py

@@ -0,0 +1,150 @@
+import zipfile
+import json
+import csv
+import os
+import tempfile
+from concurrent.futures import ThreadPoolExecutor
+import threading
+from tqdm import tqdm
+import urllib.request
+
+headers = {'User-Agent': 'John Smith johnsmith@gmail.com'}
+
+def process_file_batch(zip_file, filenames_batch):
+    """Process a batch of files from the zip archive"""
+    batch_filings = []
+    
+    for filename in filenames_batch:
+        if not filename.startswith('CIK'):
+            continue
+            
+        try:
+            # Extract CIK from filename
+            cik = int(filename.split('.')[0].split('-')[0][3:])
+            
+            # Read raw bytes and parse JSON
+            with zip_file.open(filename) as file:
+                raw_data = file.read()
+                submissions_dct = json.loads(raw_data)
+            
+            # Handle different file types
+            if 'submissions' in filename:
+                filings_data = submissions_dct
+            else:
+                filings_data = submissions_dct['filings']['recent']
+            
+            # Extract required data
+            accession_numbers = filings_data['accessionNumber']
+            filing_dates = filings_data['filingDate']
+            forms = filings_data['form']
+            
+            # Create filing records for this file
+            for j in range(len(accession_numbers)):
+                filing_record = {
+                    'accessionNumber': accession_numbers[j],
+                    'filingDate': filing_dates[j],
+                    'form': forms[j],
+                    'cik': cik
+                }
+                batch_filings.append(filing_record)
+                
+        except Exception as e:
+            print(f"Error processing {filename}: {e}")
+            continue
+    
+    return batch_filings
+
+def write_csv_chunk(output_path, filings_data, is_first_write, write_lock):
+    """Thread-safe CSV writing with lock"""
+    with write_lock:
+        if is_first_write:
+            with open(output_path, 'w', newline='') as csvfile:
+                fieldnames = ['accessionNumber', 'filingDate', 'form', 'cik']
+                writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+                writer.writeheader()
+                writer.writerows(filings_data)
+        else:
+            with open(output_path, 'a', newline='') as csvfile:
+                fieldnames = ['accessionNumber', 'filingDate', 'form', 'cik']
+                writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+                writer.writerows(filings_data)
+
+def construct_submissions_data(output_path, submissions_zip_path=None, max_workers=4, batch_size=100):
+    """Creates a list of dicts of every accession number, with filing date, submission type, and ciks"""
+    
+    if submissions_zip_path is None:
+        url = "https://www.sec.gov/Archives/edgar/daily-index/bulkdata/submissions.zip"
+        
+        temp_dir = tempfile.mkdtemp()
+        zip_path = os.path.join(temp_dir, 'submissions.zip')
+        
+        req = urllib.request.Request(url, headers=headers)
+        
+        with urllib.request.urlopen(req) as response:
+            total_size = int(response.headers.get('Content-Length', 0))
+            
+            with open(zip_path, 'wb') as f, tqdm(
+                desc="Downloading",
+                total=total_size,
+                unit='B',
+                unit_scale=True,
+                unit_divisor=1024,
+            ) as pbar:
+                while True:
+                    chunk = response.read(8192)
+                    if not chunk:
+                        break
+                    f.write(chunk)
+                    pbar.update(len(chunk))
+        
+        submissions_zip_path = zip_path
+    
+    # Keep zip file open throughout processing
+    with zipfile.ZipFile(submissions_zip_path, 'r') as zip_file:
+        # Get all CIK filenames
+        all_filenames = [f for f in zip_file.namelist() if f.startswith('CIK')]
+        
+        print(f"Processing {len(all_filenames)} files with {max_workers} workers...")
+        
+        # Create batches of filenames
+        filename_batches = []
+        for i in range(0, len(all_filenames), batch_size):
+            batch = all_filenames[i:i + batch_size]
+            filename_batches.append(batch)
+        
+        # Setup for threading
+        write_lock = threading.Lock()
+        total_filings = 0
+        is_first_write = True
+        
+        # Process batches with thread pool
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            # Submit all batch jobs
+            future_to_batch = {
+                executor.submit(process_file_batch, zip_file, batch): i 
+                for i, batch in enumerate(filename_batches)
+            }
+            
+            # Process results with progress bar
+            with tqdm(total=len(filename_batches), desc="Processing batches", unit="batch") as pbar:
+                for future in future_to_batch:
+                    try:
+                        batch_filings = future.result()
+                        
+                        if batch_filings:  # Only write if we have data
+                            write_csv_chunk(output_path, batch_filings, is_first_write, write_lock)
+                            is_first_write = False
+                            total_filings += len(batch_filings)
+                        
+                        pbar.update(1)
+                        pbar.set_postfix({
+                            'filings': total_filings,
+                            'files': len(filename_batches[future_to_batch[future]])
+                        })
+                        
+                    except Exception as e:
+                        print(f"Error processing batch: {e}")
+                        pbar.update(1)
+    
+    print(f"Complete! Processed {total_filings} total filings")
+    print(f"Data saved to {output_path}")

{datamule-1.6.0 → datamule-1.6.1}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.6.0
+Version: 1.6.1
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.6.0 → datamule-1.6.1}/datamule.egg-info/SOURCES.txt

@@ -68,4 +68,6 @@ datamule/sec/xbrl/xbrlmonitor.py
 datamule/seclibrary/__init__.py
 datamule/seclibrary/bq.py
 datamule/seclibrary/downloader.py
-datamule/seclibrary/query.py
+datamule/seclibrary/query.py
+datamule/utils/__init__.py
+datamule/utils/construct_submissions_data.py

{datamule-1.6.0 → datamule-1.6.1}/setup.py

@@ -32,7 +32,7 @@ if not os.path.exists(file_path):
 setup(
     name="datamule",
     author="John Friedman",
-    version="1.6.0",
+    version="1.6.1",
     description="Work with SEC submissions at scale.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",