datamule 1.8.3__tar.gz → 1.8.5__tar.gz

{datamule-1.8.3 → datamule-1.8.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.8.3
+Version: 1.8.5
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.8.3 → datamule-1.8.5}/datamule/datamule/sec_connector.py

@@ -2,7 +2,7 @@ import os
 import json
 import urllib.request
 import websocket
-
+import re
 
 class SecConnector:
     def __init__(self, api_key=None, quiet=False):
@@ -17,11 +17,12 @@ class SecConnector:
         if not self.quiet:
             print("Getting JWT token...")
             
-        url = f"{self.auth_url}?api_key={self.api_key}"
+        url = self.auth_url
         
-        req = urllib.request.Request(url)
-        req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
+        # Send API key in Authorization header instead of POST body
+        req = urllib.request.Request(url, method='GET')
         req.add_header('Accept', 'application/json')
+        req.add_header('Authorization', f'Bearer {self.api_key}')  # API key in header
         
         with urllib.request.urlopen(req) as response:
             data = json.loads(response.read().decode())
@@ -35,8 +36,8 @@ class SecConnector:
         return data['token'], data['websocket_ip']
     
     def connect(self, data_callback=None):
-        token,websocket_ip = self._get_jwt_token_and_ip()
-        ws_url = f"ws://{websocket_ip}/ws?token={token}"
+        token, websocket_ip = self._get_jwt_token_and_ip()
+        ws_url = f"ws://{websocket_ip}/ws"
         
         if not self.quiet:
             print("Connecting to WebSocket...")
@@ -51,22 +52,41 @@ class SecConnector:
             if not self.quiet:
                 print(f"Received data: {len(data)} items")
             if data_callback:
-                data_callback(data)  # Pass just the data array
+                data_callback(data) 
         
         def on_error(ws, error):
             if not self.quiet:
-                print(f"WebSocket error: {error}")
+                sanitized_error = self._sanitize_error_message(str(error))
+                print(f"WebSocket error: {sanitized_error}")
         
         def on_close(ws, close_status_code, close_msg):
             if not self.quiet:
                 print("WebSocket closed")
         
+        # Use Authorization header for WebSocket connection
+        headers = {'Authorization': f'Bearer {token}'}
+        
         ws = websocket.WebSocketApp(
             ws_url,
+            header=headers,
             on_open=on_open,
             on_message=on_message,
             on_error=on_error,
             on_close=on_close
         )
         
-        ws.run_forever()
+        ws.run_forever()
+    
+    def _sanitize_error_message(self, error_msg):
+        sensitive_patterns = [
+            r'Bearer\s+[A-Za-z0-9\-_\.]+',     # Bearer tokens
+            r'api_key[=:]\s*[A-Za-z0-9\-_]+',  # API key patterns
+            r'token[=:]\s*[A-Za-z0-9\-_\.]+',  # Token patterns
+            r'jwt[=:]\s*[A-Za-z0-9\-_\.]+',    # JWT patterns
+        ]
+        
+        sanitized = error_msg
+        for pattern in sensitive_patterns:
+            sanitized = re.sub(pattern, '[REDACTED]', sanitized, flags=re.IGNORECASE)
+        
+        return sanitized

{datamule-1.8.3 → datamule-1.8.5}/datamule/document/document.py

@@ -110,7 +110,6 @@ class Document:
         if type =='inline':
             if self.extension not in ['.htm','.html']:
                 return
-            
             self.xbrl = parse_inline_xbrl(self.content)
         else:
             raise ValueError("Only inline has been implemented so far.")

{datamule-1.8.3 → datamule-1.8.5}/datamule/portfolio.py

@@ -13,11 +13,12 @@ from .seclibrary.downloader import download as seclibrary_download
 from .sec.xbrl.filter_xbrl import filter_xbrl
 from .sec.submissions.monitor import Monitor
 from .portfolio_compression_utils import CompressionManager
-#from .sec.xbrl.xbrlmonitor import XBRLMonitor
 from .datamule.sec_connector import SecConnector
 from secsgml.utils import bytes_to_str, calculate_documents_locations_in_tar
 import json
 import io
+import shutil
+
 
 class Portfolio:
     def __init__(self, path):
@@ -210,7 +211,10 @@ class Portfolio:
             self.accession_numbers = new_accession_numbers
 
     def download_submissions(self, cik=None, ticker=None, submission_type=None, filing_date=None, provider=None,document_type=[],
-                             requests_per_second=5,keep_filtered_metadata=False,standardize_metadata=True,skip_existing=True, **kwargs):
+                             requests_per_second=5,keep_filtered_metadata=False,standardize_metadata=True,skip_existing=True,
+                              accession_numbers=None, **kwargs):
+        
+
         if provider is None:
             config = Config()
             provider = config.get_default_source()
@@ -218,33 +222,35 @@ class Portfolio:
         # Process CIK and metadata filters
         cik = _process_cik_and_metadata_filters(cik, ticker, **kwargs)
 
-        accession_numbers = self.accession_numbers if hasattr(self, 'accession_numbers') else None
+        filtered_accession_numbers = self.accession_numbers if hasattr(self, 'accession_numbers') else None
+
         skip_accession_numbers = []
         if skip_existing:
             skip_accession_numbers = [sub.accession for sub in self]
             
         if provider == 'datamule':
-
             seclibrary_download(
                 output_dir=self.path,
                 cik=cik,
                 api_key=self.api_key,
                 submission_type=submission_type,
                 filing_date=filing_date,
-                accession_numbers=accession_numbers,
+                filtered_accession_numbers=filtered_accession_numbers,
                 keep_document_types=document_type,
                 keep_filtered_metadata=keep_filtered_metadata,
                 standardize_metadata=standardize_metadata,
-                skip_accession_numbers=skip_accession_numbers
+                skip_accession_numbers=skip_accession_numbers,
+                accession_numbers = accession_numbers
             )
         else:
+            # will later add accession_numbers arg in the free update.
             sec_download(
                 output_dir=self.path,
                 cik=cik,
                 submission_type=submission_type,
                 filing_date=filing_date,
                 requests_per_second=requests_per_second, 
-                accession_numbers=accession_numbers,
+                filtered_accession_numbers=filtered_accession_numbers,
                 keep_document_types=document_type,
                 keep_filtered_metadata=keep_filtered_metadata,
                 standardize_metadata=standardize_metadata,
@@ -286,4 +292,11 @@ class Portfolio:
             document_types = [document_types]
             
         for submission in self.submissions:
-            yield from submission.document_type(document_types)
+            yield from submission.document_type(document_types)
+
+    def delete(self):
+        self._close_batch_handles()
+        shutil.rmtree(self.path)
+
+        # reinit
+        self.__dict__.update(Portfolio(self.path).__dict__)

{datamule-1.8.3 → datamule-1.8.5}/datamule/sec/submissions/downloader.py

@@ -4,7 +4,7 @@ from secsgml import write_sgml_file_to_tar
 from tqdm import tqdm
 
 def download(cik=None, submission_type=None, filing_date=None, location=None, name=None, 
-             requests_per_second=5, output_dir="filings", accession_numbers=None, 
+             requests_per_second=5, output_dir="filings", filtered_accession_numbers=None, 
              quiet=False, keep_document_types=[],keep_filtered_metadata=False,standardize_metadata=True,
              skip_accession_numbers=[]):
     # Make sure output directory exists
@@ -29,7 +29,7 @@ def download(cik=None, submission_type=None, filing_date=None, location=None, na
         location=location,
         requests_per_second=requests_per_second,
         document_callback=callback_wrapper,
-        accession_numbers=accession_numbers,
+        filtered_accession_numbers=filtered_accession_numbers,
         skip_accession_numbers=skip_accession_numbers,
         quiet=quiet
     )

{datamule-1.8.3 → datamule-1.8.5}/datamule/sec/submissions/streamer.py

@@ -222,7 +222,7 @@ class Streamer(EFTSQuery):
         return results
 
 def stream(cik=None, submission_type=None, filing_date=None, location=None, 
-           requests_per_second=5.0, document_callback=None, accession_numbers=None,skip_accession_numbers=[],
+           requests_per_second=5.0, document_callback=None, filtered_accession_numbers=None,skip_accession_numbers=[],
            quiet=False, name=None):
     """
     Stream EFTS results and download documents into memory.
@@ -253,14 +253,14 @@ def stream(cik=None, submission_type=None, filing_date=None, location=None,
     """
     
     # Check if acc no is empty list
-    if accession_numbers == []:
+    if filtered_accession_numbers == []:
         raise ValueError("Applied filter resulted in empty accession numbers list")
     
     async def run_stream():
         streamer = Streamer(
             requests_per_second=requests_per_second, 
             document_callback=document_callback,
-            accession_numbers=accession_numbers,
+            accession_numbers=filtered_accession_numbers,
             skip_accession_numbers=skip_accession_numbers,
             quiet=quiet
         )

{datamule-1.8.3 → datamule-1.8.5}/datamule/sec/utils.py

@@ -1,7 +1,7 @@
 import asyncio
 import time
 from collections import deque
-
+import os
 
 class RetryException(Exception):
     def __init__(self, url, retry_after=601): # SEC Rate limit is typically 10 minutes.
@@ -61,4 +61,9 @@ class RateMonitor:
         
         return round(requests_per_second, 1), round(mb_per_second, 2)
 
-headers = {'User-Agent': 'John Smith johnsmith@gmail.com'}
+
+user_agent = os.environ.get('DATAMULE_SEC_USER_AGENT')
+if user_agent is None:
+    user_agent = 'John Smith johnsmith@gmail.com'
+
+headers = {'User-Agent': user_agent}

{datamule-1.8.3 → datamule-1.8.5}/datamule/seclibrary/downloader.py

@@ -18,6 +18,9 @@ from os import cpu_count
 from secsgml import parse_sgml_content_into_memory
 from secsgml.utils import bytes_to_str
 from .datamule_lookup import datamule_lookup
+from ..utils.format_accession import format_accession
+
+# could be cleaned up
 
 # Set up logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -268,7 +271,7 @@ class Downloader:
         tar_manager = self.TarManager(output_dir, num_tar_files, max_batch_size)
         
         try:
-            with tqdm(total=len(urls), desc="Processing files") as pbar:
+            with tqdm(total=len(urls), desc="Downloading files") as pbar:
                 semaphore = asyncio.Semaphore(self.MAX_CONCURRENT_DOWNLOADS)
                 decompression_pool = ThreadPoolExecutor(max_workers=self.MAX_DECOMPRESSION_WORKERS)
 
@@ -296,29 +299,35 @@ class Downloader:
         finally:
             tar_manager.close_all()
 
-    def download(self, submission_type=None, cik=None, filing_date=None, output_dir="downloads", accession_numbers=None, keep_document_types=[], keep_filtered_metadata=False, standardize_metadata=True,
-                 skip_accession_numbers=[], max_batch_size=1024*1024*1024):
+    def download(self, submission_type=None, cik=None, filing_date=None, output_dir="downloads", filtered_accession_numbers=None, keep_document_types=[], keep_filtered_metadata=False, standardize_metadata=True,
+                 skip_accession_numbers=[], max_batch_size=1024*1024*1024,accession_numbers=None):
         if self.api_key is None:
             raise ValueError("No API key found. Please set DATAMULE_API_KEY environment variable or provide api_key in constructor")
 
         logger.debug("Querying SEC filings...")
 
-        filings = datamule_lookup(cik=cik, submission_type=submission_type, filing_date=filing_date, 
-                   columns=['accessionNumber'], distinct=True, page_size=25000, quiet=False)
+        if not accession_numbers:
+            filings = datamule_lookup(cik=cik, submission_type=submission_type, filing_date=filing_date, 
+                    columns=['accessionNumber'], distinct=True, page_size=25000, quiet=False,api_key=self.api_key)
 
-        if accession_numbers:
-            accession_numbers = [str(int(item.replace('-',''))) for item in accession_numbers]
-            filings = [filing for filing in filings if filing['accessionNumber'] in accession_numbers]
-        
-        if skip_accession_numbers:
-            skip_accession_numbers = [int(item.replace('-','')) for item in skip_accession_numbers]
-            filings = [filing for filing in filings if filing['accessionNumber'] not in skip_accession_numbers]
+            if filtered_accession_numbers:
+                filtered_accession_numbers = [str(int(item.replace('-',''))) for item in filtered_accession_numbers]
+                filings = [filing for filing in filings if filing['accessionNumber'] in filtered_accession_numbers]
+            
+            if skip_accession_numbers:
+                skip_accession_numbers = [int(item.replace('-','')) for item in skip_accession_numbers]
+                filings = [filing for filing in filings if filing['accessionNumber'] not in skip_accession_numbers]
 
-        logger.debug(f"Generating URLs for {len(filings)} filings...")
-        urls = []
-        for item in filings:
-            url = f"{self.BASE_URL}{str(item['accessionNumber']).zfill(18)}.sgml"
-            urls.append(url)
+            logger.debug(f"Generating URLs for {len(filings)} filings...")
+            urls = []
+            for item in filings:
+                url = f"{self.BASE_URL}{str(item['accessionNumber']).zfill(18)}.sgml"
+                urls.append(url)
+        else:
+            urls = []
+            for accession in accession_numbers:
+                url = f"{self.BASE_URL}{format_accession(accession,'no-dash').zfill(18)}.sgml"
+                urls.append(url)
         
         if not urls:
             logger.warning("No submissions found matching the criteria")
@@ -381,12 +390,12 @@ class Downloader:
         logger.debug(f"Processing speed: {len(urls)/elapsed_time:.2f} files/second")
 
 
-def download(submission_type=None, cik=None, filing_date=None, api_key=None, output_dir="downloads", accession_numbers=None, keep_document_types=[],keep_filtered_metadata=False,standardize_metadata=True,
-             skip_accession_numbers=[], max_batch_size=1024*1024*1024):
+def download(submission_type=None, cik=None, filing_date=None, api_key=None, output_dir="downloads", filtered_accession_numbers=None, keep_document_types=[],keep_filtered_metadata=False,standardize_metadata=True,
+             skip_accession_numbers=[], max_batch_size=1024*1024*1024,accession_numbers=None):
     
-    if accession_numbers:
-        accession_numbers = [int(str(x).replace('-', '')) for x in accession_numbers]
-    elif accession_numbers == []:
+    if filtered_accession_numbers:
+        filtered_accession_numbers = [int(str(x).replace('-', '')) for x in filtered_accession_numbers]
+    elif filtered_accession_numbers == []:
         raise ValueError("Applied filter resulted in empty accession numbers list")
     downloader = Downloader(api_key=api_key)
     downloader.download(
@@ -394,10 +403,11 @@ def download(submission_type=None, cik=None, filing_date=None, api_key=None, out
         cik=cik,
         filing_date=filing_date,
         output_dir=output_dir,
-        accession_numbers=accession_numbers,
+        filtered_accession_numbers=filtered_accession_numbers,
         keep_document_types=keep_document_types,
         keep_filtered_metadata=keep_filtered_metadata,
         standardize_metadata=standardize_metadata,
         skip_accession_numbers=skip_accession_numbers,
-        max_batch_size=max_batch_size
+        max_batch_size=max_batch_size,
+        accession_numbers=accession_numbers
     )

{datamule-1.8.3 → datamule-1.8.5}/datamule/submission.py

@@ -238,4 +238,6 @@ class Submission:
 
         for idx, doc in enumerate(self.metadata.content['documents']):
             if doc['type'] in document_types:
-                yield self._load_document_by_index(idx)
+                yield self._load_document_by_index(idx)
+
+

{datamule-1.8.3 → datamule-1.8.5}/datamule/utils/construct_submissions_data.py

@@ -7,10 +7,9 @@ from concurrent.futures import ThreadPoolExecutor
 import threading
 from tqdm import tqdm
 import urllib.request
+from ..sec.utils import headers
 
-headers = {'User-Agent': 'John Smith johnsmith@gmail.com'}
-
-def process_file_batch(zip_file, filenames_batch):
+def process_file_batch(zip_file, filenames_batch, columns, mapping):
     """Process a batch of files from the zip archive"""
     batch_filings = []
     
@@ -33,19 +32,17 @@ def process_file_batch(zip_file, filenames_batch):
             else:
                 filings_data = submissions_dct['filings']['recent']
             
-            # Extract required data
-            accession_numbers = filings_data['accessionNumber']
-            filing_dates = filings_data['filingDate']
-            forms = filings_data['form']
-            
+            # Extract required data using mapping
+            lst_lst = [filings_data[column] for column in columns]
+
             # Create filing records for this file
-            for j in range(len(accession_numbers)):
-                filing_record = {
-                    'accessionNumber': int(accession_numbers[j].replace('-','')),
-                    'filingDate': filing_dates[j],
-                    'submissionType': forms[j],
-                    'cik': cik
-                }
+            for j in range(len(filings_data['accessionNumber'])):
+                filing_record = {'cik': cik}
+
+                for i, column in enumerate(columns):
+                    mapped_key = mapping.get(column, column)
+                    filing_record[mapped_key] = lst_lst[i][j]
+
                 batch_filings.append(filing_record)
                 
         except Exception as e:
@@ -54,24 +51,26 @@ def process_file_batch(zip_file, filenames_batch):
     
     return batch_filings
 
-def write_csv_chunk(output_path, filings_data, is_first_write, write_lock):
+def write_csv_chunk(output_path, filings_data, is_first_write, write_lock, fieldnames):
     """Thread-safe CSV writing with lock"""
     with write_lock:
         if is_first_write:
             with open(output_path, 'w', newline='') as csvfile:
-                fieldnames = ['accessionNumber', 'filingDate', 'submissionType', 'cik']
                 writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
                 writer.writeheader()
                 writer.writerows(filings_data)
         else:
             with open(output_path, 'a', newline='') as csvfile:
-                fieldnames = ['accessionNumber', 'filingDate', 'submissionType', 'cik']
                 writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
                 writer.writerows(filings_data)
 
-def construct_submissions_data(output_path, submissions_zip_path=None, max_workers=4, batch_size=100):
+def construct_submissions_data(output_path, submissions_zip_path=None, max_workers=4, batch_size=100,
+                               columns = ['accessionNumber', 'filingDate', 'form'], mapping = {'form': 'submissionType'}):
     """Creates a list of dicts of every accession number, with filing date, submission type, and ciks"""
-    
+
+    # declare fieldnames
+    fieldnames = ['cik'] + [mapping.get(col, col) for col in columns]
+
     if submissions_zip_path is None:
         url = "https://www.sec.gov/Archives/edgar/daily-index/bulkdata/submissions.zip"
         
@@ -121,7 +120,7 @@ def construct_submissions_data(output_path, submissions_zip_path=None, max_worke
         with ThreadPoolExecutor(max_workers=max_workers) as executor:
             # Submit all batch jobs
             future_to_batch = {
-                executor.submit(process_file_batch, zip_file, batch): i 
+                executor.submit(process_file_batch, zip_file, batch, columns, mapping): i 
                 for i, batch in enumerate(filename_batches)
             }
             
@@ -132,7 +131,7 @@ def construct_submissions_data(output_path, submissions_zip_path=None, max_worke
                         batch_filings = future.result()
                         
                         if batch_filings:  # Only write if we have data
-                            write_csv_chunk(output_path, batch_filings, is_first_write, write_lock)
+                            write_csv_chunk(output_path, batch_filings, is_first_write, write_lock, fieldnames)
                             is_first_write = False
                             total_filings += len(batch_filings)
                         

{datamule-1.8.3 → datamule-1.8.5}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.8.3
+Version: 1.8.5
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.8.3 → datamule-1.8.5}/setup.py

@@ -32,7 +32,7 @@ if not os.path.exists(file_path):
 setup(
     name="datamule",
     author="John Friedman",
-    version="1.8.3",
+    version="1.8.5",
     description="Work with SEC submissions at scale.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",