datamule 1.5.5__tar.gz → 1.5.9__tar.gz

{datamule-1.5.5 → datamule-1.5.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.5.5
+Version: 1.5.9
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
@@ -16,3 +16,4 @@ Requires-Dist: pytz
 Requires-Dist: zstandard
 Requires-Dist: doc2dict
 Requires-Dist: secsgml
+Requires-Dist: websocket-client

datamule-1.5.9/datamule/datamule/sec_connector.py

@@ -0,0 +1,73 @@
+import os
+import json
+import urllib.request
+import websocket
+
+
+class SecConnector:
+    def __init__(self, api_key=None, quiet=False):
+        self.api_key = api_key or os.getenv('DATAMULE_API_KEY')
+        if not self.api_key:
+            raise ValueError("API key not found. Set DATAMULE_API_KEY or provide api_key parameter.")
+        
+        self.quiet = quiet
+        self.auth_url = "https://sec-websocket-auth-worker.jgfriedman99.workers.dev/"
+        self.websocket_url = "ws://3.80.249.191:8080/ws"
+        
+    def _get_jwt_token(self):
+        if not self.quiet:
+            print("Getting JWT token...")
+            
+        url = f"{self.auth_url}?api_key={self.api_key}"
+        
+        req = urllib.request.Request(url)
+        req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
+        req.add_header('Accept', 'application/json')
+        
+        with urllib.request.urlopen(req) as response:
+            data = json.loads(response.read().decode())
+            
+        if not data.get('success'):
+            raise Exception(f"Auth failed: {data.get('error')}")
+            
+        if not self.quiet:
+            print("JWT token obtained")
+            
+        return data['token']
+    
+    def connect(self, data_callback=None):
+        token = self._get_jwt_token()
+        ws_url = f"{self.websocket_url}?token={token}"
+        
+        if not self.quiet:
+            print("Connecting to WebSocket...")
+        
+        def on_open(ws):
+            if not self.quiet:
+                print("WebSocket connected")
+        
+        def on_message(ws, message):
+            response = json.loads(message)
+            data = response.get('data', [])
+            if not self.quiet:
+                print(f"Received data: {len(data)} items")
+            if data_callback:
+                data_callback(data)  # Pass just the data array
+        
+        def on_error(ws, error):
+            if not self.quiet:
+                print(f"WebSocket error: {error}")
+        
+        def on_close(ws, close_status_code, close_msg):
+            if not self.quiet:
+                print("WebSocket closed")
+        
+        ws = websocket.WebSocketApp(
+            ws_url,
+            on_open=on_open,
+            on_message=on_message,
+            on_error=on_error,
+            on_close=on_close
+        )
+        
+        ws.run_forever()

{datamule-1.5.5 → datamule-1.5.9}/datamule/portfolio.py

@@ -11,6 +11,7 @@ from .seclibrary.downloader import download as seclibrary_download
 from .sec.xbrl.filter_xbrl import filter_xbrl
 from .sec.submissions.monitor import Monitor
 #from .sec.xbrl.xbrlmonitor import XBRLMonitor
+from .datamule.sec_connector import SecConnector
 
 
 class Portfolio:
@@ -126,7 +127,7 @@ class Portfolio:
             self.accession_numbers = new_accession_numbers
 
     def download_submissions(self, cik=None, ticker=None, submission_type=None, filing_date=None, provider=None,document_type=[],
-                             requests_per_second=5,keep_filtered_metadata=False,standardize_metadata=True, **kwargs):
+                             requests_per_second=5,keep_filtered_metadata=False,standardize_metadata=True,skip_existing=True, **kwargs):
         if provider is None:
             config = Config()
             provider = config.get_default_source()
@@ -134,6 +135,11 @@ class Portfolio:
         # Process CIK and metadata filters
         cik = _process_cik_and_metadata_filters(cik, ticker, **kwargs)
 
+        accession_numbers = self.accession_numbers if hasattr(self, 'accession_numbers') else None
+        skip_accession_numbers = []
+        if skip_existing:
+            skip_accession_numbers = [sub.accession for sub in self]
+
         if provider == 'datamule':
 
             seclibrary_download(
@@ -142,10 +148,11 @@ class Portfolio:
                 api_key=self.api_key,
                 submission_type=submission_type,
                 filing_date=filing_date,
-                accession_numbers=self.accession_numbers if hasattr(self, 'accession_numbers') else None,
+                accession_numbers=accession_numbers,
                 keep_document_types=document_type,
                 keep_filtered_metadata=keep_filtered_metadata,
                 standardize_metadata=standardize_metadata,
+                skip_accession_numbers=skip_accession_numbers
             )
         else:
             sec_download(
@@ -154,10 +161,11 @@ class Portfolio:
                 submission_type=submission_type,
                 filing_date=filing_date,
                 requests_per_second=requests_per_second, 
-                accession_numbers=self.accession_numbers if hasattr(self, 'accession_numbers') else None,
+                accession_numbers=accession_numbers,
                 keep_document_types=document_type,
                 keep_filtered_metadata=keep_filtered_metadata,
                 standardize_metadata=standardize_metadata,
+                skip_accession_numbers=skip_accession_numbers
             )
 
         self.submissions_loaded = False
@@ -175,6 +183,11 @@ class Portfolio:
             validation_interval=validation_interval
         )
 
+    def stream_submissions(self,data_callback=None,quiet=False):
+
+        connector = SecConnector(api_key=self.api_key,quiet=quiet)
+        connector.connect(data_callback=data_callback)
+
         
     def __iter__(self):
         if not self.submissions_loaded:

{datamule-1.5.5 → datamule-1.5.9}/datamule/sec/submissions/downloader.py

@@ -5,7 +5,8 @@ from tqdm import tqdm
 
 def download(cik=None, submission_type=None, filing_date=None, location=None, name=None, 
              requests_per_second=5, output_dir="filings", accession_numbers=None, 
-             quiet=False, keep_document_types=[],keep_filtered_metadata=False,standardize_metadata=True):
+             quiet=False, keep_document_types=[],keep_filtered_metadata=False,standardize_metadata=True,
+             skip_accession_numbers=[]):
     # Make sure output directory exists
     os.makedirs(output_dir, exist_ok=True)
 
@@ -29,5 +30,6 @@ def download(cik=None, submission_type=None, filing_date=None, location=None, na
         requests_per_second=requests_per_second,
         document_callback=callback_wrapper,
         accession_numbers=accession_numbers,
+        skip_accession_numbers=skip_accession_numbers,
         quiet=quiet
     )

{datamule-1.5.5 → datamule-1.5.9}/datamule/sec/submissions/monitor.py

@@ -47,9 +47,22 @@ async def poll_rss(limiter):
     return results
 
 def clean_efts_hits(hits):
-    # clean hits
-    hits = [{'accession': int(hit['_source']['adsh'].replace('-','')), 'filing_date': hit['_source']['file_date'], 'ciks': hit['_source']['ciks'], 'submission_type': hit['_source']['file_type']} for hit in hits]
-    return hits
+    # clean hits and standardize CIKs to string(int)
+    cleaned_hits = []
+    for hit in hits:
+        # Get CIKs from the source, ensure it's a list
+        raw_ciks = hit['_source'].get('ciks', []) 
+        
+        # Standardize each CIK: convert to int (removes leading zeros) then back to string
+        standardized_ciks = [str(int(cik)) for cik in raw_ciks if cik.isdigit()] # Added .isdigit() for robustness
+        
+        cleaned_hits.append({
+            'accession': int(hit['_source']['adsh'].replace('-','')),
+            'filing_date': hit['_source']['file_date'],
+            'ciks': standardized_ciks, # Use the standardized CIKs here
+            'submission_type': hit['_source']['file_type']
+        })
+    return cleaned_hits
 
 class Monitor():
     def __init__(self):

{datamule-1.5.5 → datamule-1.5.9}/datamule/sec/submissions/streamer.py

@@ -21,7 +21,7 @@ def fix_filing_url(url):
     return url
 
 class Streamer(EFTSQuery):
-    def __init__(self, requests_per_second=5.0, document_callback=None, accession_numbers=None, quiet=False):
+    def __init__(self, requests_per_second=5.0, document_callback=None, accession_numbers=None,skip_accession_numbers=None, quiet=False):
         super().__init__(requests_per_second=requests_per_second, quiet=quiet)
         self.document_callback = document_callback
         self.document_queue = asyncio.Queue()
@@ -32,6 +32,7 @@ class Streamer(EFTSQuery):
         self.documents_processed = 0
         self.total_documents = 0
         self.accession_numbers = accession_numbers
+        self.skip_accession_numbers = skip_accession_numbers
         self.skipped_documents = 0
         
     async def _fetch_worker(self):
@@ -81,6 +82,9 @@ class Streamer(EFTSQuery):
             if self.accession_numbers is not None and accno_w_dash not in self.accession_numbers:
                 return None, None, None
             
+            if self.skip_accession_numbers is not None and accno_w_dash in self.skip_accession_numbers:
+                return None, None, None
+            
             # Construct the URL
             url = f"https://www.sec.gov/Archives/edgar/data/{cik}/{accno_no_dash}/{accno_w_dash}.txt"
             url = fix_filing_url(url)
@@ -218,7 +222,7 @@ class Streamer(EFTSQuery):
         return results
 
 def stream(cik=None, submission_type=None, filing_date=None, location=None, 
-           requests_per_second=5.0, document_callback=None, accession_numbers=None,
+           requests_per_second=5.0, document_callback=None, accession_numbers=None,skip_accession_numbers=[],
            quiet=False, name=None):
     """
     Stream EFTS results and download documents into memory.
@@ -257,6 +261,7 @@ def stream(cik=None, submission_type=None, filing_date=None, location=None,
             requests_per_second=requests_per_second, 
             document_callback=document_callback,
             accession_numbers=accession_numbers,
+            skip_accession_numbers=skip_accession_numbers,
             quiet=quiet
         )
         return await streamer.stream(cik, submission_type, filing_date, location, name)

{datamule-1.5.5 → datamule-1.5.9}/datamule/sec/xbrl/filter_xbrl.py

@@ -5,8 +5,6 @@ from ..utils import headers
 def fetch_frame(taxonomy, concept, unit, period):
     url = f"https://data.sec.gov/api/xbrl/frames/{taxonomy}/{concept}/{unit}/{period}.json"
     response = requests.get(url, headers=headers)
-    print(url)
-    print(response)
     return response.json()
 
 

{datamule-1.5.5 → datamule-1.5.9}/datamule/seclibrary/downloader.py

@@ -14,7 +14,6 @@ from queue import Queue, Empty
 from threading import Thread
 from .query import query
 from os import cpu_count
-from ..submission import Submission
 from secsgml import write_sgml_file_to_tar
 
 
@@ -235,7 +234,8 @@ class Downloader:
             processor.stop_workers()
             decompression_pool.shutdown()
 
-    def download(self, submission_type=None, cik=None, filing_date=None, output_dir="downloads", accession_numbers=None, keep_document_types=[], keep_filtered_metadata=False, standardize_metadata=True):
+    def download(self, submission_type=None, cik=None, filing_date=None, output_dir="downloads", accession_numbers=None, keep_document_types=[], keep_filtered_metadata=False, standardize_metadata=True,
+                 skip_accession_numbers=[]):
         """
         Query SEC filings and download/process them.
         
@@ -259,10 +259,18 @@ class Downloader:
             filing_date=filing_date,
             api_key=self.api_key
         )
+
+
         # After querying but before generating URLs
         if accession_numbers:
+            accession_numbers = [str(int(item.replace('-',''))) for item in accession_numbers]
             filings = [filing for filing in filings if filing['accession_number'] in accession_numbers]
         
+
+        if skip_accession_numbers:
+            skip_accession_numbers = [int(item.replace('-','')) for item in skip_accession_numbers]
+            filings = [filing for filing in filings if filing['accession_number'] not in skip_accession_numbers]
+
         # Generate URLs from the query results
         
         print(f"Generating URLs for {len(filings)} filings...")
@@ -355,7 +363,8 @@ class Downloader:
         print(f"Processing speed: {len(urls)/elapsed_time:.2f} files/second")
 
 
-def download(submission_type=None, cik=None, filing_date=None, api_key=None, output_dir="downloads", accession_numbers=None, keep_document_types=[],keep_filtered_metadata=False,standardize_metadata=True):
+def download(submission_type=None, cik=None, filing_date=None, api_key=None, output_dir="downloads", accession_numbers=None, keep_document_types=[],keep_filtered_metadata=False,standardize_metadata=True,
+             skip_accession_numbers=[]):
     """
     Query SEC filings and download/process them.
     
@@ -383,28 +392,6 @@ def download(submission_type=None, cik=None, filing_date=None, api_key=None, out
         accession_numbers=accession_numbers,
         keep_document_types=keep_document_types,
         keep_filtered_metadata=keep_filtered_metadata,
-        standardize_metadata=standardize_metadata
+        standardize_metadata=standardize_metadata,
+        skip_accession_numbers=skip_accession_numbers
     )
-
-def download_files_using_filename(filenames, api_key=None, output_dir="downloads", keep_document_types=[], keep_filtered_metadata=False, standardize_metadata=True):
-    """
-    Download and process SEC filings using specific filenames.
-    
-    Parameters:
-    - filenames: List of specific filenames to download (e.g., ['000091205797006494.sgml', '000100704297000007.sgml.zst'])
-    - api_key: API key for datamule service (optional if DATAMULE_API_KEY env var is set)
-    - output_dir: Directory to save downloaded files
-    - keep_document_types: List of document types to keep (e.g., ['10-K', 'EX-10.1'])
-    - keep_filtered_metadata: Whether to keep metadata for filtered documents
-    - standardize_metadata: Whether to standardize metadata format
-    """
-    downloader = Downloader(api_key=api_key)
-    downloader.QUEUE_SIZE = 1
-    downloader.MAX_CONCURRENT_DOWNLOADS = 1
-    downloader.download_files_using_filename(
-        filenames=filenames,
-        output_dir=output_dir,
-        keep_document_types=keep_document_types,
-        keep_filtered_metadata=keep_filtered_metadata,
-        standardize_metadata=standardize_metadata
-    )

{datamule-1.5.5 → datamule-1.5.9}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.5.5
+Version: 1.5.9
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
@@ -16,3 +16,4 @@ Requires-Dist: pytz
 Requires-Dist: zstandard
 Requires-Dist: doc2dict
 Requires-Dist: secsgml
+Requires-Dist: websocket-client

{datamule-1.5.5 → datamule-1.5.9}/datamule.egg-info/SOURCES.txt

@@ -13,6 +13,8 @@ datamule.egg-info/dependency_links.txt
 datamule.egg-info/requires.txt
 datamule.egg-info/top_level.txt
 datamule/data/listed_filer_metadata.csv
+datamule/datamule/__init__.py
+datamule/datamule/sec_connector.py
 datamule/document/__init__.py
 datamule/document/document.py
 datamule/document/processing.py

{datamule-1.5.5 → datamule-1.5.9}/datamule.egg-info/requires.txt

@@ -10,3 +10,4 @@ pytz
 zstandard
 doc2dict
 secsgml
+websocket-client

{datamule-1.5.5 → datamule-1.5.9}/setup.py

@@ -32,7 +32,7 @@ if not os.path.exists(file_path):
 setup(
     name="datamule",
     author="John Friedman",
-    version="1.5.5",
+    version="1.5.9",
     description="Work with SEC submissions at scale.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",
@@ -49,6 +49,7 @@ setup(
         'zstandard',
         'doc2dict',
         'secsgml',
+        'websocket-client',
     ],
     # Include the data directory in the package
     package_data={