datamule 1.5.4__py3-none-any.whl → 1.5.8__py3-none-any.whl

datamule/datamule/sec_connector.py

@@ -0,0 +1,73 @@
+import os
+import json
+import urllib.request
+import websocket
+
+
+class SecConnector:
+    def __init__(self, api_key=None, quiet=False):
+        self.api_key = api_key or os.getenv('DATAMULE_API_KEY')
+        if not self.api_key:
+            raise ValueError("API key not found. Set DATAMULE_API_KEY or provide api_key parameter.")
+        
+        self.quiet = quiet
+        self.auth_url = "https://sec-websocket-auth-worker.jgfriedman99.workers.dev/"
+        self.websocket_url = "ws://3.80.249.191:8080/ws"
+        
+    def _get_jwt_token(self):
+        if not self.quiet:
+            print("Getting JWT token...")
+            
+        url = f"{self.auth_url}?api_key={self.api_key}"
+        
+        req = urllib.request.Request(url)
+        req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
+        req.add_header('Accept', 'application/json')
+        
+        with urllib.request.urlopen(req) as response:
+            data = json.loads(response.read().decode())
+            
+        if not data.get('success'):
+            raise Exception(f"Auth failed: {data.get('error')}")
+            
+        if not self.quiet:
+            print("JWT token obtained")
+            
+        return data['token']
+    
+    def connect(self, data_callback=None):
+        token = self._get_jwt_token()
+        ws_url = f"{self.websocket_url}?token={token}"
+        
+        if not self.quiet:
+            print("Connecting to WebSocket...")
+        
+        def on_open(ws):
+            if not self.quiet:
+                print("WebSocket connected")
+        
+        def on_message(ws, message):
+            response = json.loads(message)
+            data = response.get('data', [])
+            if not self.quiet:
+                print(f"Received data: {len(data)} items")
+            if data_callback:
+                data_callback(data)  # Pass just the data array
+        
+        def on_error(ws, error):
+            if not self.quiet:
+                print(f"WebSocket error: {error}")
+        
+        def on_close(ws, close_status_code, close_msg):
+            if not self.quiet:
+                print("WebSocket closed")
+        
+        ws = websocket.WebSocketApp(
+            ws_url,
+            on_open=on_open,
+            on_message=on_message,
+            on_error=on_error,
+            on_close=on_close
+        )
+        
+        ws.run_forever()

datamule/portfolio.py

@@ -11,6 +11,7 @@ from .seclibrary.downloader import download as seclibrary_download
 from .sec.xbrl.filter_xbrl import filter_xbrl
 from .sec.submissions.monitor import Monitor
 #from .sec.xbrl.xbrlmonitor import XBRLMonitor
+from .datamule.sec_connector import SecConnector
 
 
 class Portfolio:
@@ -175,6 +176,11 @@ class Portfolio:
             validation_interval=validation_interval
         )
 
+    def stream_submissions(self,data_callback=None,quiet=False):
+
+        connector = SecConnector(api_key=self.api_key,quiet=quiet)
+        connector.connect(data_callback=data_callback)
+
         
     def __iter__(self):
         if not self.submissions_loaded:

datamule/sec/submissions/monitor.py

@@ -7,7 +7,7 @@ import asyncio
 from ..utils import headers, PreciseRateLimiter
 from .eftsquery import EFTSQuery
 import aiohttp
-
+from zoneinfo import ZoneInfo 
 
 async def poll_rss(limiter):
     base_url = 'https://www.sec.gov/cgi-bin/browse-edgar?count=100&action=getcurrent&output=rss'
@@ -47,9 +47,22 @@ async def poll_rss(limiter):
     return results
 
 def clean_efts_hits(hits):
-    # clean hits
-    hits = [{'accession': int(hit['_source']['adsh'].replace('-','')), 'filing_date': hit['_source']['file_date'], 'ciks': hit['_source']['ciks'], 'submission_type': hit['_source']['file_type']} for hit in hits]
-    return hits
+    # clean hits and standardize CIKs to string(int)
+    cleaned_hits = []
+    for hit in hits:
+        # Get CIKs from the source, ensure it's a list
+        raw_ciks = hit['_source'].get('ciks', []) 
+        
+        # Standardize each CIK: convert to int (removes leading zeros) then back to string
+        standardized_ciks = [str(int(cik)) for cik in raw_ciks if cik.isdigit()] # Added .isdigit() for robustness
+        
+        cleaned_hits.append({
+            'accession': int(hit['_source']['adsh'].replace('-','')),
+            'filing_date': hit['_source']['file_date'],
+            'ciks': standardized_ciks, # Use the standardized CIKs here
+            'submission_type': hit['_source']['file_type']
+        })
+    return cleaned_hits
 
 class Monitor():
     def __init__(self):
@@ -92,7 +105,7 @@ class Monitor():
 
         # Backfill if start_date is provided
         if start_date is not None:
-            today_date = datetime.now().date().strftime('%Y-%m-%d')
+            today_date = datetime.now(ZoneInfo("America/New_York")).strftime('%Y-%m-%d')
             if not quiet:
                 print(f"Backfilling from {start_date} to {today_date}")
 
@@ -135,7 +148,7 @@ class Monitor():
             # EFTS validation (if enabled)
             if do_validation and (current_time - last_validation_time) >= validation_interval/1000:
                 # Get submissions from the last 24 hours for validation
-                today_date = datetime.now().strftime('%Y-%m-%d')
+                today_date = datetime.now(ZoneInfo("America/New_York")).strftime('%Y-%m-%d')
                 if not quiet:
                     print(f"Validating submissions from {today_date}")
 

{datamule-1.5.4.dist-info → datamule-1.5.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.5.4
+Version: 1.5.8
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
@@ -16,4 +16,5 @@ Requires-Dist: pytz
 Requires-Dist: zstandard
 Requires-Dist: doc2dict
 Requires-Dist: secsgml
+Requires-Dist: websocket-client
 

{datamule-1.5.4.dist-info → datamule-1.5.8.dist-info}/RECORD

@@ -3,10 +3,12 @@ datamule/config.py,sha256=Y--CVv7JcgrjJkMOSLrvm2S8B9ost6RMSkGviP-MKtg,883
 datamule/helper.py,sha256=KqhAmTMdvATEh3I-O4xLcAcrHB9zXQERBuwzue7zyQw,3674
 datamule/index.py,sha256=Rrcna9FJV-Oh_K6O2IuUEIDmtay_7UZ4l4jgKCi7A7I,2079
 datamule/package_updater.py,sha256=Z9zaa_y0Z5cknpRn8oPea3gg4kquFHfpfhduKKCZ6NU,958
-datamule/portfolio.py,sha256=iW54frGfoCQb-6aYfocDqQQPe0gc_22voedv0It_1q0,7517
+datamule/portfolio.py,sha256=Ijx4JFRHSzPoGJRdOTv8c90x79M80LlAXUhUncwYZSo,7755
 datamule/sheet.py,sha256=TvFqK9eAYuVoJ2uWdAlx5EN6vS9lke-aZf7FqtUiDBc,22304
 datamule/submission.py,sha256=6JIi-ayLL-jENVj6Q4IhmrYlAreJI7xBAHP_NYaDB6k,12918
 datamule/data/listed_filer_metadata.csv,sha256=dT9fQ8AC5P1-Udf_UF0ZkdXJ88jNxJb_tuhi5YYL1rc,2426827
+datamule/datamule/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamule/datamule/sec_connector.py,sha256=T3edE7I-d4oHysqj7zYlIOxH3Fuauj9tfw39UdFWvB8,2393
 datamule/document/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/document/document.py,sha256=04Rivdphq0D1HEGIBjtl1LelJr-IyQU1qCMi8yNJajw,14038
 datamule/document/processing.py,sha256=jDCEzBFDSQtq7nQxRScIsbALnFcvMPOkNkMUCa7mFxg,31921
@@ -46,7 +48,7 @@ datamule/sec/infrastructure/submissions_metadata.py,sha256=f1KarzFSryKm0EV8DCDNs
 datamule/sec/submissions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/submissions/downloader.py,sha256=tDWn8bsK9XabQo2pBGYSiqTw37MmqM8rEma8Ph7zp-o,1391
 datamule/sec/submissions/eftsquery.py,sha256=mSZon8rlW8dxma7M49ZW5V02Fn-ENOdt9TNO6elBrhE,27983
-datamule/sec/submissions/monitor.py,sha256=CvpHywnrn4Lwk_3rWRE5K5UNYrdJ9Gyon97Uo0Ocq-4,8985
+datamule/sec/submissions/monitor.py,sha256=ll0nfHzG8FI3bA8zVFrfsfZGnbt5qAD4rRZ4LG2SORY,9567
 datamule/sec/submissions/streamer.py,sha256=EXyWNCD9N6mZmvm9lFSCFodF19zSQ8jfIbWPZNp0K5Y,11253
 datamule/sec/submissions/textsearch.py,sha256=MKDXEz_VI_0ljl73_aw2lx4MVzJW5uDt8KxjvJBwPwM,5794
 datamule/sec/xbrl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -58,7 +60,7 @@ datamule/seclibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 datamule/seclibrary/bq.py,sha256=C8sb_rpXTvchprrFLcbRar4Qi0XWW25tnv1YsHSS5o4,18025
 datamule/seclibrary/downloader.py,sha256=wNRURTGb3eqg12Ltt4578L0WcAm7DmCWg0Rm0Om6Z4U,17959
 datamule/seclibrary/query.py,sha256=qGuursTERRbOGfoDcYcpo4oWkW3PCBW6x1Qf1Puiak4,7352
-datamule-1.5.4.dist-info/METADATA,sha256=jl-zXUtvVrWz4Etn1BW8zsZ2AQ7CaE-zDF18sS0Lf7E,469
-datamule-1.5.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-1.5.4.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-1.5.4.dist-info/RECORD,,
+datamule-1.5.8.dist-info/METADATA,sha256=kfV8_aDjqzk6OZKmJn4GIffpvTW-SYi55O1qSOEnsGQ,501
+datamule-1.5.8.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-1.5.8.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-1.5.8.dist-info/RECORD,,