datamule 1.5.4__tar.gz → 1.5.8__tar.gz

{datamule-1.5.4 → datamule-1.5.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.5.4
+Version: 1.5.8
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
@@ -16,3 +16,4 @@ Requires-Dist: pytz
 Requires-Dist: zstandard
 Requires-Dist: doc2dict
 Requires-Dist: secsgml
+Requires-Dist: websocket-client

datamule-1.5.8/datamule/datamule/sec_connector.py

@@ -0,0 +1,73 @@
+import os
+import json
+import urllib.request
+import websocket
+
+
+class SecConnector:
+    def __init__(self, api_key=None, quiet=False):
+        self.api_key = api_key or os.getenv('DATAMULE_API_KEY')
+        if not self.api_key:
+            raise ValueError("API key not found. Set DATAMULE_API_KEY or provide api_key parameter.")
+        
+        self.quiet = quiet
+        self.auth_url = "https://sec-websocket-auth-worker.jgfriedman99.workers.dev/"
+        self.websocket_url = "ws://3.80.249.191:8080/ws"
+        
+    def _get_jwt_token(self):
+        if not self.quiet:
+            print("Getting JWT token...")
+            
+        url = f"{self.auth_url}?api_key={self.api_key}"
+        
+        req = urllib.request.Request(url)
+        req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
+        req.add_header('Accept', 'application/json')
+        
+        with urllib.request.urlopen(req) as response:
+            data = json.loads(response.read().decode())
+            
+        if not data.get('success'):
+            raise Exception(f"Auth failed: {data.get('error')}")
+            
+        if not self.quiet:
+            print("JWT token obtained")
+            
+        return data['token']
+    
+    def connect(self, data_callback=None):
+        token = self._get_jwt_token()
+        ws_url = f"{self.websocket_url}?token={token}"
+        
+        if not self.quiet:
+            print("Connecting to WebSocket...")
+        
+        def on_open(ws):
+            if not self.quiet:
+                print("WebSocket connected")
+        
+        def on_message(ws, message):
+            response = json.loads(message)
+            data = response.get('data', [])
+            if not self.quiet:
+                print(f"Received data: {len(data)} items")
+            if data_callback:
+                data_callback(data)  # Pass just the data array
+        
+        def on_error(ws, error):
+            if not self.quiet:
+                print(f"WebSocket error: {error}")
+        
+        def on_close(ws, close_status_code, close_msg):
+            if not self.quiet:
+                print("WebSocket closed")
+        
+        ws = websocket.WebSocketApp(
+            ws_url,
+            on_open=on_open,
+            on_message=on_message,
+            on_error=on_error,
+            on_close=on_close
+        )
+        
+        ws.run_forever()

{datamule-1.5.4 → datamule-1.5.8}/datamule/portfolio.py

@@ -11,6 +11,7 @@ from .seclibrary.downloader import download as seclibrary_download
 from .sec.xbrl.filter_xbrl import filter_xbrl
 from .sec.submissions.monitor import Monitor
 #from .sec.xbrl.xbrlmonitor import XBRLMonitor
+from .datamule.sec_connector import SecConnector
 
 
 class Portfolio:
@@ -175,6 +176,11 @@ class Portfolio:
             validation_interval=validation_interval
         )
 
+    def stream_submissions(self,data_callback=None,quiet=False):
+
+        connector = SecConnector(api_key=self.api_key,quiet=quiet)
+        connector.connect(data_callback=data_callback)
+
         
     def __iter__(self):
         if not self.submissions_loaded:

{datamule-1.5.4 → datamule-1.5.8}/datamule/sec/submissions/monitor.py

@@ -7,7 +7,7 @@ import asyncio
 from ..utils import headers, PreciseRateLimiter
 from .eftsquery import EFTSQuery
 import aiohttp
-
+from zoneinfo import ZoneInfo 
 
 async def poll_rss(limiter):
     base_url = 'https://www.sec.gov/cgi-bin/browse-edgar?count=100&action=getcurrent&output=rss'
@@ -47,9 +47,22 @@ async def poll_rss(limiter):
     return results
 
 def clean_efts_hits(hits):
-    # clean hits
-    hits = [{'accession': int(hit['_source']['adsh'].replace('-','')), 'filing_date': hit['_source']['file_date'], 'ciks': hit['_source']['ciks'], 'submission_type': hit['_source']['file_type']} for hit in hits]
-    return hits
+    # clean hits and standardize CIKs to string(int)
+    cleaned_hits = []
+    for hit in hits:
+        # Get CIKs from the source, ensure it's a list
+        raw_ciks = hit['_source'].get('ciks', []) 
+        
+        # Standardize each CIK: convert to int (removes leading zeros) then back to string
+        standardized_ciks = [str(int(cik)) for cik in raw_ciks if cik.isdigit()] # Added .isdigit() for robustness
+        
+        cleaned_hits.append({
+            'accession': int(hit['_source']['adsh'].replace('-','')),
+            'filing_date': hit['_source']['file_date'],
+            'ciks': standardized_ciks, # Use the standardized CIKs here
+            'submission_type': hit['_source']['file_type']
+        })
+    return cleaned_hits
 
 class Monitor():
     def __init__(self):
@@ -92,7 +105,7 @@ class Monitor():
 
         # Backfill if start_date is provided
         if start_date is not None:
-            today_date = datetime.now().date().strftime('%Y-%m-%d')
+            today_date = datetime.now(ZoneInfo("America/New_York")).strftime('%Y-%m-%d')
             if not quiet:
                 print(f"Backfilling from {start_date} to {today_date}")
 
@@ -135,7 +148,7 @@ class Monitor():
             # EFTS validation (if enabled)
             if do_validation and (current_time - last_validation_time) >= validation_interval/1000:
                 # Get submissions from the last 24 hours for validation
-                today_date = datetime.now().strftime('%Y-%m-%d')
+                today_date = datetime.now(ZoneInfo("America/New_York")).strftime('%Y-%m-%d')
                 if not quiet:
                     print(f"Validating submissions from {today_date}")
 

{datamule-1.5.4 → datamule-1.5.8}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.5.4
+Version: 1.5.8
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
@@ -16,3 +16,4 @@ Requires-Dist: pytz
 Requires-Dist: zstandard
 Requires-Dist: doc2dict
 Requires-Dist: secsgml
+Requires-Dist: websocket-client

{datamule-1.5.4 → datamule-1.5.8}/datamule.egg-info/SOURCES.txt

@@ -13,6 +13,8 @@ datamule.egg-info/dependency_links.txt
 datamule.egg-info/requires.txt
 datamule.egg-info/top_level.txt
 datamule/data/listed_filer_metadata.csv
+datamule/datamule/__init__.py
+datamule/datamule/sec_connector.py
 datamule/document/__init__.py
 datamule/document/document.py
 datamule/document/processing.py

{datamule-1.5.4 → datamule-1.5.8}/datamule.egg-info/requires.txt

@@ -10,3 +10,4 @@ pytz
 zstandard
 doc2dict
 secsgml
+websocket-client

{datamule-1.5.4 → datamule-1.5.8}/setup.py

@@ -32,7 +32,7 @@ if not os.path.exists(file_path):
 setup(
     name="datamule",
     author="John Friedman",
-    version="1.5.4",
+    version="1.5.8",
     description="Work with SEC submissions at scale.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",
@@ -49,6 +49,7 @@ setup(
         'zstandard',
         'doc2dict',
         'secsgml',
+        'websocket-client',
     ],
     # Include the data directory in the package
     package_data={