datamule 1.6.4__py3-none-any.whl → 1.6.6__py3-none-any.whl

datamule/datamule/sec_connector.py

@@ -12,9 +12,8 @@ class SecConnector:
         
         self.quiet = quiet
         self.auth_url = "https://sec-websocket-auth-worker.jgfriedman99.workers.dev/"
-        self.websocket_url = "ws://3.80.249.191:8080/ws"
         
-    def _get_jwt_token(self):
+    def _get_jwt_token_and_ip(self):
         if not self.quiet:
             print("Getting JWT token...")
             
@@ -33,11 +32,12 @@ class SecConnector:
         if not self.quiet:
             print("JWT token obtained")
             
-        return data['token']
+        return data['token'], data['websocket_ip']
     
     def connect(self, data_callback=None):
-        token = self._get_jwt_token()
-        ws_url = f"{self.websocket_url}?token={token}"
+        token,websocket_ip = self._get_jwt_token_and_ip()
+        ws_url = f"ws://{websocket_ip}/ws?token={token}"
+        print(ws_url)
         
         if not self.quiet:
             print("Connecting to WebSocket...")

datamule/sec/submissions/monitor.py

@@ -65,6 +65,7 @@ def clean_efts_hits(hits):
 class Monitor():
     def __init__(self):
         self.accessions = deque(maxlen=50000)
+        self.accessions_set = set()
         self.ratelimiters = {'sec.gov': PreciseRateLimiter(rate=5)}
         self.efts_query = EFTSQuery(quiet=True)
         self.efts_query.limiter = self.ratelimiters['sec.gov']
@@ -250,7 +251,14 @@ class Monitor():
         new_items = []
         for item in items:
             accession = item['accession']
-            if accession not in self.accessions:
+            if accession not in self.accessions_set:  # O(1) lookup instead of O(n)
+                # Check if deque is at capacity and will evict an item
+                if len(self.accessions) == self.accessions.maxlen:
+                    evicted_accession = self.accessions[0]  # Get leftmost item that will be evicted
+                    self.accessions_set.discard(evicted_accession)  # Remove from set
+                
+                # Add to both structures
                 self.accessions.append(accession)
+                self.accessions_set.add(accession)
                 new_items.append(item)
         return new_items

{datamule-1.6.4.dist-info → datamule-1.6.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.6.4
+Version: 1.6.6
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.6.4.dist-info → datamule-1.6.6.dist-info}/RECORD

@@ -9,7 +9,7 @@ datamule/sheet.py,sha256=TvFqK9eAYuVoJ2uWdAlx5EN6vS9lke-aZf7FqtUiDBc,22304
 datamule/submission.py,sha256=ooLsesZ5HkgSWyEFID4u08CobTxdo35eAUHSCB6fw2k,10332
 datamule/data/listed_filer_metadata.csv,sha256=dT9fQ8AC5P1-Udf_UF0ZkdXJ88jNxJb_tuhi5YYL1rc,2426827
 datamule/datamule/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/datamule/sec_connector.py,sha256=T3edE7I-d4oHysqj7zYlIOxH3Fuauj9tfw39UdFWvB8,2393
+datamule/datamule/sec_connector.py,sha256=sr80z_43xXJe3hS8Wfir8LOsNgAAuFIXsFq7xeixc-I,2409
 datamule/document/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/document/document.py,sha256=YGo-Iz_sBXekUeKEAoNJV0BiLDtSOgD9OXFo2FocYq8,14439
 datamule/document/processing.py,sha256=jDCEzBFDSQtq7nQxRScIsbALnFcvMPOkNkMUCa7mFxg,31921
@@ -52,7 +52,7 @@ datamule/sec/infrastructure/submissions_metadata.py,sha256=f1KarzFSryKm0EV8DCDNs
 datamule/sec/submissions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/submissions/downloader.py,sha256=zGS0oJJI8tVF_GnVpZm20MymdYxnjrEjQioSVggw7Ck,1486
 datamule/sec/submissions/eftsquery.py,sha256=mSZon8rlW8dxma7M49ZW5V02Fn-ENOdt9TNO6elBrhE,27983
-datamule/sec/submissions/monitor.py,sha256=1JUMRYsTqtd31hX3UrUA_aXFUmZN6n-V7h0i1gavNOs,11395
+datamule/sec/submissions/monitor.py,sha256=6mE0NZFdPId69t4V53GwBb9sqtRN7HE54sU3WpU0bnY,11900
 datamule/sec/submissions/streamer.py,sha256=Qydj40CmWB_wsPv2dibefRohmCokegG2pR7iZ9C3xLQ,11584
 datamule/sec/submissions/textsearch.py,sha256=MKDXEz_VI_0ljl73_aw2lx4MVzJW5uDt8KxjvJBwPwM,5794
 datamule/sec/xbrl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -66,7 +66,7 @@ datamule/seclibrary/downloader.py,sha256=3jEy67oiEg8BF20KcKCx2KC0UjHzhiepdu29TOa
 datamule/seclibrary/query.py,sha256=qGuursTERRbOGfoDcYcpo4oWkW3PCBW6x1Qf1Puiak4,7352
 datamule/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/utils/construct_submissions_data.py,sha256=aX7ZaAp3zXHLcv4TFk_rGwjb8r7yNDQDFVg4nPf60kM,5934
-datamule-1.6.4.dist-info/METADATA,sha256=4v85X90MyUpvQC37rMyiEA_3UA-2scIHz4tTF1xH2e4,524
-datamule-1.6.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-datamule-1.6.4.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-1.6.4.dist-info/RECORD,,
+datamule-1.6.6.dist-info/METADATA,sha256=qpBESqH9BTqVYWawc44WBq5ki5rUF2XO5Gh4h7Olx8o,524
+datamule-1.6.6.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+datamule-1.6.6.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-1.6.6.dist-info/RECORD,,