datamule 1.0.2__py3-none-any.whl → 1.0.3__py3-none-any.whl

datamule/mapping_dicts/txt_mapping_dicts.py

@@ -29,10 +29,12 @@ dict_sgml = {
     }
 }
 
-item_pattern_mapping = r"^\n\n\s*(ITEM|Item)\b"
-part_pattern_mapping = r"^\n\n\s*(PART|Part)\b"
+item_pattern_mapping = r"^\n\n\s*(ITEM|Item)\s+(\d+[a-zA-Z]?|ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN|ELEVEN|TWELVE|THIRTEEN|FOURTEEN|FIFTEEN|SIXTEEN|[0-9]+[a-zA-Z]?)\.?"
+item_pattern_mapping_8k = r"^\n\n\s*(ITEM|Item)\s+(\d+(?:\.\d+)?[a-zA-Z]?|ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN|ELEVEN|TWELVE|THIRTEEN|FOURTEEN|FIFTEEN|SIXTEEN|[0-9]+[a-zA-Z]?)\.?"
+part_pattern_mapping = r"^\n\n\s*(PART|Part)\s+(?:I{1,3}|IV)\.?"
 
 item_pattern_standardization = r"^\s*(?:ITEM|Item)\s+(\d+[a-zA-Z]?|ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN|ELEVEN|TWELVE|THIRTEEN|FOURTEEN|FIFTEEN|SIXTEEN|[0-9]+[a-zA-Z]?)\.?"
+item_pattern_standardization_8k = r"^\s*(?:ITEM|Item)\s+(\d+(?:\.\d+)?[a-zA-Z]?|ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN|ELEVEN|TWELVE|THIRTEEN|FOURTEEN|FIFTEEN|SIXTEEN)\.?"
 part_pattern_standardization =  r"^\s*(?:PART|Part)\s+([IVX]+)"
         
 
@@ -194,7 +196,7 @@ dict_8k["rules"]["mappings"].extend([
             {
                 "type": "hierarchy",
                 "name": "item",
-                "pattern": item_pattern_mapping,
+                "pattern": item_pattern_mapping_8k,
                 "hierarchy": 0
             },
             ])
@@ -204,7 +206,7 @@ dict_8k['transformations'] = [
         "type": "standardize", 
         "match": {
             "type": "item",
-            "text_pattern": item_pattern_standardization
+            "text_pattern": item_pattern_standardization_8k
         },
         "output": {
             "format": "item{}",

datamule/monitor.py

@@ -11,6 +11,15 @@ def _get_current_eastern_date():
     eastern = pytz.timezone('America/New_York')
     return datetime.now(eastern)
 
+def _parse_date(date_str):
+    """Parse YYYY-MM-DD date string to datetime object in Eastern timezone"""
+    try:
+        date = datetime.strptime(date_str, '%Y-%m-%d')
+        eastern = pytz.timezone('America/New_York')
+        return eastern.localize(date)
+    except ValueError as e:
+        raise ValueError(f"Invalid date format. Please use YYYY-MM-DD. Error: {str(e)}")
+
 class PreciseRateLimiter:
     def __init__(self, rate, interval=1.0):
         self.rate = rate  # requests per interval
@@ -67,7 +76,8 @@ class RateMonitor:
 class Monitor:
     def __init__(self):
         self.last_total = 0
-        self.last_date = _get_current_eastern_date()
+        self.last_date = None
+        self.current_monitor_date = None
         self.submissions = []
         self.max_hits = 10000
         self.limiter = PreciseRateLimiter(5)  # 5 requests per second
@@ -91,16 +101,29 @@ class Monitor:
         """Poll API until new submissions are found."""
         while True:
             current_date = _get_current_eastern_date()
-            date_str = current_date.strftime('%Y-%m-%d')
-            timestamp = int(time.time())  # Add this line
             
-            if self.last_date != current_date.strftime('%Y-%m-%d'):
-                print(f"New date: {date_str}")
+            # If we're caught up to current date, use it, otherwise use our tracking date
+            if self.current_monitor_date.date() >= current_date.date():
+                self.current_monitor_date = current_date
+            else:
+                # If we're behind current date and haven't finished current date's processing,
+                # continue with current date
+                if self.last_date == self.current_monitor_date.strftime('%Y-%m-%d'):
+                    pass
+                else:
+                    # Move to next day
+                    self.current_monitor_date += timedelta(days=1)
+                    
+            date_str = self.current_monitor_date.strftime('%Y-%m-%d')
+            timestamp = int(time.time())
+            
+            if self.last_date != date_str:
+                print(f"Processing date: {date_str}")
                 self.last_total = 0
                 self.submissions = []
                 self.last_date = date_str
             
-            poll_url = f"{base_url}&startdt={date_str}&enddt={date_str}&v={timestamp}"  # Modified this line
+            poll_url = f"{base_url}&startdt={date_str}&enddt={date_str}&v={timestamp}"
             if not quiet:
                 print(f"Polling {poll_url}")
             
@@ -109,10 +132,16 @@ class Monitor:
                 if data:
                     current_total = data['hits']['total']['value']
                     if current_total > self.last_total:
-                        print(f"Found {current_total - self.last_total} new submissions")
+                        print(f"Found {current_total - self.last_total} new submissions for {date_str}")
                         self.last_total = current_total
                         return current_total, data, poll_url
                     self.last_total = current_total
+                    
+                    # If we have no hits and we're processing a past date,
+                    # we can move to the next day immediately
+                    if current_total == 0 and self.current_monitor_date.date() < current_date.date():
+                        continue
+                    
             except Exception as e:
                 print(f"Error in poll: {str(e)}")
             
@@ -120,7 +149,6 @@ class Monitor:
 
     async def _retrieve_batch(self, session, poll_url, from_positions, quiet):
         """Retrieve a batch of submissions concurrently."""
-        # The poll_url already contains the timestamp from _poll
         tasks = [
             self._fetch_json(
                 session,
@@ -176,11 +204,17 @@ class Monitor:
         
         return submissions
 
-    async def _monitor(self, callback, form=None, cik=None, ticker=None, poll_interval=1000, quiet=True):
+    async def _monitor(self, callback, form=None, cik=None, ticker=None, start_date=None, poll_interval=1000, quiet=True):
         """Main monitoring loop with parallel processing."""
         if poll_interval < 100:
             raise ValueError("SEC rate limit is 10 requests per second, set poll_interval to 100ms or higher")
 
+        # Set up initial monitoring date
+        if start_date:
+            self.current_monitor_date = _parse_date(start_date)
+        else:
+            self.current_monitor_date = _get_current_eastern_date()
+
         # Handle form parameter
         if form is None:
             form = ['-0']
@@ -233,6 +267,17 @@ class Monitor:
                 
                 await asyncio.sleep(poll_interval / 1000)
 
-    def monitor_submissions(self, callback=None, form=None, cik=None, ticker=None, poll_interval=1000, quiet=True):
-        """Start the monitoring process."""
-        asyncio.run(self._monitor(callback, form, cik, ticker, poll_interval, quiet))
+    def monitor_submissions(self, callback=None, form=None, cik=None, ticker=None, start_date=None, poll_interval=1000, quiet=True):
+        """
+        Start the monitoring process.
+        
+        Parameters:
+            callback (callable, optional): Function to call when new submissions are found
+            form (str or list, optional): Form type(s) to monitor
+            cik (str or list, optional): CIK(s) to monitor
+            ticker (str, optional): Ticker symbol to monitor
+            start_date (str, optional): Start date in YYYY-MM-DD format
+            poll_interval (int, optional): Polling interval in milliseconds
+            quiet (bool, optional): Suppress verbose output
+        """
+        asyncio.run(self._monitor(callback, form, cik, ticker, start_date, poll_interval, quiet))

{datamule-1.0.2.dist-info → datamule-1.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.0.2
+Version: 1.0.3
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.0.2.dist-info → datamule-1.0.3.dist-info}/RECORD

@@ -2,7 +2,7 @@ datamule/__init__.py,sha256=IDVK3i5i5DxLlQJ_71aYkloGNi528JOUx8hU6bDzLXM,1255
 datamule/config.py,sha256=Y--CVv7JcgrjJkMOSLrvm2S8B9ost6RMSkGviP-MKtg,883
 datamule/document.py,sha256=-XT3nkRb1oCOken7n3AUCI8itQNfuKVN86cuD3rys3E,10912
 datamule/helper.py,sha256=tr3AQWus9dHNZFKpLSglWjcb8zmm5qDXjOWACMhvMxQ,4594
-datamule/monitor.py,sha256=AfhGqC_GFTYWemRKgYE85V7rIGMN_pbcpxW6kORQtpw,9273
+datamule/monitor.py,sha256=NgzOB_RCLAZDrowYagPWIM4PGNngd1lA3K9Qplkk3Ys,11325
 datamule/packageupdater.py,sha256=vEGqlTj6FudIeVHBVJltPh2eBDEqMG9HYmnyrRVKeSU,9595
 datamule/portfolio.py,sha256=U_QRNk_CbMmi3nJ0VBIwc9SVEGq6kA8LCZHBj9nOGXs,4032
 datamule/submission.py,sha256=uioIYJbsoe-87nRPyzlo-LZ8Hp7HG7A4KPGSnw86PKY,2790
@@ -17,7 +17,7 @@ datamule/data/sec-glossary.csv,sha256=-cN7GjiadLw5C1sv4zSeCnfeZZDYeSgJl-0ydarMAo
 datamule/data/xbrl_descriptions.csv,sha256=SQ9wUURNqG424rnTiZtopsxV2q-PvU4NMj52LqgDsvg,2621524
 datamule/downloader/downloader.py,sha256=vnMsw0oWqRa84scu6ZcywxbJxsIn38vLV0tybakx3jQ,15217
 datamule/downloader/premiumdownloader.py,sha256=YhGFwkYqjLkdc5ex2YKM-L7nBAPm5MMCdTwVVP0JO78,14314
-datamule/mapping_dicts/txt_mapping_dicts.py,sha256=Eh6qYhseuKjjnxGh0A5blHr7mbq9CigFn6Zv9xcG2zU,5783
+datamule/mapping_dicts/txt_mapping_dicts.py,sha256=DQPrGYbAPQxomRUtt4iiMGrwuF7BHc_LeFBQuYBzU9o,6311
 datamule/mapping_dicts/xml_mapping_dicts.py,sha256=Z22yDVwKYonUfM5foQP00dVDE8EHhhMKp0CLqVKV5OI,438
 datamule/mulebot/__init__.py,sha256=YvZXV6xQ0iP-oGD8rloufjdwJL6D46P3NNr0CY9PQCA,29
 datamule/mulebot/helper.py,sha256=olztOwltfELZ-IERM2bRNLBavD04kfB6ueWTisJAleA,1080
@@ -37,7 +37,7 @@ datamule/mulebot/mulebot_server/static/scripts/suggestions.js,sha256=TCyz8OYuXeI
 datamule/mulebot/mulebot_server/static/scripts/tableArtifacts.js,sha256=UtkUpLvELNI4Ibpb7VstgVA9Tk-8jbkxXhmXsgufFa4,4437
 datamule/mulebot/mulebot_server/static/scripts/utils.js,sha256=oGPMtyT9dvuqHqrfZj33t4vLZiF8UJrMXB1hpPXRNu4,1255
 datamule/mulebot/mulebot_server/templates/chat-minimalist.html,sha256=MsTbgpnLD0JCQiKKP3XeeNJRNsRqKsRa1j_XXW7nBKw,6975
-datamule-1.0.2.dist-info/METADATA,sha256=GF3kWzUe2GxJjJwRyewMfLh8intVCMZuSIngk6o1J4g,732
-datamule-1.0.2.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-datamule-1.0.2.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-1.0.2.dist-info/RECORD,,
+datamule-1.0.3.dist-info/METADATA,sha256=8PZAcyMcoQTNaV21b9N09t8cd4Uw0Kxm6aImKXlSsCo,732
+datamule-1.0.3.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+datamule-1.0.3.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-1.0.3.dist-info/RECORD,,