datamule 1.0.0__py3-none-any.whl → 1.0.3__py3-none-any.whl

datamule/{book.py → book/book.py}

@@ -6,8 +6,26 @@
 # likely new bottleneck will be local parsing() - will be bypassed in future when we have parsed archive
 # wow parsed archive is going to be crazy fast - like every 10k in 1 minute.
 
+# example queries filter by sic = 7372, xbrl query = dei:operatingprofit > 0 in date range 2018-2019
+
+# hmm do we go for sql esq or not.
+# I think we do.
+# i think we remove cik, ticker, sic, etc and just have a query object
+# should be sql esq so users can use it easily w/o learnign new syntax
+
+# WHERE submission_type = '10-K'
+# AND us-gaap:ResearchAndDevelopmentExpense > 0 
+# AND dei:debt_to_equity < 2
+# AND filing_date BETWEEN '2023-01-01' AND '2023-12-31'
+# AND CIK in (123, 456, 789)
+# AND SIC in (123, 456, 789)
+# AND ticker in ('AAPL', 'GOOGL', 'AMZN')
+# AND document_type = 'EX-99.1' # to select attachments
+
+from .eftsquery import EFTSQuery
+
+
 class Book():
-    pass
     def process_submissions(self,cik,ticker,sic,submission_type,document_type,date,
                             xbrl_query={},
                             metadata_callback=None,

datamule/book/eftsquery.py

@@ -0,0 +1,127 @@
+import asyncio
+import aiohttp
+from tqdm import tqdm
+from datetime import datetime
+from urllib.parse import urlencode
+import time
+
+class PreciseRateLimiter:
+    def __init__(self, rate=10, interval=1.0):
+        self.rate = rate  # requests per interval
+        self.interval = interval  # in seconds
+        self.token_time = self.interval / self.rate  # time per token
+        self.last_time = time.time()
+        self.lock = asyncio.Lock()
+
+    async def acquire(self):
+        async with self.lock:
+            now = time.time()
+            wait_time = self.last_time + self.token_time - now
+            if wait_time > 0:
+                await asyncio.sleep(wait_time)
+            self.last_time = time.time()
+            return True
+
+class EFTSQuery:
+    def __init__(self):
+        self.headers = {
+            'User-Agent': 'Your Name yourname@email.com',
+            'Accept-Encoding': 'gzip, deflate',
+            'Host': 'efts.sec.gov'
+        }
+        self.session = None
+        self.limiter = PreciseRateLimiter(10)
+
+    async def __aenter__(self):
+        if not self.session:
+            self.session = aiohttp.ClientSession(headers=self.headers)
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self.session:
+            await self.session.close()
+            self.session = None
+
+    async def _fetch_json(self, url):
+        await self.limiter.acquire()
+        try:
+            async with self.session.get(url) as response:
+                if response.status == 429:
+                    await asyncio.sleep(61)
+                    return await self._fetch_json(url)
+                return await response.json()
+        except Exception as e:
+            print(f"Error fetching {url}: {str(e)}")
+            return None
+
+    async def _get_accession_numbers(self, base_url):
+        data = await self._fetch_json(f"{base_url}&from=0&size=1")
+        if not data or 'hits' not in data:
+            return []
+            
+        total_hits = data['hits']['total']['value']
+        if not total_hits:
+            return []
+
+        accession_numbers = []
+        start = 0
+        page_size = 100
+        batch_size = 10  # Number of concurrent requests
+        
+        with tqdm(total=total_hits) as pbar:
+            while start < total_hits:
+                tasks = []
+                for i in range(batch_size):
+                    if start + i * page_size >= total_hits:
+                        break
+                    url = f"{base_url}&from={start + i * page_size}&size={page_size}"
+                    tasks.append(self._fetch_json(url))
+                
+                if not tasks:
+                    break
+                    
+                results = await asyncio.gather(*tasks)
+                
+                for data in results:
+                    if data and 'hits' in data:
+                        hits = data['hits']['hits']
+                        batch_numbers = [
+                            f"{hit['_source']['ciks'][0]}/{hit['_id'].split(':')[0]}"
+                            for hit in hits
+                        ]
+                        accession_numbers.extend(batch_numbers)
+                        pbar.update(len(hits))
+                
+                start += batch_size * page_size
+
+        return accession_numbers
+
+    def query_efts(self, cik=None, ticker=None, submission_type=None, filing_date=None, search_text=None):
+        async def _download():
+            async with self as downloader:
+                params = {}
+                
+                if cik:
+                    params['ciks'] = str(cik).zfill(10)
+
+                if submission_type:
+                    params['forms'] = ','.join(submission_type) if isinstance(submission_type, list) else submission_type
+
+                if isinstance(filing_date, list):
+                    dates = [(d, d) for d in filing_date]
+                elif isinstance(filing_date, tuple):
+                    dates = [filing_date]
+                else:
+                    date_str = filing_date if filing_date else f"2001-01-01,{datetime.now().strftime('%Y-%m-%d')}"
+                    start, end = date_str.split(',')
+                    dates = [(start, end)]
+                
+                params['startdt'], params['enddt'] = dates[0]
+
+                if search_text:
+                    params['q'] = f'"{search_text}"'
+
+                base_url = f"https://efts.sec.gov/LATEST/search-index?{urlencode(params, doseq=True)}"
+                return await self._get_accession_numbers(base_url)
+
+        return asyncio.run(_download())

datamule/book/xbrl_retriever.py

@@ -0,0 +1,88 @@
+import asyncio
+import aiohttp
+import time
+
+class PreciseRateLimiter:
+    def __init__(self, rate=10, interval=1.0):
+        self.rate = rate
+        self.interval = interval
+        self.token_time = self.interval / self.rate
+        self.last_time = time.time()
+        self.lock = asyncio.Lock()
+
+    async def acquire(self):
+        async with self.lock:
+            now = time.time()
+            wait_time = self.last_time + self.token_time - now
+            if wait_time > 0:
+                await asyncio.sleep(wait_time)
+            self.last_time = time.time()
+            return True
+
+class XBRLRetriever:
+    def __init__(self):
+        self.base_url = "https://data.sec.gov/api/xbrl/frames"
+        self.headers = {
+            'User-Agent': 'Your Name yourname@email.com',
+            'Accept-Encoding': 'gzip, deflate',
+            'Host': 'data.sec.gov'
+        }
+        self.session = None
+        self.limiter = PreciseRateLimiter(10)
+
+    async def __aenter__(self):
+        if not self.session:
+            self.session = aiohttp.ClientSession(headers=self.headers)
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self.session:
+            await self.session.close()
+            self.session = None
+
+    async def _fetch_json(self, url):
+        await self.limiter.acquire()
+        try:
+            async with self.session.get(url) as response:
+                if response.status == 429:
+                    await asyncio.sleep(61)
+                    return await self._fetch_json(url)
+                elif response.status == 200:
+                    return await response.json()
+                else:
+                    print(f"Error {response.status} for URL: {url}")
+                    return None
+        except Exception as e:
+            print(f"Error fetching {url}: {str(e)}")
+            return None
+
+    def _build_url(self, params):
+        taxonomy = params.get('taxonomy')
+        concept = params.get('concept')
+        unit = params.get('unit')
+        period = params.get('period')
+        
+        if not all([taxonomy, concept, unit, period]):
+            raise ValueError("Missing required parameters")
+            
+        return f"{self.base_url}/{taxonomy}/{concept}/{unit}/{period}.json"
+
+    async def _get_xbrl_data(self, params_list):
+        tasks = []
+        urls = {}
+        
+        for params in params_list:
+            url = self._build_url(params)
+            urls[url] = params
+            tasks.append(self._fetch_json(url))
+        
+        results = await asyncio.gather(*tasks)
+        
+        return {url: result for url, result in zip(urls.keys(), results) if result is not None}
+
+    def get_xbrl_frames(self, params_list):
+        async def _download():
+            async with self as downloader:
+                return await self._get_xbrl_data(params_list)
+
+        return asyncio.run(_download())

datamule/document.py

@@ -2,7 +2,7 @@ import json
 import csv
 from .helper import convert_to_dashed_accession
 import re
-from doc2dict import xml2dict, txt2dict
+from doc2dict import xml2dict, txt2dict, dict2dict
 from doc2dict.mapping import flatten_hierarchy
 from .mapping_dicts import txt_mapping_dicts
 from .mapping_dicts import xml_mapping_dicts
@@ -126,7 +126,8 @@ class Document:
             elif self.type == 'SC 13G':
                 mapping_dict = txt_mapping_dicts.dict_13g
             
-            self.data = txt2dict(content=self.content, mapping_dict=mapping_dict)
+            self.data = {}
+            self.data['document'] = dict2dict(txt2dict(content=self.content, mapping_dict=mapping_dict))
         return self.data
     
     def write_json(self, output_filename=None):

datamule/mapping_dicts/txt_mapping_dicts.py

@@ -29,10 +29,12 @@ dict_sgml = {
     }
 }
 
-item_pattern_mapping = r"^\n\n\s*(ITEM|Item)\b"
-part_pattern_mapping = r"^\n\n\s*(PART|Part)\b"
+item_pattern_mapping = r"^\n\n\s*(ITEM|Item)\s+(\d+[a-zA-Z]?|ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN|ELEVEN|TWELVE|THIRTEEN|FOURTEEN|FIFTEEN|SIXTEEN|[0-9]+[a-zA-Z]?)\.?"
+item_pattern_mapping_8k = r"^\n\n\s*(ITEM|Item)\s+(\d+(?:\.\d+)?[a-zA-Z]?|ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN|ELEVEN|TWELVE|THIRTEEN|FOURTEEN|FIFTEEN|SIXTEEN|[0-9]+[a-zA-Z]?)\.?"
+part_pattern_mapping = r"^\n\n\s*(PART|Part)\s+(?:I{1,3}|IV)\.?"
 
 item_pattern_standardization = r"^\s*(?:ITEM|Item)\s+(\d+[a-zA-Z]?|ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN|ELEVEN|TWELVE|THIRTEEN|FOURTEEN|FIFTEEN|SIXTEEN|[0-9]+[a-zA-Z]?)\.?"
+item_pattern_standardization_8k = r"^\s*(?:ITEM|Item)\s+(\d+(?:\.\d+)?[a-zA-Z]?|ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN|ELEVEN|TWELVE|THIRTEEN|FOURTEEN|FIFTEEN|SIXTEEN)\.?"
 part_pattern_standardization =  r"^\s*(?:PART|Part)\s+([IVX]+)"
         
 
@@ -194,7 +196,7 @@ dict_8k["rules"]["mappings"].extend([
             {
                 "type": "hierarchy",
                 "name": "item",
-                "pattern": item_pattern_mapping,
+                "pattern": item_pattern_mapping_8k,
                 "hierarchy": 0
             },
             ])
@@ -204,7 +206,7 @@ dict_8k['transformations'] = [
         "type": "standardize", 
         "match": {
             "type": "item",
-            "text_pattern": item_pattern_standardization
+            "text_pattern": item_pattern_standardization_8k
         },
         "output": {
             "format": "item{}",

datamule/monitor.py

@@ -11,6 +11,15 @@ def _get_current_eastern_date():
     eastern = pytz.timezone('America/New_York')
     return datetime.now(eastern)
 
+def _parse_date(date_str):
+    """Parse YYYY-MM-DD date string to datetime object in Eastern timezone"""
+    try:
+        date = datetime.strptime(date_str, '%Y-%m-%d')
+        eastern = pytz.timezone('America/New_York')
+        return eastern.localize(date)
+    except ValueError as e:
+        raise ValueError(f"Invalid date format. Please use YYYY-MM-DD. Error: {str(e)}")
+
 class PreciseRateLimiter:
     def __init__(self, rate, interval=1.0):
         self.rate = rate  # requests per interval
@@ -67,7 +76,8 @@ class RateMonitor:
 class Monitor:
     def __init__(self):
         self.last_total = 0
-        self.last_date = _get_current_eastern_date()
+        self.last_date = None
+        self.current_monitor_date = None
         self.submissions = []
         self.max_hits = 10000
         self.limiter = PreciseRateLimiter(5)  # 5 requests per second
@@ -91,16 +101,29 @@ class Monitor:
         """Poll API until new submissions are found."""
         while True:
             current_date = _get_current_eastern_date()
-            date_str = current_date.strftime('%Y-%m-%d')
-            timestamp = int(time.time())  # Add this line
             
-            if self.last_date != current_date.strftime('%Y-%m-%d'):
-                print(f"New date: {date_str}")
+            # If we're caught up to current date, use it, otherwise use our tracking date
+            if self.current_monitor_date.date() >= current_date.date():
+                self.current_monitor_date = current_date
+            else:
+                # If we're behind current date and haven't finished current date's processing,
+                # continue with current date
+                if self.last_date == self.current_monitor_date.strftime('%Y-%m-%d'):
+                    pass
+                else:
+                    # Move to next day
+                    self.current_monitor_date += timedelta(days=1)
+                    
+            date_str = self.current_monitor_date.strftime('%Y-%m-%d')
+            timestamp = int(time.time())
+            
+            if self.last_date != date_str:
+                print(f"Processing date: {date_str}")
                 self.last_total = 0
                 self.submissions = []
                 self.last_date = date_str
             
-            poll_url = f"{base_url}&startdt={date_str}&enddt={date_str}&v={timestamp}"  # Modified this line
+            poll_url = f"{base_url}&startdt={date_str}&enddt={date_str}&v={timestamp}"
             if not quiet:
                 print(f"Polling {poll_url}")
             
@@ -109,10 +132,16 @@ class Monitor:
                 if data:
                     current_total = data['hits']['total']['value']
                     if current_total > self.last_total:
-                        print(f"Found {current_total - self.last_total} new submissions")
+                        print(f"Found {current_total - self.last_total} new submissions for {date_str}")
                         self.last_total = current_total
                         return current_total, data, poll_url
                     self.last_total = current_total
+                    
+                    # If we have no hits and we're processing a past date,
+                    # we can move to the next day immediately
+                    if current_total == 0 and self.current_monitor_date.date() < current_date.date():
+                        continue
+                    
             except Exception as e:
                 print(f"Error in poll: {str(e)}")
             
@@ -120,7 +149,6 @@ class Monitor:
 
     async def _retrieve_batch(self, session, poll_url, from_positions, quiet):
         """Retrieve a batch of submissions concurrently."""
-        # The poll_url already contains the timestamp from _poll
         tasks = [
             self._fetch_json(
                 session,
@@ -176,11 +204,17 @@ class Monitor:
         
         return submissions
 
-    async def _monitor(self, callback, form=None, cik=None, ticker=None, poll_interval=1000, quiet=True):
+    async def _monitor(self, callback, form=None, cik=None, ticker=None, start_date=None, poll_interval=1000, quiet=True):
         """Main monitoring loop with parallel processing."""
         if poll_interval < 100:
             raise ValueError("SEC rate limit is 10 requests per second, set poll_interval to 100ms or higher")
 
+        # Set up initial monitoring date
+        if start_date:
+            self.current_monitor_date = _parse_date(start_date)
+        else:
+            self.current_monitor_date = _get_current_eastern_date()
+
         # Handle form parameter
         if form is None:
             form = ['-0']
@@ -233,6 +267,17 @@ class Monitor:
                 
                 await asyncio.sleep(poll_interval / 1000)
 
-    def monitor_submissions(self, callback=None, form=None, cik=None, ticker=None, poll_interval=1000, quiet=True):
-        """Start the monitoring process."""
-        asyncio.run(self._monitor(callback, form, cik, ticker, poll_interval, quiet))
+    def monitor_submissions(self, callback=None, form=None, cik=None, ticker=None, start_date=None, poll_interval=1000, quiet=True):
+        """
+        Start the monitoring process.
+        
+        Parameters:
+            callback (callable, optional): Function to call when new submissions are found
+            form (str or list, optional): Form type(s) to monitor
+            cik (str or list, optional): CIK(s) to monitor
+            ticker (str, optional): Ticker symbol to monitor
+            start_date (str, optional): Start date in YYYY-MM-DD format
+            poll_interval (int, optional): Polling interval in milliseconds
+            quiet (bool, optional): Suppress verbose output
+        """
+        asyncio.run(self._monitor(callback, form, cik, ticker, start_date, poll_interval, quiet))

{datamule-1.0.0.dist-info → datamule-1.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.0.0
+Version: 1.0.3
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.0.0.dist-info → datamule-1.0.3.dist-info}/RECORD

@@ -1,12 +1,15 @@
 datamule/__init__.py,sha256=IDVK3i5i5DxLlQJ_71aYkloGNi528JOUx8hU6bDzLXM,1255
-datamule/book.py,sha256=hzp5Ae_PfTg3c_h_LdRQOc9fU6OifapKtV0bU-SyIOw,775
 datamule/config.py,sha256=Y--CVv7JcgrjJkMOSLrvm2S8B9ost6RMSkGviP-MKtg,883
-datamule/document.py,sha256=P1UeF-GLta08T7Ur77IFOp7vJBIYcQPbz3gfuWJ3fi0,10851
+datamule/document.py,sha256=-XT3nkRb1oCOken7n3AUCI8itQNfuKVN86cuD3rys3E,10912
 datamule/helper.py,sha256=tr3AQWus9dHNZFKpLSglWjcb8zmm5qDXjOWACMhvMxQ,4594
-datamule/monitor.py,sha256=AfhGqC_GFTYWemRKgYE85V7rIGMN_pbcpxW6kORQtpw,9273
+datamule/monitor.py,sha256=NgzOB_RCLAZDrowYagPWIM4PGNngd1lA3K9Qplkk3Ys,11325
 datamule/packageupdater.py,sha256=vEGqlTj6FudIeVHBVJltPh2eBDEqMG9HYmnyrRVKeSU,9595
 datamule/portfolio.py,sha256=U_QRNk_CbMmi3nJ0VBIwc9SVEGq6kA8LCZHBj9nOGXs,4032
 datamule/submission.py,sha256=uioIYJbsoe-87nRPyzlo-LZ8Hp7HG7A4KPGSnw86PKY,2790
+datamule/book/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamule/book/book.py,sha256=ns7wVk_gLX5KhAghnSTxKcWIB8bKg7pwsM3es5GfCyM,1422
+datamule/book/eftsquery.py,sha256=MW-rMb3L-S1sgAR_Y-Ca3zGa1W9hclTzH52twUK87oM,4596
+datamule/book/xbrl_retriever.py,sha256=qLzrskqrQ8guEMFudlUCWFvRqAgSthhql5FXh_E3eCo,2910
 datamule/data/company_former_names.csv,sha256=HE9cAv-_QKFX6jT-_-D0rHmaDyQuAzL4MJwank5O1U8,706380
 datamule/data/company_metadata.csv,sha256=yPovrCVjYwLWTU_hBUFJymp8iNO0NBYuq_QwOkRLoN8,3068599
 datamule/data/company_tickers.csv,sha256=GW6lOP54RiGJCx-d9N5jEBy7tGVgU3zI-5xHJXrZfSI,400363
@@ -14,7 +17,7 @@ datamule/data/sec-glossary.csv,sha256=-cN7GjiadLw5C1sv4zSeCnfeZZDYeSgJl-0ydarMAo
 datamule/data/xbrl_descriptions.csv,sha256=SQ9wUURNqG424rnTiZtopsxV2q-PvU4NMj52LqgDsvg,2621524
 datamule/downloader/downloader.py,sha256=vnMsw0oWqRa84scu6ZcywxbJxsIn38vLV0tybakx3jQ,15217
 datamule/downloader/premiumdownloader.py,sha256=YhGFwkYqjLkdc5ex2YKM-L7nBAPm5MMCdTwVVP0JO78,14314
-datamule/mapping_dicts/txt_mapping_dicts.py,sha256=Eh6qYhseuKjjnxGh0A5blHr7mbq9CigFn6Zv9xcG2zU,5783
+datamule/mapping_dicts/txt_mapping_dicts.py,sha256=DQPrGYbAPQxomRUtt4iiMGrwuF7BHc_LeFBQuYBzU9o,6311
 datamule/mapping_dicts/xml_mapping_dicts.py,sha256=Z22yDVwKYonUfM5foQP00dVDE8EHhhMKp0CLqVKV5OI,438
 datamule/mulebot/__init__.py,sha256=YvZXV6xQ0iP-oGD8rloufjdwJL6D46P3NNr0CY9PQCA,29
 datamule/mulebot/helper.py,sha256=olztOwltfELZ-IERM2bRNLBavD04kfB6ueWTisJAleA,1080
@@ -34,7 +37,7 @@ datamule/mulebot/mulebot_server/static/scripts/suggestions.js,sha256=TCyz8OYuXeI
 datamule/mulebot/mulebot_server/static/scripts/tableArtifacts.js,sha256=UtkUpLvELNI4Ibpb7VstgVA9Tk-8jbkxXhmXsgufFa4,4437
 datamule/mulebot/mulebot_server/static/scripts/utils.js,sha256=oGPMtyT9dvuqHqrfZj33t4vLZiF8UJrMXB1hpPXRNu4,1255
 datamule/mulebot/mulebot_server/templates/chat-minimalist.html,sha256=MsTbgpnLD0JCQiKKP3XeeNJRNsRqKsRa1j_XXW7nBKw,6975
-datamule-1.0.0.dist-info/METADATA,sha256=EMOJtZUCrYTNu7YOrhfZmzapdumfM0TXsnRhDBs0x6A,732
-datamule-1.0.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-datamule-1.0.0.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-1.0.0.dist-info/RECORD,,
+datamule-1.0.3.dist-info/METADATA,sha256=8PZAcyMcoQTNaV21b9N09t8cd4Uw0Kxm6aImKXlSsCo,732
+datamule-1.0.3.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+datamule-1.0.3.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-1.0.3.dist-info/RECORD,,