datamule 1.0.0__tar.gz → 1.0.2__tar.gz

{datamule-1.0.0 → datamule-1.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.0.0
+Version: 1.0.2
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.0.0/datamule → datamule-1.0.2/datamule/book}/book.py

@@ -6,8 +6,26 @@
 # likely new bottleneck will be local parsing() - will be bypassed in future when we have parsed archive
 # wow parsed archive is going to be crazy fast - like every 10k in 1 minute.
 
+# example queries filter by sic = 7372, xbrl query = dei:operatingprofit > 0 in date range 2018-2019
+
+# hmm do we go for sql esq or not.
+# I think we do.
+# i think we remove cik, ticker, sic, etc and just have a query object
+# should be sql esq so users can use it easily w/o learnign new syntax
+
+# WHERE submission_type = '10-K'
+# AND us-gaap:ResearchAndDevelopmentExpense > 0 
+# AND dei:debt_to_equity < 2
+# AND filing_date BETWEEN '2023-01-01' AND '2023-12-31'
+# AND CIK in (123, 456, 789)
+# AND SIC in (123, 456, 789)
+# AND ticker in ('AAPL', 'GOOGL', 'AMZN')
+# AND document_type = 'EX-99.1' # to select attachments
+
+from .eftsquery import EFTSQuery
+
+
 class Book():
-    pass
     def process_submissions(self,cik,ticker,sic,submission_type,document_type,date,
                             xbrl_query={},
                             metadata_callback=None,

datamule-1.0.2/datamule/book/eftsquery.py

@@ -0,0 +1,127 @@
+import asyncio
+import aiohttp
+from tqdm import tqdm
+from datetime import datetime
+from urllib.parse import urlencode
+import time
+
+class PreciseRateLimiter:
+    def __init__(self, rate=10, interval=1.0):
+        self.rate = rate  # requests per interval
+        self.interval = interval  # in seconds
+        self.token_time = self.interval / self.rate  # time per token
+        self.last_time = time.time()
+        self.lock = asyncio.Lock()
+
+    async def acquire(self):
+        async with self.lock:
+            now = time.time()
+            wait_time = self.last_time + self.token_time - now
+            if wait_time > 0:
+                await asyncio.sleep(wait_time)
+            self.last_time = time.time()
+            return True
+
+class EFTSQuery:
+    def __init__(self):
+        self.headers = {
+            'User-Agent': 'Your Name yourname@email.com',
+            'Accept-Encoding': 'gzip, deflate',
+            'Host': 'efts.sec.gov'
+        }
+        self.session = None
+        self.limiter = PreciseRateLimiter(10)
+
+    async def __aenter__(self):
+        if not self.session:
+            self.session = aiohttp.ClientSession(headers=self.headers)
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self.session:
+            await self.session.close()
+            self.session = None
+
+    async def _fetch_json(self, url):
+        await self.limiter.acquire()
+        try:
+            async with self.session.get(url) as response:
+                if response.status == 429:
+                    await asyncio.sleep(61)
+                    return await self._fetch_json(url)
+                return await response.json()
+        except Exception as e:
+            print(f"Error fetching {url}: {str(e)}")
+            return None
+
+    async def _get_accession_numbers(self, base_url):
+        data = await self._fetch_json(f"{base_url}&from=0&size=1")
+        if not data or 'hits' not in data:
+            return []
+            
+        total_hits = data['hits']['total']['value']
+        if not total_hits:
+            return []
+
+        accession_numbers = []
+        start = 0
+        page_size = 100
+        batch_size = 10  # Number of concurrent requests
+        
+        with tqdm(total=total_hits) as pbar:
+            while start < total_hits:
+                tasks = []
+                for i in range(batch_size):
+                    if start + i * page_size >= total_hits:
+                        break
+                    url = f"{base_url}&from={start + i * page_size}&size={page_size}"
+                    tasks.append(self._fetch_json(url))
+                
+                if not tasks:
+                    break
+                    
+                results = await asyncio.gather(*tasks)
+                
+                for data in results:
+                    if data and 'hits' in data:
+                        hits = data['hits']['hits']
+                        batch_numbers = [
+                            f"{hit['_source']['ciks'][0]}/{hit['_id'].split(':')[0]}"
+                            for hit in hits
+                        ]
+                        accession_numbers.extend(batch_numbers)
+                        pbar.update(len(hits))
+                
+                start += batch_size * page_size
+
+        return accession_numbers
+
+    def query_efts(self, cik=None, ticker=None, submission_type=None, filing_date=None, search_text=None):
+        async def _download():
+            async with self as downloader:
+                params = {}
+                
+                if cik:
+                    params['ciks'] = str(cik).zfill(10)
+
+                if submission_type:
+                    params['forms'] = ','.join(submission_type) if isinstance(submission_type, list) else submission_type
+
+                if isinstance(filing_date, list):
+                    dates = [(d, d) for d in filing_date]
+                elif isinstance(filing_date, tuple):
+                    dates = [filing_date]
+                else:
+                    date_str = filing_date if filing_date else f"2001-01-01,{datetime.now().strftime('%Y-%m-%d')}"
+                    start, end = date_str.split(',')
+                    dates = [(start, end)]
+                
+                params['startdt'], params['enddt'] = dates[0]
+
+                if search_text:
+                    params['q'] = f'"{search_text}"'
+
+                base_url = f"https://efts.sec.gov/LATEST/search-index?{urlencode(params, doseq=True)}"
+                return await self._get_accession_numbers(base_url)
+
+        return asyncio.run(_download())

datamule-1.0.2/datamule/book/xbrl_retriever.py

@@ -0,0 +1,88 @@
+import asyncio
+import aiohttp
+import time
+
+class PreciseRateLimiter:
+    def __init__(self, rate=10, interval=1.0):
+        self.rate = rate
+        self.interval = interval
+        self.token_time = self.interval / self.rate
+        self.last_time = time.time()
+        self.lock = asyncio.Lock()
+
+    async def acquire(self):
+        async with self.lock:
+            now = time.time()
+            wait_time = self.last_time + self.token_time - now
+            if wait_time > 0:
+                await asyncio.sleep(wait_time)
+            self.last_time = time.time()
+            return True
+
+class XBRLRetriever:
+    def __init__(self):
+        self.base_url = "https://data.sec.gov/api/xbrl/frames"
+        self.headers = {
+            'User-Agent': 'Your Name yourname@email.com',
+            'Accept-Encoding': 'gzip, deflate',
+            'Host': 'data.sec.gov'
+        }
+        self.session = None
+        self.limiter = PreciseRateLimiter(10)
+
+    async def __aenter__(self):
+        if not self.session:
+            self.session = aiohttp.ClientSession(headers=self.headers)
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self.session:
+            await self.session.close()
+            self.session = None
+
+    async def _fetch_json(self, url):
+        await self.limiter.acquire()
+        try:
+            async with self.session.get(url) as response:
+                if response.status == 429:
+                    await asyncio.sleep(61)
+                    return await self._fetch_json(url)
+                elif response.status == 200:
+                    return await response.json()
+                else:
+                    print(f"Error {response.status} for URL: {url}")
+                    return None
+        except Exception as e:
+            print(f"Error fetching {url}: {str(e)}")
+            return None
+
+    def _build_url(self, params):
+        taxonomy = params.get('taxonomy')
+        concept = params.get('concept')
+        unit = params.get('unit')
+        period = params.get('period')
+        
+        if not all([taxonomy, concept, unit, period]):
+            raise ValueError("Missing required parameters")
+            
+        return f"{self.base_url}/{taxonomy}/{concept}/{unit}/{period}.json"
+
+    async def _get_xbrl_data(self, params_list):
+        tasks = []
+        urls = {}
+        
+        for params in params_list:
+            url = self._build_url(params)
+            urls[url] = params
+            tasks.append(self._fetch_json(url))
+        
+        results = await asyncio.gather(*tasks)
+        
+        return {url: result for url, result in zip(urls.keys(), results) if result is not None}
+
+    def get_xbrl_frames(self, params_list):
+        async def _download():
+            async with self as downloader:
+                return await self._get_xbrl_data(params_list)
+
+        return asyncio.run(_download())

{datamule-1.0.0 → datamule-1.0.2}/datamule/document.py

@@ -2,7 +2,7 @@ import json
 import csv
 from .helper import convert_to_dashed_accession
 import re
-from doc2dict import xml2dict, txt2dict
+from doc2dict import xml2dict, txt2dict, dict2dict
 from doc2dict.mapping import flatten_hierarchy
 from .mapping_dicts import txt_mapping_dicts
 from .mapping_dicts import xml_mapping_dicts
@@ -126,7 +126,8 @@ class Document:
             elif self.type == 'SC 13G':
                 mapping_dict = txt_mapping_dicts.dict_13g
             
-            self.data = txt2dict(content=self.content, mapping_dict=mapping_dict)
+            self.data = {}
+            self.data['document'] = dict2dict(txt2dict(content=self.content, mapping_dict=mapping_dict))
         return self.data
     
     def write_json(self, output_filename=None):

{datamule-1.0.0 → datamule-1.0.2}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.0.0
+Version: 1.0.2
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.0.0 → datamule-1.0.2}/datamule.egg-info/SOURCES.txt

@@ -1,6 +1,5 @@
 setup.py
 datamule/__init__.py
-datamule/book.py
 datamule/config.py
 datamule/document.py
 datamule/helper.py
@@ -13,6 +12,10 @@ datamule.egg-info/SOURCES.txt
 datamule.egg-info/dependency_links.txt
 datamule.egg-info/requires.txt
 datamule.egg-info/top_level.txt
+datamule/book/__init__.py
+datamule/book/book.py
+datamule/book/eftsquery.py
+datamule/book/xbrl_retriever.py
 datamule/data/company_former_names.csv
 datamule/data/company_metadata.csv
 datamule/data/company_tickers.csv

{datamule-1.0.0 → datamule-1.0.2}/setup.py

@@ -15,7 +15,7 @@ extras["all"] = list(all_dependencies)
 setup(
     name="datamule",
     author="John Friedman",
-    version="1.0.0",
+    version="1.0.2",
     description="Making it easier to use SEC filings.",
     packages=find_namespace_packages(include=['datamule*']),
     url="https://github.com/john-friedman/datamule-python",