datamule 0.381__py3-none-any.whl → 1.0.2__py3-none-any.whl

datamule/__init__.py

@@ -1,87 +1,47 @@
-# datamule/__init__.py
-import sys
-from importlib.util import find_spec
-from functools import lru_cache
-
-# Lazy load nest_asyncio only when needed
-def _setup_jupyter():
-    """Setup Jupyter-specific configurations if needed."""
-    if _is_jupyter():
-        import nest_asyncio
-        nest_asyncio.apply()
-
-def _is_jupyter():
-    """Check if the code is running in a Jupyter environment."""
-    try:
-        shell = get_ipython().__class__.__name__
-        return shell == 'ZMQInteractiveShell'
-    except NameError:
-        return False
-
-# Lazy loading for main components
-@lru_cache(None)
-def get_downloader():
-    from .downloader.sec_downloader import Downloader
-    return Downloader
-
-@lru_cache(None)
-def get_parser():
-    from .parser.sec_parser import Parser
-    return Parser
-
-@lru_cache(None)
-def get_filing():
-    from .sec_filing import Filing
-    return Filing
-
-@lru_cache(None)
-def get_dataset_builder():
-    if find_spec('pandas') is not None:
-        try:
-            from .dataset_builder.dataset_builder import DatasetBuilder
-            return DatasetBuilder
-        except ImportError:
-            return None
-    return None
-
-# Helper functions that can be imported directly
-from .datamule_api import parse_textual_filing
-from .helper import load_package_csv, load_package_dataset
-from .global_vars import *
-from .parser.sgml_parser import parse_submission
-
-# Define classes with delayed initialization
-class Downloader:
-    def __new__(cls, *args, **kwargs):
-        return get_downloader()(*args, **kwargs)
-
-class Parser:
-    def __new__(cls, *args, **kwargs):
-        return get_parser()(*args, **kwargs)
-
-class Filing:
-    def __new__(cls, *args, **kwargs):
-        return get_filing()(*args, **kwargs)
-
-class DatasetBuilder:
-    def __new__(cls, *args, **kwargs):
-        builder_cls = get_dataset_builder()
-        if builder_cls is None:
-            raise ImportError(
-                "DatasetBuilder requires pandas. "
-                "Install with: pip install datamule[dataset_builder]"
-            )
-        return builder_cls(*args, **kwargs)
-
-# Set up Jupyter support only when imported
-_setup_jupyter()
-
-__all__ = [
-    'Downloader',
-    'parse_textual_filing',
-    'load_package_csv',
-    'load_package_dataset',
-    'Parser',
-    'Filing',
-    'DatasetBuilder'
+from .downloader.downloader import Downloader
+from .downloader.premiumdownloader import PremiumDownloader
+from .monitor import Monitor
+from .packageupdater import PackageUpdater
+from .submission import Submission
+from .portfolio import Portfolio
+from .document import Document
+from secsgml import parse_sgml_submission
+from .helper import load_package_csv, load_package_dataset
+from .config import Config
+
+
+# Keep the notebook environment setup
+def _is_notebook_env():
+    """Check if the code is running in a Jupyter or Colab environment."""
+    try:
+        shell = get_ipython().__class__.__name__
+        return shell in ('ZMQInteractiveShell', 'Shell', 'Google.Colab')
+    except NameError:
+        return False
+
+from functools import lru_cache
+
+@lru_cache(maxsize=1)
+def _setup_notebook_env():
+    """Setup Jupyter/Colab-specific configurations if needed."""
+    if _is_notebook_env():
+        import nest_asyncio
+        nest_asyncio.apply()
+
+# Set up notebook environment
+_setup_notebook_env()
+
+__all__ = [
+    'Downloader',
+    'PremiumDownloader',
+    'load_package_csv',
+    'load_package_dataset',
+    'Filing',
+    'Portfolio',
+    'Monitor',
+    'PackageUpdater',
+    'Submission',
+    'Document',
+    'parse_sgml_submission',
+    'Config'
 ]

datamule/book/book.py

@@ -0,0 +1,34 @@
+# Streams data rather than downloading it. 
+# additional functionality such as query by xbrl, and other db
+# also this is basically our experimental rework of portfolio w/o disturbing existing users
+# this is highly experimental and may not work as expected
+# only for datamule source
+# likely new bottleneck will be local parsing() - will be bypassed in future when we have parsed archive
+# wow parsed archive is going to be crazy fast - like every 10k in 1 minute.
+
+# example queries filter by sic = 7372, xbrl query = dei:operatingprofit > 0 in date range 2018-2019
+
+# hmm do we go for sql esq or not.
+# I think we do.
+# i think we remove cik, ticker, sic, etc and just have a query object
+# should be sql esq so users can use it easily w/o learnign new syntax
+
+# WHERE submission_type = '10-K'
+# AND us-gaap:ResearchAndDevelopmentExpense > 0 
+# AND dei:debt_to_equity < 2
+# AND filing_date BETWEEN '2023-01-01' AND '2023-12-31'
+# AND CIK in (123, 456, 789)
+# AND SIC in (123, 456, 789)
+# AND ticker in ('AAPL', 'GOOGL', 'AMZN')
+# AND document_type = 'EX-99.1' # to select attachments
+
+from .eftsquery import EFTSQuery
+
+
+class Book():
+    def process_submissions(self,cik,ticker,sic,submission_type,document_type,date,
+                            xbrl_query={},
+                            metadata_callback=None,
+                            document_callback=None,):
+        # grabs data and processes it
+        pass 

datamule/book/eftsquery.py

@@ -0,0 +1,127 @@
+import asyncio
+import aiohttp
+from tqdm import tqdm
+from datetime import datetime
+from urllib.parse import urlencode
+import time
+
+class PreciseRateLimiter:
+    def __init__(self, rate=10, interval=1.0):
+        self.rate = rate  # requests per interval
+        self.interval = interval  # in seconds
+        self.token_time = self.interval / self.rate  # time per token
+        self.last_time = time.time()
+        self.lock = asyncio.Lock()
+
+    async def acquire(self):
+        async with self.lock:
+            now = time.time()
+            wait_time = self.last_time + self.token_time - now
+            if wait_time > 0:
+                await asyncio.sleep(wait_time)
+            self.last_time = time.time()
+            return True
+
+class EFTSQuery:
+    def __init__(self):
+        self.headers = {
+            'User-Agent': 'Your Name yourname@email.com',
+            'Accept-Encoding': 'gzip, deflate',
+            'Host': 'efts.sec.gov'
+        }
+        self.session = None
+        self.limiter = PreciseRateLimiter(10)
+
+    async def __aenter__(self):
+        if not self.session:
+            self.session = aiohttp.ClientSession(headers=self.headers)
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self.session:
+            await self.session.close()
+            self.session = None
+
+    async def _fetch_json(self, url):
+        await self.limiter.acquire()
+        try:
+            async with self.session.get(url) as response:
+                if response.status == 429:
+                    await asyncio.sleep(61)
+                    return await self._fetch_json(url)
+                return await response.json()
+        except Exception as e:
+            print(f"Error fetching {url}: {str(e)}")
+            return None
+
+    async def _get_accession_numbers(self, base_url):
+        data = await self._fetch_json(f"{base_url}&from=0&size=1")
+        if not data or 'hits' not in data:
+            return []
+            
+        total_hits = data['hits']['total']['value']
+        if not total_hits:
+            return []
+
+        accession_numbers = []
+        start = 0
+        page_size = 100
+        batch_size = 10  # Number of concurrent requests
+        
+        with tqdm(total=total_hits) as pbar:
+            while start < total_hits:
+                tasks = []
+                for i in range(batch_size):
+                    if start + i * page_size >= total_hits:
+                        break
+                    url = f"{base_url}&from={start + i * page_size}&size={page_size}"
+                    tasks.append(self._fetch_json(url))
+                
+                if not tasks:
+                    break
+                    
+                results = await asyncio.gather(*tasks)
+                
+                for data in results:
+                    if data and 'hits' in data:
+                        hits = data['hits']['hits']
+                        batch_numbers = [
+                            f"{hit['_source']['ciks'][0]}/{hit['_id'].split(':')[0]}"
+                            for hit in hits
+                        ]
+                        accession_numbers.extend(batch_numbers)
+                        pbar.update(len(hits))
+                
+                start += batch_size * page_size
+
+        return accession_numbers
+
+    def query_efts(self, cik=None, ticker=None, submission_type=None, filing_date=None, search_text=None):
+        async def _download():
+            async with self as downloader:
+                params = {}
+                
+                if cik:
+                    params['ciks'] = str(cik).zfill(10)
+
+                if submission_type:
+                    params['forms'] = ','.join(submission_type) if isinstance(submission_type, list) else submission_type
+
+                if isinstance(filing_date, list):
+                    dates = [(d, d) for d in filing_date]
+                elif isinstance(filing_date, tuple):
+                    dates = [filing_date]
+                else:
+                    date_str = filing_date if filing_date else f"2001-01-01,{datetime.now().strftime('%Y-%m-%d')}"
+                    start, end = date_str.split(',')
+                    dates = [(start, end)]
+                
+                params['startdt'], params['enddt'] = dates[0]
+
+                if search_text:
+                    params['q'] = f'"{search_text}"'
+
+                base_url = f"https://efts.sec.gov/LATEST/search-index?{urlencode(params, doseq=True)}"
+                return await self._get_accession_numbers(base_url)
+
+        return asyncio.run(_download())

datamule/book/xbrl_retriever.py

@@ -0,0 +1,88 @@
+import asyncio
+import aiohttp
+import time
+
+class PreciseRateLimiter:
+    def __init__(self, rate=10, interval=1.0):
+        self.rate = rate
+        self.interval = interval
+        self.token_time = self.interval / self.rate
+        self.last_time = time.time()
+        self.lock = asyncio.Lock()
+
+    async def acquire(self):
+        async with self.lock:
+            now = time.time()
+            wait_time = self.last_time + self.token_time - now
+            if wait_time > 0:
+                await asyncio.sleep(wait_time)
+            self.last_time = time.time()
+            return True
+
+class XBRLRetriever:
+    def __init__(self):
+        self.base_url = "https://data.sec.gov/api/xbrl/frames"
+        self.headers = {
+            'User-Agent': 'Your Name yourname@email.com',
+            'Accept-Encoding': 'gzip, deflate',
+            'Host': 'data.sec.gov'
+        }
+        self.session = None
+        self.limiter = PreciseRateLimiter(10)
+
+    async def __aenter__(self):
+        if not self.session:
+            self.session = aiohttp.ClientSession(headers=self.headers)
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self.session:
+            await self.session.close()
+            self.session = None
+
+    async def _fetch_json(self, url):
+        await self.limiter.acquire()
+        try:
+            async with self.session.get(url) as response:
+                if response.status == 429:
+                    await asyncio.sleep(61)
+                    return await self._fetch_json(url)
+                elif response.status == 200:
+                    return await response.json()
+                else:
+                    print(f"Error {response.status} for URL: {url}")
+                    return None
+        except Exception as e:
+            print(f"Error fetching {url}: {str(e)}")
+            return None
+
+    def _build_url(self, params):
+        taxonomy = params.get('taxonomy')
+        concept = params.get('concept')
+        unit = params.get('unit')
+        period = params.get('period')
+        
+        if not all([taxonomy, concept, unit, period]):
+            raise ValueError("Missing required parameters")
+            
+        return f"{self.base_url}/{taxonomy}/{concept}/{unit}/{period}.json"
+
+    async def _get_xbrl_data(self, params_list):
+        tasks = []
+        urls = {}
+        
+        for params in params_list:
+            url = self._build_url(params)
+            urls[url] = params
+            tasks.append(self._fetch_json(url))
+        
+        results = await asyncio.gather(*tasks)
+        
+        return {url: result for url, result in zip(urls.keys(), results) if result is not None}
+
+    def get_xbrl_frames(self, params_list):
+        async def _download():
+            async with self as downloader:
+                return await self._get_xbrl_data(params_list)
+
+        return asyncio.run(_download())

datamule/config.py

@@ -0,0 +1,29 @@
+import json
+import os
+
+class Config:
+    def __init__(self):
+        self.config_path = os.path.expanduser("~/.datamule/config.json")
+        self._ensure_config_exists()
+        
+    def _ensure_config_exists(self):
+        os.makedirs(os.path.dirname(self.config_path), exist_ok=True)
+        if not os.path.exists(self.config_path):
+            self._save_config({"default_source": None})
+
+    def _save_config(self, config):
+        with open(self.config_path, 'w') as f:
+            json.dump(config, f)
+
+    def set_default_source(self, source):
+        config = self._load_config()
+        config["default_source"] = source
+        self._save_config(config)
+
+    def get_default_source(self):
+        config = self._load_config()
+        return config.get("default_source")
+
+    def _load_config(self):
+        with open(self.config_path) as f:
+            return json.load(f)