datamule 1.8.6__tar.gz → 2.0.0__tar.gz

{datamule-1.8.6 → datamule-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.8.6
+Version: 2.0.0
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
@@ -18,3 +18,4 @@ Requires-Dist: doc2dict
 Requires-Dist: secxbrl
 Requires-Dist: secsgml
 Requires-Dist: websocket-client
+Requires-Dist: company_fundamentals

{datamule-1.8.6/datamule/seclibrary → datamule-2.0.0/datamule/datamule}/datamule_lookup.py

@@ -3,7 +3,6 @@ import asyncio
 import aiohttp
 import urllib.parse
 import ssl
-import json
 import time
 from tqdm import tqdm
 

datamule-2.0.0/datamule/datamule/datamule_mysql_rds.py

@@ -0,0 +1,275 @@
+import os
+import asyncio
+import aiohttp
+import json
+import ssl
+import time
+from tqdm import tqdm
+
+class DatamuleMySQL:
+    def __init__(self, api_key=None):
+        self.API_BASE_URL = "https://datamule-mysql-rds.jgfriedman99.workers.dev"
+        self._api_key = api_key
+        self.total_cost = 0
+        self.remaining_balance = None
+        self.start_time = None
+
+    @property
+    def api_key(self):
+        return getattr(self, '_api_key', None) or os.getenv('DATAMULE_API_KEY')
+
+    @api_key.setter
+    def api_key(self, value):
+        if not value:
+            raise ValueError("API key cannot be empty")
+        self._api_key = value
+
+    async def _fetch_page(self, session, table, database, filters, page=1, page_size=25000):
+        payload = {
+            "table": table,
+            "database": database,
+            "filters": filters,
+            "page": page,
+            "pageSize": page_size
+        }
+        
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.api_key}",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+        }
+        
+        async with session.post(self.API_BASE_URL, json=payload, headers=headers) as response:
+            data = await response.json()
+            if not data.get('success'):
+                raise ValueError(f"API request failed: {data.get('error')}")
+            
+            # Track costs and balance
+            billing = data['metadata']['billing']
+            page_cost = billing['total_charge']
+            self.total_cost += page_cost
+            self.remaining_balance = billing['remaining_balance']
+            
+            return data['data'], data['metadata']['pagination'], page_cost
+
+    async def execute_query(self, table, **kwargs):
+        if self.api_key is None:
+            raise ValueError("No API key found. Please set DATAMULE_API_KEY environment variable or provide api_key in constructor")
+        
+        # Extract pagination and display options
+        page_size = kwargs.pop('page_size', 25000)
+        quiet = kwargs.pop('quiet', False)
+        
+        # Determine database from table
+        if table == 'simple_xbrl':
+            database = 'xbrl_db'
+        elif table == 'accession_cik':
+            database = 'lookup_db'
+        elif table == 'submission_details':
+            database = 'lookup_db'
+        else:
+            raise ValueError(f"Unsupported table: {table}")
+        
+        # Process filters: tuples = range, lists = OR, single = exact
+        filters = {}
+        for key, value in kwargs.items():
+            # Skip None values entirely
+            if value is None:
+                continue
+                
+            # Special logic for cik
+            if key == 'cik':
+                if isinstance(value, list):
+                    value = [int(val) for val in value]
+                else:
+                    value = [int(value)]
+                filters[key] = {"type": "or", "values": value}
+            elif isinstance(value, tuple):
+                filters[key] = {"type": "range", "values": list(value)}
+            elif isinstance(value, list):
+                filters[key] = {"type": "or", "values": value}
+            else:
+                filters[key] = {"type": "or", "values": [value]}
+        
+        self.start_time = time.time()
+        total_items = 0
+        pages_processed = 0
+        
+        # Display query parameters
+        query_desc = [f"Table={table}"]
+        for key, filter_obj in filters.items():
+            if filter_obj["type"] == "range":
+                query_desc.append(f"{key}={filter_obj['values'][0]} to {filter_obj['values'][1]}")
+            elif len(filter_obj["values"]) == 1:
+                query_desc.append(f"{key}={filter_obj['values'][0]}")
+            else:
+                query_desc.append(f"{key}={filter_obj['values']}")
+        
+        if not quiet:
+            print(f"QUERY: {', '.join(query_desc)}")
+        
+        connector = aiohttp.TCPConnector(ssl=ssl.create_default_context())
+        async with aiohttp.ClientSession(connector=connector) as session:
+            # Initialize progress bar only if not quiet
+            if not quiet:
+                pbar = tqdm(unit="page", bar_format="{desc}: {n_fmt} {unit} [{elapsed}<{remaining}, {rate_fmt}{postfix}]")
+                pbar.set_description("Fetching data")
+            
+            current_page = 1
+            has_more = True
+            results = []
+            
+            while has_more:
+                # Fetch page
+                page_results, pagination, page_cost = await self._fetch_page(
+                    session, 
+                    table=table,
+                    database=database,
+                    filters=filters,
+                    page=current_page,
+                    page_size=page_size
+                )
+                
+                # Accumulate results
+                results.extend(page_results)
+                
+                pages_processed += 1
+                total_items += len(page_results)
+                
+                # Update progress bar only if not quiet
+                if not quiet:
+                    pbar.set_description(f"Fetching data (page {current_page})")
+                    pbar.set_postfix_str(f"cost=${self.total_cost:.4f} | balance=${self.remaining_balance:.2f}")
+                    pbar.update(1)
+                
+                # Check if we need to fetch more pages
+                has_more = pagination.get('hasMore', False)
+                current_page += 1
+                
+                # For the first page, display record info only if not quiet
+                if pages_processed == 1 and not quiet:
+                    records_per_page = pagination.get('currentPageRecords', len(page_results))
+                    if records_per_page > 0:
+                        pbar.write(f"Retrieved {records_per_page} records (page 1) - Fetching additional pages...")
+                    else:
+                        pbar.write("No records found matching criteria")
+                        break
+            
+            if not quiet:
+                pbar.close()
+            
+            # Final summary only if not quiet
+            if not quiet:
+                elapsed_time = time.time() - self.start_time
+                print("\nQuery complete:")
+                print(f"- Retrieved {total_items} records across {pages_processed} pages")
+                print(f"- Total cost: ${self.total_cost:.4f}")
+                print(f"- Remaining balance: ${self.remaining_balance:.2f}")
+                print(f"- Time: {elapsed_time:.1f} seconds")
+            
+            return results
+
+
+def query_mysql_rds(table, api_key=None, **kwargs):
+    """
+    Query MySQL RDS data from Datamule with optional filtering and automatic pagination
+    
+    Parameters:
+    - table: Table name (e.g., 'simple_xbrl')
+    - cik: Company CIK number(s), can be int, string, or list
+    - Any other filter parameters as keyword arguments
+    - page_size: Number of records per page (max 25000, default 25000)
+    - quiet: Boolean, whether to suppress progress output and summary (default False)
+    - api_key: Optional API key (can also use DATAMULE_API_KEY environment variable)
+    
+    Filter value types:
+    - Single value: Exact match
+    - List: OR condition (any of the values)
+    - Tuple: Range condition (between first and second values)
+    
+    Returns:
+    - List of dictionaries containing the requested data (ready for pandas DataFrame)
+    """
+    # For backwards compatibility, handle non-paginated single requests
+    if kwargs.get('_single_page', False):
+        # Remove the flag and use original synchronous implementation
+        kwargs.pop('_single_page')
+        return _query_mysql_rds_single(table, api_key, **kwargs)
+    
+    # Create a DatamuleMySQL instance for this request
+    dm = DatamuleMySQL(api_key=api_key)
+    
+    # Run the paginated query and return results
+    return asyncio.run(dm.execute_query(table=table, **kwargs))
+
+
+def _query_mysql_rds_single(table, api_key=None, **kwargs):
+    """Original synchronous implementation for single page requests"""
+    import urllib.request
+    import urllib.error
+    
+    endpoint_url = "https://datamule-mysql-rds.jgfriedman99.workers.dev"
+    
+    # Get API key from parameter or environment
+    if api_key is None:
+        api_key = os.getenv('DATAMULE_API_KEY')
+    
+    if not api_key:
+        return {"error": "API key required. Pass api_key parameter or set DATAMULE_API_KEY environment variable"}
+
+    # Process filters: tuples = range, lists = OR, single = exact
+    filters = {}
+    for key, value in kwargs.items():
+        # Skip None values entirely
+        if value is None:
+            continue
+            
+        # special logic for cik
+        if key == 'cik':
+            if isinstance(value, list):
+                value = [int(val) for val in value]
+            else:
+                value = [int(value)]
+            filters[key] = {"type": "or", "values": value}
+        elif isinstance(value, tuple):
+            filters[key] = {"type": "range", "values": list(value)}
+        elif isinstance(value, list):
+            filters[key] = {"type": "or", "values": value}
+        else:
+            filters[key] = {"type": "or", "values": [value]}
+    
+    payload = {"filters": filters}
+    # add table to payload
+    payload['table'] = table
+
+    if table == 'simple_xbrl':
+        payload['database'] = 'xbrl_db'
+    else:
+        raise ValueError("table not found")
+
+    data = json.dumps(payload).encode('utf-8')
+    req = urllib.request.Request(
+        endpoint_url,
+        data=data,
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {api_key}",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+        }
+    )
+
+    try:
+        with urllib.request.urlopen(req, timeout=6000) as response:
+            result = json.loads(response.read().decode('utf-8'))
+            # Return just the data for single page requests
+            return result.get('data', []) if result.get('success') else result
+    except urllib.error.HTTPError as e:
+        # Print the error response body
+        error_body = e.read().decode('utf-8')
+        print(f"HTTP Error {e.code}: {error_body}")
+        try:
+            error_json = json.loads(error_body)
+            print(f"Error details: {error_json}")
+        except json.JSONDecodeError:
+            print(f"Raw error response: {error_body}")
+        raise

{datamule-1.8.6 → datamule-2.0.0}/datamule/document/document.py

@@ -13,6 +13,7 @@ from pathlib import Path
 import webbrowser
 from secsgml.utils import bytes_to_str
 from secxbrl import parse_inline_xbrl
+from company_fundamentals import construct_fundamentals
 
 class Document:
     def __init__(self, type, content, extension,accession,filing_date,path=None):
@@ -35,6 +36,7 @@ class Document:
         # this will be filled by parsed
         self.data = None
         self.xbrl = None
+        self.fundamentals = None
 
     #_load_text_content
     def _preprocess_txt_content(self):
@@ -113,6 +115,59 @@ class Document:
             self.xbrl = parse_inline_xbrl(self.content)
         else:
             raise ValueError("Only inline has been implemented so far.")
+        
+    def parse_fundamentals(self,categories=None):
+        self.parse_xbrl()
+        # Transform XBRL records into the format needed by construct_fundamentals
+        xbrl = []
+        
+        for xbrl_record in self.xbrl:
+            try:
+                # Extract basic fields
+                value = xbrl_record.get('_val', None)
+                taxonomy, name = xbrl_record['_attributes']['name'].split(':')
+                
+                # Handle scaling if present
+                if xbrl_record.get('_attributes', {}).get('scale') is not None:
+                    scale = int(xbrl_record['_attributes']['scale'])
+                    try:
+                        value = str(Decimal(value.replace(',', '')) * (Decimal(10) ** scale))
+                    except:
+                        pass
+                
+                # Extract period dates
+                period_start_date = None
+                period_end_date = None
+                
+                if xbrl_record.get('_context'):
+                    context = xbrl_record['_context']
+                    period_start_date = context.get('context_period_instant') or context.get('context_period_startdate')
+                    period_end_date = context.get('context_period_enddate')
+                
+                # Create record in the format expected by construct_fundamentals
+                record = {
+                    'taxonomy': taxonomy,
+                    'name': name,
+                    'value': value,
+                    'period_start_date': period_start_date,
+                    'period_end_date': period_end_date
+                }
+                
+                xbrl.append(record)
+                
+            except Exception as e:
+                # Skip malformed records
+                continue
+        
+        # Call construct_fundamentals with the transformed data
+        fundamentals = construct_fundamentals(xbrl, 
+                            taxonomy_key='taxonomy', 
+                            concept_key='name', 
+                            start_date_key='period_start_date', 
+                            end_date_key='period_end_date',
+                            categories=categories)
+        
+        self.fundamentals = fundamentals
 
     # Note: this method will be heavily modified in the future
     def parse(self):

{datamule-1.8.6 → datamule-2.0.0}/datamule/portfolio.py

@@ -9,14 +9,11 @@ import os
 import tarfile
 from threading import Lock
 from .helper import _process_cik_and_metadata_filters
-from .seclibrary.downloader import download as seclibrary_download
+from .datamule.downloader import download as seclibrary_download
 from .sec.xbrl.filter_xbrl import filter_xbrl
 from .sec.submissions.monitor import Monitor
 from .portfolio_compression_utils import CompressionManager
 from .datamule.sec_connector import SecConnector
-from secsgml.utils import bytes_to_str, calculate_documents_locations_in_tar
-import json
-import io
 import shutil
 
 

{datamule-1.8.6 → datamule-2.0.0}/datamule/sheet.py

@@ -3,8 +3,8 @@ import csv
 import os
 from .helper import _process_cik_and_metadata_filters, load_package_dataset
 from .sec.xbrl.downloadcompanyfacts import download_company_facts
-from .seclibrary.datamule_lookup import datamule_lookup
-
+from .datamule.datamule_lookup import datamule_lookup
+from .datamule.datamule_mysql_rds import query_mysql_rds
 # slated for deprecation?
 from .seclibrary.bq import get_information_table, get_345, get_proxy_voting_record
 
@@ -12,11 +12,16 @@ class Sheet:
     def __init__(self, path):
         self.path = Path(path)
 
+    # Keep
     def get_submissions(self,cik=None, accession_number=None, submission_type=None, filing_date=None, 
                    columns=None, distinct=False, page_size=25000, quiet=False, api_key=None):
         
         return datamule_lookup(cik, accession_number, submission_type, filing_date, 
                    columns, distinct, page_size, quiet, api_key)
+    
+    def get_table(self,table,cik=None,ticker=None,**kwargs):
+        cik = _process_cik_and_metadata_filters(cik, ticker)
+        return query_mysql_rds(table=table,cik=cik,**kwargs)
 
     def download_xbrl(
         self, 

{datamule-1.8.6 → datamule-2.0.0}/datamule/submission.py

@@ -10,8 +10,6 @@ import zstandard as zstd
 import gzip
 import urllib.request
 
-
-
 class Submission:
     def __init__(self, path=None, sgml_content=None, keep_document_types=None,
                  batch_tar_path=None, accession_prefix=None, portfolio_ref=None,url=None):

{datamule-1.8.6 → datamule-2.0.0}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.8.6
+Version: 2.0.0
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
@@ -18,3 +18,4 @@ Requires-Dist: doc2dict
 Requires-Dist: secxbrl
 Requires-Dist: secsgml
 Requires-Dist: websocket-client
+Requires-Dist: company_fundamentals

{datamule-1.8.6 → datamule-2.0.0}/datamule.egg-info/SOURCES.txt

@@ -15,6 +15,9 @@ datamule.egg-info/requires.txt
 datamule.egg-info/top_level.txt
 datamule/data/listed_filer_metadata.csv
 datamule/datamule/__init__.py
+datamule/datamule/datamule_lookup.py
+datamule/datamule/datamule_mysql_rds.py
+datamule/datamule/downloader.py
 datamule/datamule/sec_connector.py
 datamule/document/__init__.py
 datamule/document/document.py
@@ -65,9 +68,6 @@ datamule/sec/xbrl/streamcompanyfacts.py
 datamule/sec/xbrl/xbrlmonitor.py
 datamule/seclibrary/__init__.py
 datamule/seclibrary/bq.py
-datamule/seclibrary/datamule_lookup.py
-datamule/seclibrary/downloader.py
-datamule/seclibrary/query.py
 datamule/utils/__init__.py
 datamule/utils/construct_submissions_data.py
 datamule/utils/format_accession.py

{datamule-1.8.6 → datamule-2.0.0}/datamule.egg-info/requires.txt

@@ -12,3 +12,4 @@ doc2dict
 secxbrl
 secsgml
 websocket-client
+company_fundamentals

{datamule-1.8.6 → datamule-2.0.0}/setup.py

@@ -32,7 +32,7 @@ if not os.path.exists(file_path):
 setup(
     name="datamule",
     author="John Friedman",
-    version="1.8.6",
+    version="2.0.0",
     description="Work with SEC submissions at scale.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",
@@ -51,6 +51,7 @@ setup(
         'secxbrl',
         'secsgml',
         'websocket-client',
+        'company_fundamentals'
     ],
     # Include the data directory in the package
     package_data={

datamule-1.8.6/datamule/seclibrary/query.py

@@ -1,181 +0,0 @@
-import os
-import asyncio
-import aiohttp
-import urllib.parse
-import ssl
-import json
-import time
-from tqdm import tqdm
-
-class Query:
-    def __init__(self, api_key=None):
-        self.API_BASE_URL = "https://sec-library.jgfriedman99.workers.dev/"
-        self._api_key = api_key
-        self.total_cost = 0
-        self.remaining_balance = None
-        self.start_time = None
-
-    @property
-    def api_key(self):
-        return getattr(self, '_api_key', None) or os.getenv('DATAMULE_API_KEY')
-
-    @api_key.setter
-    def api_key(self, value):
-        if not value:
-            raise ValueError("API key cannot be empty")
-        self._api_key = value
-
-    async def _fetch_page(self, session, submission_type=None, cik=None, filing_date=None, page=1):
-        params = {
-            'api_key': self.api_key,
-            'page': page
-        }
-        
-        # Handle submission_type parameter
-        if submission_type:
-            if isinstance(submission_type, list):
-                params['submission_type'] = ','.join(str(x) for x in submission_type)
-            else:
-                params['submission_type'] = str(submission_type)
-        
-        # Handle CIK parameter
-        if cik:
-            if isinstance(cik, list):
-                params['cik'] = ','.join(str(x) for x in cik)
-            else:
-                params['cik'] = str(cik)
-        
-        # Handle filing_date parameter
-        if filing_date:
-            if isinstance(filing_date, tuple):
-                params['startdt'] = str(filing_date[0])
-                params['enddt'] = str(filing_date[1])
-            else:
-                if isinstance(filing_date, list):
-                    params['filing_date'] = ','.join(str(x) for x in filing_date)
-                else:
-                    params['filing_date'] = str(filing_date)
-
-        url = f"{self.API_BASE_URL}?{urllib.parse.urlencode(params)}"
-        
-        async with session.get(url) as response:
-            data = await response.json()
-            if not data.get('success'):
-                raise ValueError(f"API request failed: {data.get('error')}")
-            
-            # Track costs and balance
-            charges = data['metadata']['billing']['charges']
-            page_cost = charges['total']
-            self.total_cost += page_cost
-            self.remaining_balance = data['metadata']['billing']['remaining_balance']
-            
-            return data['data'], data['metadata']['pagination'], page_cost
-
-    async def execute_query(self, submission_type=None, cik=None, filing_date=None):
-        if self.api_key is None:
-            raise ValueError("No API key found. Please set DATAMULE_API_KEY environment variable or provide api_key in constructor")
-        
-        self.start_time = time.time()
-        total_items = 0
-        pages_processed = 0
-        
-        # Display query parameters
-        query_desc = []
-        if cik:
-            query_desc.append(f"CIK={cik}")
-        if submission_type:
-            query_desc.append(f"Type={submission_type}")
-        if filing_date:
-            if isinstance(filing_date, tuple):
-                query_desc.append(f"Date={filing_date[0]} to {filing_date[1]}")
-            else:
-                query_desc.append(f"Date={filing_date}")
-        
-        if query_desc:
-            print(f"QUERY: {', '.join(query_desc)}")
-        
-        connector = aiohttp.TCPConnector(ssl=ssl.create_default_context())
-        async with aiohttp.ClientSession(connector=connector) as session:
-            # Initialize progress bar with a custom format to avoid extra colons
-            pbar = tqdm(unit="page", bar_format="{desc}: {n_fmt} {unit} [{elapsed}<{remaining}, {rate_fmt}{postfix}]")
-            pbar.set_description("Fetching data")
-            
-            current_page = 1
-            has_more = True
-            results = []
-            
-            while has_more:
-                # Fetch page
-                page_results, pagination, page_cost = await self._fetch_page(session, 
-                                                                    submission_type=submission_type, 
-                                                                    cik=cik, 
-                                                                    filing_date=filing_date, 
-                                                                    page=current_page)
-                
-                # Accumulate results
-                results.extend(page_results)
-                
-                pages_processed += 1
-                total_items += len(page_results)
-                
-                # Update progress bar with cleaner format
-                pbar.set_description(f"Fetching data (page {current_page})")
-                pbar.set_postfix_str(f"cost=${self.total_cost:.2f} | balance=${self.remaining_balance:.2f}")
-                pbar.update(1)
-                
-                # Check if we need to fetch more pages
-                has_more = pagination.get('hasMore', False)
-                current_page += 1
-                
-                # For the first page, display record info using pbar.write instead of print
-                if pages_processed == 1:
-                    records_per_page = pagination.get('currentPageRecords', len(page_results))
-                    total_records = pagination.get('totalRecords', None)
-                    if total_records:
-                        pbar.write(f"Retrieved {records_per_page} records (page 1) of {total_records} total - Fetching additional pages...")
-                    else:
-                        pbar.write(f"Retrieved {records_per_page} records (page 1) - Fetching additional pages...")
-            
-            pbar.close()
-            
-            # Final summary
-            elapsed_time = time.time() - self.start_time
-            print("\nQuery complete:")
-            print(f"- Retrieved {total_items} filings across {pages_processed} pages")
-            print(f"- Total cost: ${self.total_cost:.2f}")
-            print(f"- Remaining balance: ${self.remaining_balance:.2f}")
-            print(f"- Time: {elapsed_time:.1f} seconds")
-            
-            return results
-
-
-def query(cik=None, submission_type=None, filing_date=None, api_key=None):
-    """
-    Query SEC filings data with optional filtering
-    
-    Parameters:
-    - cik: Company CIK number(s), can be string, int, or list
-    - submission_type: Filing type(s), can be string or list (e.g., '10-K', ['10-K', '10-Q'])
-    - filing_date: Filing date(s), can be string, list, or tuple of (start_date, end_date)
-    - api_key: Optional API key (can also use DATAMULE_API_KEY environment variable)
-    
-    Returns:
-    - List of all matching submission data
-    """
-    # Create a Query instance for this request
-    q = Query(api_key=api_key)
-    # remove dash from filing_date
-    if isinstance(filing_date, tuple):
-        filing_date = (filing_date[0].replace('-', ''), filing_date[1].replace('-', ''))
-    elif isinstance(filing_date, str):
-        filing_date = filing_date.replace('-', '')
-    elif isinstance(filing_date, list):
-        filing_date = [x.replace('-', '') for x in filing_date]
-
-    print(filing_date)
-    # Run the query and return results
-    return asyncio.run(q.execute_query(
-        submission_type=submission_type,
-        cik=cik,
-        filing_date=filing_date
-    ))