datamule 1.8.5__tar.gz → 1.9.0__tar.gz

{datamule-1.8.5 → datamule-1.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.8.5
+Version: 1.9.0
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
@@ -18,3 +18,4 @@ Requires-Dist: doc2dict
 Requires-Dist: secxbrl
 Requires-Dist: secsgml
 Requires-Dist: websocket-client
+Requires-Dist: company_fundamentals

{datamule-1.8.5/datamule/seclibrary → datamule-1.9.0/datamule/datamule}/datamule_lookup.py

@@ -3,7 +3,6 @@ import asyncio
 import aiohttp
 import urllib.parse
 import ssl
-import json
 import time
 from tqdm import tqdm
 

datamule-1.9.0/datamule/datamule/datamule_mysql_rds.py

@@ -0,0 +1,3 @@
+# connection to worker
+# basically everything should be handled on worker end
+# except for like dates. - nah even dates

{datamule-1.8.5 → datamule-1.9.0}/datamule/datamule/sec_connector.py

@@ -23,6 +23,8 @@ class SecConnector:
         req = urllib.request.Request(url, method='GET')
         req.add_header('Accept', 'application/json')
         req.add_header('Authorization', f'Bearer {self.api_key}')  # API key in header
+        req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
+
         
         with urllib.request.urlopen(req) as response:
             data = json.loads(response.read().decode())

{datamule-1.8.5 → datamule-1.9.0}/datamule/document/document.py

@@ -13,6 +13,7 @@ from pathlib import Path
 import webbrowser
 from secsgml.utils import bytes_to_str
 from secxbrl import parse_inline_xbrl
+from company_fundamentals import construct_fundamentals
 
 class Document:
     def __init__(self, type, content, extension,accession,filing_date,path=None):
@@ -35,6 +36,7 @@ class Document:
         # this will be filled by parsed
         self.data = None
         self.xbrl = None
+        self.fundamentals = None
 
     #_load_text_content
     def _preprocess_txt_content(self):
@@ -113,6 +115,59 @@ class Document:
             self.xbrl = parse_inline_xbrl(self.content)
         else:
             raise ValueError("Only inline has been implemented so far.")
+        
+    def parse_fundamentals(self,categories=None):
+        self.parse_xbrl()
+        # Transform XBRL records into the format needed by construct_fundamentals
+        xbrl = []
+        
+        for xbrl_record in self.xbrl:
+            try:
+                # Extract basic fields
+                value = xbrl_record.get('_val', None)
+                taxonomy, name = xbrl_record['_attributes']['name'].split(':')
+                
+                # Handle scaling if present
+                if xbrl_record.get('_attributes', {}).get('scale') is not None:
+                    scale = int(xbrl_record['_attributes']['scale'])
+                    try:
+                        value = str(Decimal(value.replace(',', '')) * (Decimal(10) ** scale))
+                    except:
+                        pass
+                
+                # Extract period dates
+                period_start_date = None
+                period_end_date = None
+                
+                if xbrl_record.get('_context'):
+                    context = xbrl_record['_context']
+                    period_start_date = context.get('context_period_instant') or context.get('context_period_startdate')
+                    period_end_date = context.get('context_period_enddate')
+                
+                # Create record in the format expected by construct_fundamentals
+                record = {
+                    'taxonomy': taxonomy,
+                    'name': name,
+                    'value': value,
+                    'period_start_date': period_start_date,
+                    'period_end_date': period_end_date
+                }
+                
+                xbrl.append(record)
+                
+            except Exception as e:
+                # Skip malformed records
+                continue
+        
+        # Call construct_fundamentals with the transformed data
+        fundamentals = construct_fundamentals(xbrl, 
+                            taxonomy_key='taxonomy', 
+                            concept_key='name', 
+                            start_date_key='period_start_date', 
+                            end_date_key='period_end_date',
+                            categories=categories)
+        
+        self.fundamentals = fundamentals
 
     # Note: this method will be heavily modified in the future
     def parse(self):

{datamule-1.8.5 → datamule-1.9.0}/datamule/portfolio.py

@@ -9,14 +9,11 @@ import os
 import tarfile
 from threading import Lock
 from .helper import _process_cik_and_metadata_filters
-from .seclibrary.downloader import download as seclibrary_download
+from .datamule.downloader import download as seclibrary_download
 from .sec.xbrl.filter_xbrl import filter_xbrl
 from .sec.submissions.monitor import Monitor
 from .portfolio_compression_utils import CompressionManager
 from .datamule.sec_connector import SecConnector
-from secsgml.utils import bytes_to_str, calculate_documents_locations_in_tar
-import json
-import io
 import shutil
 
 

{datamule-1.8.5 → datamule-1.9.0}/datamule/sheet.py

@@ -3,7 +3,7 @@ import csv
 import os
 from .helper import _process_cik_and_metadata_filters, load_package_dataset
 from .sec.xbrl.downloadcompanyfacts import download_company_facts
-from .seclibrary.datamule_lookup import datamule_lookup
+from .datamule.datamule_lookup import datamule_lookup
 
 # slated for deprecation?
 from .seclibrary.bq import get_information_table, get_345, get_proxy_voting_record
@@ -12,11 +12,16 @@ class Sheet:
     def __init__(self, path):
         self.path = Path(path)
 
+    # Keep
     def get_submissions(self,cik=None, accession_number=None, submission_type=None, filing_date=None, 
                    columns=None, distinct=False, page_size=25000, quiet=False, api_key=None):
         
         return datamule_lookup(cik, accession_number, submission_type, filing_date, 
                    columns, distinct, page_size, quiet, api_key)
+    
+    # Implement
+    def get_table(self,table,**kwargs):
+        pass
 
     def download_xbrl(
         self, 

{datamule-1.8.5 → datamule-1.9.0}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.8.5
+Version: 1.9.0
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
@@ -18,3 +18,4 @@ Requires-Dist: doc2dict
 Requires-Dist: secxbrl
 Requires-Dist: secsgml
 Requires-Dist: websocket-client
+Requires-Dist: company_fundamentals

{datamule-1.8.5 → datamule-1.9.0}/datamule.egg-info/SOURCES.txt

@@ -15,6 +15,9 @@ datamule.egg-info/requires.txt
 datamule.egg-info/top_level.txt
 datamule/data/listed_filer_metadata.csv
 datamule/datamule/__init__.py
+datamule/datamule/datamule_lookup.py
+datamule/datamule/datamule_mysql_rds.py
+datamule/datamule/downloader.py
 datamule/datamule/sec_connector.py
 datamule/document/__init__.py
 datamule/document/document.py
@@ -65,9 +68,6 @@ datamule/sec/xbrl/streamcompanyfacts.py
 datamule/sec/xbrl/xbrlmonitor.py
 datamule/seclibrary/__init__.py
 datamule/seclibrary/bq.py
-datamule/seclibrary/datamule_lookup.py
-datamule/seclibrary/downloader.py
-datamule/seclibrary/query.py
 datamule/utils/__init__.py
 datamule/utils/construct_submissions_data.py
 datamule/utils/format_accession.py

{datamule-1.8.5 → datamule-1.9.0}/datamule.egg-info/requires.txt

@@ -12,3 +12,4 @@ doc2dict
 secxbrl
 secsgml
 websocket-client
+company_fundamentals

{datamule-1.8.5 → datamule-1.9.0}/setup.py

@@ -32,7 +32,7 @@ if not os.path.exists(file_path):
 setup(
     name="datamule",
     author="John Friedman",
-    version="1.8.5",
+    version="1.9.0",
     description="Work with SEC submissions at scale.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",
@@ -51,6 +51,7 @@ setup(
         'secxbrl',
         'secsgml',
         'websocket-client',
+        'company_fundamentals'
     ],
     # Include the data directory in the package
     package_data={

datamule-1.8.5/datamule/seclibrary/query.py

@@ -1,181 +0,0 @@
-import os
-import asyncio
-import aiohttp
-import urllib.parse
-import ssl
-import json
-import time
-from tqdm import tqdm
-
-class Query:
-    def __init__(self, api_key=None):
-        self.API_BASE_URL = "https://sec-library.jgfriedman99.workers.dev/"
-        self._api_key = api_key
-        self.total_cost = 0
-        self.remaining_balance = None
-        self.start_time = None
-
-    @property
-    def api_key(self):
-        return getattr(self, '_api_key', None) or os.getenv('DATAMULE_API_KEY')
-
-    @api_key.setter
-    def api_key(self, value):
-        if not value:
-            raise ValueError("API key cannot be empty")
-        self._api_key = value
-
-    async def _fetch_page(self, session, submission_type=None, cik=None, filing_date=None, page=1):
-        params = {
-            'api_key': self.api_key,
-            'page': page
-        }
-        
-        # Handle submission_type parameter
-        if submission_type:
-            if isinstance(submission_type, list):
-                params['submission_type'] = ','.join(str(x) for x in submission_type)
-            else:
-                params['submission_type'] = str(submission_type)
-        
-        # Handle CIK parameter
-        if cik:
-            if isinstance(cik, list):
-                params['cik'] = ','.join(str(x) for x in cik)
-            else:
-                params['cik'] = str(cik)
-        
-        # Handle filing_date parameter
-        if filing_date:
-            if isinstance(filing_date, tuple):
-                params['startdt'] = str(filing_date[0])
-                params['enddt'] = str(filing_date[1])
-            else:
-                if isinstance(filing_date, list):
-                    params['filing_date'] = ','.join(str(x) for x in filing_date)
-                else:
-                    params['filing_date'] = str(filing_date)
-
-        url = f"{self.API_BASE_URL}?{urllib.parse.urlencode(params)}"
-        
-        async with session.get(url) as response:
-            data = await response.json()
-            if not data.get('success'):
-                raise ValueError(f"API request failed: {data.get('error')}")
-            
-            # Track costs and balance
-            charges = data['metadata']['billing']['charges']
-            page_cost = charges['total']
-            self.total_cost += page_cost
-            self.remaining_balance = data['metadata']['billing']['remaining_balance']
-            
-            return data['data'], data['metadata']['pagination'], page_cost
-
-    async def execute_query(self, submission_type=None, cik=None, filing_date=None):
-        if self.api_key is None:
-            raise ValueError("No API key found. Please set DATAMULE_API_KEY environment variable or provide api_key in constructor")
-        
-        self.start_time = time.time()
-        total_items = 0
-        pages_processed = 0
-        
-        # Display query parameters
-        query_desc = []
-        if cik:
-            query_desc.append(f"CIK={cik}")
-        if submission_type:
-            query_desc.append(f"Type={submission_type}")
-        if filing_date:
-            if isinstance(filing_date, tuple):
-                query_desc.append(f"Date={filing_date[0]} to {filing_date[1]}")
-            else:
-                query_desc.append(f"Date={filing_date}")
-        
-        if query_desc:
-            print(f"QUERY: {', '.join(query_desc)}")
-        
-        connector = aiohttp.TCPConnector(ssl=ssl.create_default_context())
-        async with aiohttp.ClientSession(connector=connector) as session:
-            # Initialize progress bar with a custom format to avoid extra colons
-            pbar = tqdm(unit="page", bar_format="{desc}: {n_fmt} {unit} [{elapsed}<{remaining}, {rate_fmt}{postfix}]")
-            pbar.set_description("Fetching data")
-            
-            current_page = 1
-            has_more = True
-            results = []
-            
-            while has_more:
-                # Fetch page
-                page_results, pagination, page_cost = await self._fetch_page(session, 
-                                                                    submission_type=submission_type, 
-                                                                    cik=cik, 
-                                                                    filing_date=filing_date, 
-                                                                    page=current_page)
-                
-                # Accumulate results
-                results.extend(page_results)
-                
-                pages_processed += 1
-                total_items += len(page_results)
-                
-                # Update progress bar with cleaner format
-                pbar.set_description(f"Fetching data (page {current_page})")
-                pbar.set_postfix_str(f"cost=${self.total_cost:.2f} | balance=${self.remaining_balance:.2f}")
-                pbar.update(1)
-                
-                # Check if we need to fetch more pages
-                has_more = pagination.get('hasMore', False)
-                current_page += 1
-                
-                # For the first page, display record info using pbar.write instead of print
-                if pages_processed == 1:
-                    records_per_page = pagination.get('currentPageRecords', len(page_results))
-                    total_records = pagination.get('totalRecords', None)
-                    if total_records:
-                        pbar.write(f"Retrieved {records_per_page} records (page 1) of {total_records} total - Fetching additional pages...")
-                    else:
-                        pbar.write(f"Retrieved {records_per_page} records (page 1) - Fetching additional pages...")
-            
-            pbar.close()
-            
-            # Final summary
-            elapsed_time = time.time() - self.start_time
-            print("\nQuery complete:")
-            print(f"- Retrieved {total_items} filings across {pages_processed} pages")
-            print(f"- Total cost: ${self.total_cost:.2f}")
-            print(f"- Remaining balance: ${self.remaining_balance:.2f}")
-            print(f"- Time: {elapsed_time:.1f} seconds")
-            
-            return results
-
-
-def query(cik=None, submission_type=None, filing_date=None, api_key=None):
-    """
-    Query SEC filings data with optional filtering
-    
-    Parameters:
-    - cik: Company CIK number(s), can be string, int, or list
-    - submission_type: Filing type(s), can be string or list (e.g., '10-K', ['10-K', '10-Q'])
-    - filing_date: Filing date(s), can be string, list, or tuple of (start_date, end_date)
-    - api_key: Optional API key (can also use DATAMULE_API_KEY environment variable)
-    
-    Returns:
-    - List of all matching submission data
-    """
-    # Create a Query instance for this request
-    q = Query(api_key=api_key)
-    # remove dash from filing_date
-    if isinstance(filing_date, tuple):
-        filing_date = (filing_date[0].replace('-', ''), filing_date[1].replace('-', ''))
-    elif isinstance(filing_date, str):
-        filing_date = filing_date.replace('-', '')
-    elif isinstance(filing_date, list):
-        filing_date = [x.replace('-', '') for x in filing_date]
-
-    print(filing_date)
-    # Run the query and return results
-    return asyncio.run(q.execute_query(
-        submission_type=submission_type,
-        cik=cik,
-        filing_date=filing_date
-    ))