datamule 1.0.3__tar.gz → 1.0.7__tar.gz

{datamule-1.0.3 → datamule-1.0.7}/PKG-INFO

@@ -1,9 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.0.3
+Version: 1.0.7
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
-Provides-Extra: mulebot
-Provides-Extra: mulebot_server
-Provides-Extra: all

{datamule-1.0.3 → datamule-1.0.7}/datamule/__init__.py

@@ -1,12 +1,7 @@
-from .downloader.downloader import Downloader
-from .downloader.premiumdownloader import PremiumDownloader
-from .monitor import Monitor
-from .packageupdater import PackageUpdater
 from .submission import Submission
 from .portfolio import Portfolio
 from .document import Document
-from secsgml import parse_sgml_submission
-from .helper import load_package_csv, load_package_dataset
+from .helper import _load_package_csv, load_package_dataset
 from .config import Config
 
 
@@ -32,16 +27,10 @@ def _setup_notebook_env():
 _setup_notebook_env()
 
 __all__ = [
-    'Downloader',
-    'PremiumDownloader',
-    'load_package_csv',
+    '_load_package_csv',
     'load_package_dataset',
-    'Filing',
     'Portfolio',
-    'Monitor',
-    'PackageUpdater',
     'Submission',
     'Document',
-    'parse_sgml_submission',
     'Config'
 ]

{datamule-1.0.3 → datamule-1.0.7}/datamule/document.py

@@ -1,11 +1,10 @@
 import json
 import csv
-from .helper import convert_to_dashed_accession
 import re
 from doc2dict import xml2dict, txt2dict, dict2dict
 from doc2dict.mapping import flatten_hierarchy
-from .mapping_dicts import txt_mapping_dicts
-from .mapping_dicts import xml_mapping_dicts
+from .mapping_dicts.txt_mapping_dicts import dict_10k, dict_10q, dict_8k, dict_13d, dict_13g
+from .mapping_dicts.xml_mapping_dicts import dict_345
 from selectolax.parser import HTMLParser
 
 class Document:
@@ -107,7 +106,7 @@ class Document:
 
         if self.path.suffix == '.xml':
             if self.type in ['3', '4', '5']:
-                mapping_dict = xml_mapping_dicts.dict_345
+                mapping_dict = dict_345
 
             self.load_content()
             self.data = xml2dict(content=self.content, mapping_dict=mapping_dict)
@@ -116,15 +115,15 @@ class Document:
             self._load_file_content()
 
             if self.type == '10-K':
-                mapping_dict = txt_mapping_dicts.dict_10k
+                mapping_dict = dict_10k
             elif self.type == '10-Q':
-                mapping_dict = txt_mapping_dicts.dict_10q
+                mapping_dict = dict_10q
             elif self.type == '8-K':
-                mapping_dict = txt_mapping_dicts.dict_8k
+                mapping_dict = dict_8k
             elif self.type == 'SC 13D':
-                mapping_dict = txt_mapping_dicts.dict_13d
+                mapping_dict = dict_13d
             elif self.type == 'SC 13G':
-                mapping_dict = txt_mapping_dicts.dict_13g
+                mapping_dict = dict_13g
             
             self.data = {}
             self.data['document'] = dict2dict(txt2dict(content=self.content, mapping_dict=mapping_dict))

datamule-1.0.7/datamule/helper.py

@@ -0,0 +1,103 @@
+from functools import lru_cache
+import csv
+from pathlib import Path
+
+def _load_package_csv(name):
+    """Load CSV files from ~/.datamule/ directory"""
+    data_dir = Path.home() / ".datamule"
+    csv_path = data_dir / f"{name}.csv"
+    
+    data = []
+    
+    with open(csv_path, 'r') as csvfile:
+        csv_reader = csv.DictReader(csvfile)
+        for row in csv_reader:
+            data.append(row)
+    
+    return data
+
+def load_package_dataset(dataset):
+    if dataset =='listed_filer_metadata':
+        return _load_package_csv('listed_filer_metadata')
+
+@lru_cache(maxsize=128)
+def get_cik_from_dataset(dataset_name, key, value):
+    dataset = load_package_dataset(dataset_name)
+    
+    if dataset_name == 'listed_filer_metadata' and key == 'ticker':
+        key = 'tickers'
+    
+    result = []
+    for company in dataset:
+        if key in ['tickers', 'exchanges'] and dataset_name == 'listed_filer_metadata':
+            # Parse the string representation of list into an actual list
+            list_values = [i.strip() for i in company[key][1:-1].replace("'", "").replace('"', '').split(',')]
+            if str(value) in list_values:
+                result.append(company['cik'])
+        elif str(value) == company[key]:
+            result.append(company['cik'])
+    
+    return result
+
+
+
+@lru_cache(maxsize=128)
+def get_ciks_from_metadata_filters(**kwargs):
+    """Get CIKs from listed_filer_metadata.csv that match all provided filters."""
+    
+    # Start with None to get all CIKs from first filter
+    result_ciks = None
+    
+    # For each filter, get matching CIKs and keep intersection
+    for key, value in kwargs.items():
+        # Get CIKs for this filter
+        ciks = get_cik_from_dataset('listed_filer_metadata', key, value)
+        ciks = [int(cik) for cik in ciks]
+        
+        # If this is the first filter, set as initial result
+        if result_ciks is None:
+            result_ciks = set(ciks)
+        # Otherwise, take intersection with previous results
+        else:
+            result_ciks &= set(ciks)
+            
+        # If no matches left, we can exit early
+        if not result_ciks:
+            return []
+    
+    return list(result_ciks)
+
+
+def _process_cik_and_metadata_filters(cik=None, ticker=None, **kwargs):
+        """ 
+        Helper method to process CIK, ticker, and metadata filters.
+        Returns a list of CIKs after processing.
+        """
+        # Input validation
+        if cik is not None and ticker is not None:
+            raise ValueError("Only one of cik or ticker should be provided, not both.")
+
+        # Convert ticker to CIK if provided
+        if ticker is not None:
+            cik = get_cik_from_dataset('listed_filer_metadata', 'ticker', ticker)
+
+        # Normalize CIK format
+        if cik is not None:
+            if isinstance(cik, str):
+                cik = [int(cik)]
+            elif isinstance(cik, int):
+                cik = [cik]
+            elif isinstance(cik, list):
+                cik = [int(x) for x in cik]
+
+        # Process metadata filters if provided
+        if kwargs:
+            metadata_ciks = get_ciks_from_metadata_filters(**kwargs)
+
+            if cik is not None:
+                cik = list(set(cik).intersection(metadata_ciks))
+            else:
+                cik = metadata_ciks
+                
+        return cik
+        

datamule-1.0.7/datamule/portfolio.py

@@ -0,0 +1,182 @@
+from pathlib import Path
+from tqdm import tqdm
+from concurrent.futures import ThreadPoolExecutor
+from .submission import Submission
+from .sec.submissions.downloader import download as sec_download
+from .sec.submissions.textsearch import filter_text
+from .config import Config
+import os
+from .helper import _process_cik_and_metadata_filters
+from .seclibrary.downloader import download as seclibrary_download
+from .sec.xbrl.filter_xbrl import filter_xbrl
+from .sec.submissions.monitor import monitor
+from .sec.xbrl.xbrlmonitor import XBRLMonitor
+
+
+class Portfolio:
+    def __init__(self, path):
+        self.path = Path(path)
+        self.submissions = []
+        self.submissions_loaded = False
+        self.MAX_WORKERS = os.cpu_count() - 1 
+        
+        if self.path.exists():
+            self._load_submissions()
+            self.submissions_loaded = True
+        else:
+            self.path.mkdir(parents=True, exist_ok=True)
+    
+    def _load_submissions(self):
+        folders = [f for f in self.path.iterdir() if f.is_dir()]
+        print(f"Loading {len(folders)} submissions")
+        
+        def load_submission(folder):
+            try:
+                return Submission(folder)
+            except Exception as e:
+                print(f"Error loading submission from {folder}: {str(e)}")
+                return None
+        
+        with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            self.submissions = list(tqdm(
+                executor.map(load_submission, folders),
+                total=len(folders),
+                desc="Loading submissions"
+            ))
+            
+        # Filter out None values from failed submissions
+        self.submissions = [s for s in self.submissions if s is not None]
+        print(f"Successfully loaded {len(self.submissions)} submissions")
+
+    def process_submissions(self, callback):
+        """Process all submissions using a thread pool."""
+        if not self.submissions_loaded:
+            self._load_submissions()
+        with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            results = list(tqdm(
+                executor.map(callback, self.submissions),
+                total=len(self.submissions),
+                desc="Processing submissions"
+            ))
+            return results
+
+    def process_documents(self, callback):
+        """Process all documents using a thread pool."""
+        if not self.submissions_loaded:
+            self._load_submissions()
+
+        documents = [doc for sub in self.submissions for doc in sub]
+        
+        with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
+            results = list(tqdm(
+                executor.map(callback, documents),
+                total=len(documents),
+                desc="Processing documents"
+            ))
+            return results
+    
+    def filter_text(self, text_query, cik=None, ticker=None, submission_type=None, filing_date=None, **kwargs):
+        """
+        Filter text based on query and various parameters.
+        When called multiple times, takes the intersection of results.
+        Now supports metadata filters through kwargs.
+        """
+        # Process CIK and metadata filters
+        cik = _process_cik_and_metadata_filters(cik, ticker, **kwargs)
+        
+        # Call the filter_text function with processed parameters
+        new_accession_numbers = filter_text(
+            text_query=text_query,
+            cik=cik,
+            submission_type=submission_type,
+            filing_date=filing_date
+        )
+        
+        # If we already have accession numbers, take the intersection
+        if hasattr(self, 'accession_numbers') and self.accession_numbers:
+            self.accession_numbers = list(set(self.accession_numbers).intersection(new_accession_numbers))
+        else:
+            # First query, just set the accession numbers
+            self.accession_numbers = new_accession_numbers
+
+    def filter_xbrl(self, taxonomy, concept, unit, period, logic, value):
+        """
+        Filter XBRL data based on logic and value.
+        """
+        new_accession_numbers = filter_xbrl(
+            taxonomy=taxonomy,
+            concept=concept,
+            unit=unit,
+            period=period,
+            logic=logic,
+            value=value
+        )
+        
+        # If we already have accession numbers, take the intersection
+        if hasattr(self, 'accession_numbers') and self.accession_numbers:
+            self.accession_numbers = list(set(self.accession_numbers).intersection(new_accession_numbers))
+        else:
+            # First query, just set the accession numbers
+            self.accession_numbers = new_accession_numbers
+
+    def download_submissions(self, cik=None, ticker=None, submission_type=None, filing_date=None, provider=None, **kwargs):
+        if provider is None:
+            config = Config()
+            provider = config.get_default_source()
+
+        # Process CIK and metadata filters
+        cik = _process_cik_and_metadata_filters(cik, ticker, **kwargs)
+
+        if provider == 'datamule':
+
+            seclibrary_download(
+                output_dir=self.path,
+                cik=cik,
+                submission_type=submission_type,
+                filing_date=filing_date,
+                accession_numbers=self.accession_numbers if hasattr(self, 'accession_numbers') else None
+            )
+        else:
+            sec_download(
+                output_dir=self.path,
+                cik=cik,
+                submission_type=submission_type,
+                filing_date=filing_date,
+                requests_per_second=5, # Revisit this later.
+                accession_numbers=self.accession_numbers if hasattr(self, 'accession_numbers') else None
+            )
+
+        self.submissions_loaded = False
+    def monitor_submissions(self,data_callback=None, poll_callback=None, submission_type=None, cik=None, 
+           polling_interval=200, requests_per_second=5, quiet=False, start_date=None, ticker=None, **kwargs):
+        
+        cik = _process_cik_and_metadata_filters(cik, ticker, **kwargs)
+
+        monitor(
+            data_callback=data_callback,
+            poll_callback=poll_callback,
+            cik=cik,
+            submission_type=submission_type,
+            polling_interval=polling_interval,
+            requests_per_second=requests_per_second,
+            quiet=quiet,
+            start_date=start_date
+        )
+
+        
+        
+        
+    def __iter__(self):
+        if not self.submissions_loaded:
+            self._load_submissions()
+        return iter(self.submissions)
+    
+    def document_type(self, document_types):
+        """Filter documents by type(s)."""
+        if not self.submissions_loaded:
+            self._load_submissions()
+        if isinstance(document_types, str):
+            document_types = [document_types]
+            
+        for submission in self.submissions:
+            yield from submission.document_type(document_types)

datamule-1.0.7/datamule/submission.py

@@ -0,0 +1,38 @@
+from pathlib import Path
+import json
+from .document import Document
+
+class Submission:
+    def __init__(self, path):
+        self.path = Path(path)
+        self._load_metadata()
+    
+    def _load_metadata(self):
+        metadata_path = self.path / 'metadata.json'
+        with metadata_path.open('r') as f:
+            self.metadata = json.load(f)
+
+    def document_type(self, document_type):
+        # Convert single document type to list for consistent handling
+        if isinstance(document_type, str):
+            document_types = [document_type]
+        else:
+            document_types = document_type
+
+        for doc in self.metadata['documents']:
+            if doc['type'] in document_types:
+                filename = doc.get('filename')
+                if filename is None:
+                    continue
+                    
+                document_path = self.path / filename
+                yield Document(doc['type'], document_path)
+    
+    def __iter__(self):
+        for doc in self.metadata['documents']:
+            filename = doc.get('filename')
+            if filename is None:
+                continue
+                
+            document_path = self.path / filename
+            yield Document(doc['type'], document_path)

{datamule-1.0.3 → datamule-1.0.7}/datamule.egg-info/PKG-INFO

@@ -1,9 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.0.3
+Version: 1.0.7
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman
-Provides-Extra: mulebot
-Provides-Extra: mulebot_server
-Provides-Extra: all

datamule-1.0.7/datamule.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+setup.py
+datamule/__init__.py
+datamule/config.py
+datamule/document.py
+datamule/helper.py
+datamule/portfolio.py
+datamule/submission.py
+datamule.egg-info/PKG-INFO
+datamule.egg-info/SOURCES.txt
+datamule.egg-info/dependency_links.txt
+datamule.egg-info/requires.txt
+datamule.egg-info/top_level.txt

{datamule-1.0.3 → datamule-1.0.7}/datamule.egg-info/requires.txt

@@ -11,13 +11,4 @@ pytz
 zstandard
 doc2dict
 secsgml
-
-[all]
-openai
-flask
-
-[mulebot]
-openai
-
-[mulebot_server]
-flask
+lxml

datamule-1.0.7/setup.py

@@ -0,0 +1,52 @@
+from setuptools import setup
+from setuptools import find_namespace_packages
+import os
+import gzip
+import shutil
+import urllib.request
+from pathlib import Path
+
+# Create data directory in user's home
+data_dir = Path.home() / ".datamule"
+data_dir.mkdir(exist_ok=True)
+
+# Download data file
+file_url = "https://github.com/john-friedman/datamule-data/raw/master/data/filer_metadata/listed_filer_metadata.csv.gz"
+file_path = data_dir / "listed_filer_metadata.csv"
+temp_gz_path = data_dir / "listed_filer_metadata.csv.gz"
+
+if not file_path.exists():
+    print(f"Downloading data to {data_dir}")
+    urllib.request.urlretrieve(file_url, temp_gz_path)
+    
+    with gzip.open(temp_gz_path, 'rb') as f_in:
+        with open(file_path, 'wb') as f_out:
+            shutil.copyfileobj(f_in, f_out)
+            
+    os.remove(temp_gz_path)
+    print(f"Data downloaded to {file_path}")
+
+setup(
+    name="datamule",
+    author="John Friedman",
+    version="1.0.7",
+    description="Making it easier to use SEC filings.",
+    packages=find_namespace_packages(include=['datamule']),
+    url="https://github.com/john-friedman/datamule-python",
+    install_requires=[
+        'aiohttp',
+        'aiolimiter',
+        'tqdm',
+        'requests',
+        'nest_asyncio',
+        'aiofiles',
+        'polars',
+        'setuptools',
+        'selectolax',
+        'pytz',
+        'zstandard',
+        'doc2dict',
+        'secsgml',
+        'lxml'
+    ]
+)

datamule-1.0.3/datamule/book/book.py

@@ -1,34 +0,0 @@
-# Streams data rather than downloading it. 
-# additional functionality such as query by xbrl, and other db
-# also this is basically our experimental rework of portfolio w/o disturbing existing users
-# this is highly experimental and may not work as expected
-# only for datamule source
-# likely new bottleneck will be local parsing() - will be bypassed in future when we have parsed archive
-# wow parsed archive is going to be crazy fast - like every 10k in 1 minute.
-
-# example queries filter by sic = 7372, xbrl query = dei:operatingprofit > 0 in date range 2018-2019
-
-# hmm do we go for sql esq or not.
-# I think we do.
-# i think we remove cik, ticker, sic, etc and just have a query object
-# should be sql esq so users can use it easily w/o learnign new syntax
-
-# WHERE submission_type = '10-K'
-# AND us-gaap:ResearchAndDevelopmentExpense > 0 
-# AND dei:debt_to_equity < 2
-# AND filing_date BETWEEN '2023-01-01' AND '2023-12-31'
-# AND CIK in (123, 456, 789)
-# AND SIC in (123, 456, 789)
-# AND ticker in ('AAPL', 'GOOGL', 'AMZN')
-# AND document_type = 'EX-99.1' # to select attachments
-
-from .eftsquery import EFTSQuery
-
-
-class Book():
-    def process_submissions(self,cik,ticker,sic,submission_type,document_type,date,
-                            xbrl_query={},
-                            metadata_callback=None,
-                            document_callback=None,):
-        # grabs data and processes it
-        pass 

datamule-1.0.3/datamule/book/eftsquery.py

@@ -1,127 +0,0 @@
-import asyncio
-import aiohttp
-from tqdm import tqdm
-from datetime import datetime
-from urllib.parse import urlencode
-import time
-
-class PreciseRateLimiter:
-    def __init__(self, rate=10, interval=1.0):
-        self.rate = rate  # requests per interval
-        self.interval = interval  # in seconds
-        self.token_time = self.interval / self.rate  # time per token
-        self.last_time = time.time()
-        self.lock = asyncio.Lock()
-
-    async def acquire(self):
-        async with self.lock:
-            now = time.time()
-            wait_time = self.last_time + self.token_time - now
-            if wait_time > 0:
-                await asyncio.sleep(wait_time)
-            self.last_time = time.time()
-            return True
-
-class EFTSQuery:
-    def __init__(self):
-        self.headers = {
-            'User-Agent': 'Your Name yourname@email.com',
-            'Accept-Encoding': 'gzip, deflate',
-            'Host': 'efts.sec.gov'
-        }
-        self.session = None
-        self.limiter = PreciseRateLimiter(10)
-
-    async def __aenter__(self):
-        if not self.session:
-            self.session = aiohttp.ClientSession(headers=self.headers)
-        return self
-
-    async def __aexit__(self, exc_type, exc_val, exc_tb):
-        if self.session:
-            await self.session.close()
-            self.session = None
-
-    async def _fetch_json(self, url):
-        await self.limiter.acquire()
-        try:
-            async with self.session.get(url) as response:
-                if response.status == 429:
-                    await asyncio.sleep(61)
-                    return await self._fetch_json(url)
-                return await response.json()
-        except Exception as e:
-            print(f"Error fetching {url}: {str(e)}")
-            return None
-
-    async def _get_accession_numbers(self, base_url):
-        data = await self._fetch_json(f"{base_url}&from=0&size=1")
-        if not data or 'hits' not in data:
-            return []
-            
-        total_hits = data['hits']['total']['value']
-        if not total_hits:
-            return []
-
-        accession_numbers = []
-        start = 0
-        page_size = 100
-        batch_size = 10  # Number of concurrent requests
-        
-        with tqdm(total=total_hits) as pbar:
-            while start < total_hits:
-                tasks = []
-                for i in range(batch_size):
-                    if start + i * page_size >= total_hits:
-                        break
-                    url = f"{base_url}&from={start + i * page_size}&size={page_size}"
-                    tasks.append(self._fetch_json(url))
-                
-                if not tasks:
-                    break
-                    
-                results = await asyncio.gather(*tasks)
-                
-                for data in results:
-                    if data and 'hits' in data:
-                        hits = data['hits']['hits']
-                        batch_numbers = [
-                            f"{hit['_source']['ciks'][0]}/{hit['_id'].split(':')[0]}"
-                            for hit in hits
-                        ]
-                        accession_numbers.extend(batch_numbers)
-                        pbar.update(len(hits))
-                
-                start += batch_size * page_size
-
-        return accession_numbers
-
-    def query_efts(self, cik=None, ticker=None, submission_type=None, filing_date=None, search_text=None):
-        async def _download():
-            async with self as downloader:
-                params = {}
-                
-                if cik:
-                    params['ciks'] = str(cik).zfill(10)
-
-                if submission_type:
-                    params['forms'] = ','.join(submission_type) if isinstance(submission_type, list) else submission_type
-
-                if isinstance(filing_date, list):
-                    dates = [(d, d) for d in filing_date]
-                elif isinstance(filing_date, tuple):
-                    dates = [filing_date]
-                else:
-                    date_str = filing_date if filing_date else f"2001-01-01,{datetime.now().strftime('%Y-%m-%d')}"
-                    start, end = date_str.split(',')
-                    dates = [(start, end)]
-                
-                params['startdt'], params['enddt'] = dates[0]
-
-                if search_text:
-                    params['q'] = f'"{search_text}"'
-
-                base_url = f"https://efts.sec.gov/LATEST/search-index?{urlencode(params, doseq=True)}"
-                return await self._get_accession_numbers(base_url)
-
-        return asyncio.run(_download())