datamule 1.1.6__py3-none-any.whl → 1.1.8__py3-none-any.whl

datamule/sheet.py

@@ -1,6 +1,9 @@
 from pathlib import Path
+import csv
+import os
 from .helper import _process_cik_and_metadata_filters, load_package_dataset
 from .sec.xbrl.downloadcompanyfacts import download_company_facts
+from .seclibrary.bq import get_information_table
 
 class Sheet:
     def __init__(self, path):
@@ -26,9 +29,220 @@ class Sheet:
         # Download facts for all CIKs in parallel
         download_company_facts(cik=cik_list, output_dir=self.path)
 
-    def query_345():
-        pass
-    def query_xbrl():
-        pass
-    def query_13fhr():
-        pass
+    def get_information_table(
+        self,
+        # Required parameters
+        table_type="INFORMATION_TABLE",
+        
+        # Optional filtering parameters
+        columns=None,
+        name_of_issuer=None,
+        title_of_class=None,
+        cusip=None,
+        value=None,
+        ssh_prnamt=None,
+        ssh_prnamt_type=None,
+        investment_discretion=None,
+        voting_authority_sole=None,
+        voting_authority_shared=None,
+        voting_authority_none=None,
+        reporting_owner_cik=None,
+        put_call=None,
+        other_manager=None,
+        figi=None,
+        accession=None,
+        filing_date=None,
+        
+        # API key handling
+        api_key=None,
+        
+        # Additional options
+        print_cost=True,
+        verbose=False
+    ):
+        """
+        Query the SEC BigQuery API for 13F-HR information table data.
+        
+        Parameters:
+        -----------
+        table_type : str
+            The table to query (default is "INFORMATION_TABLE")
+        columns : List[str], optional
+            Specific columns to return. If None, all columns are returned.
+        
+        # Filter parameters
+        name_of_issuer, title_of_class, etc. : Various filters that can be:
+            - str: Exact match
+            - List[str]: Match any in list
+            - tuple: (min, max) range for numeric/date fields
+        
+        api_key : str, optional
+            SEC BigQuery API key. If None, looks for DATAMULE_API_KEY env variable.
+        print_cost : bool
+            Whether to print the query cost information
+        verbose : bool
+            Whether to print additional information about the query
+            
+        Returns:
+        --------
+        List[Dict]
+            A list of dictionaries containing the query results
+            
+        Raises:
+        -------
+        ValueError
+            If API key is missing or invalid
+        Exception
+            For API errors or other issues
+        """
+
+        return get_information_table(
+            table_type=table_type,
+            columns=columns,
+            name_of_issuer=name_of_issuer,
+            title_of_class=title_of_class,
+            cusip=cusip,
+            value=value,
+            ssh_prnamt=ssh_prnamt,
+            ssh_prnamt_type=ssh_prnamt_type,
+            investment_discretion=investment_discretion,
+            voting_authority_sole=voting_authority_sole,
+            voting_authority_shared=voting_authority_shared,
+            voting_authority_none=voting_authority_none,
+            reporting_owner_cik=reporting_owner_cik,
+            put_call=put_call,
+            other_manager=other_manager,
+            figi=figi,
+            accession=accession,
+            filing_date=filing_date,
+            
+            # API key handling
+            api_key=api_key,
+            
+            # Additional options
+            print_cost=print_cost,
+            verbose=verbose
+        )
+
+    def download_information_table(
+        self,
+        filepath,
+        # Required parameters
+        table_type="INFORMATION_TABLE",
+        
+        # Optional filtering parameters
+        columns=None,
+        name_of_issuer=None,
+        title_of_class=None,
+        cusip=None,
+        value=None,
+        ssh_prnamt=None,
+        ssh_prnamt_type=None,
+        investment_discretion=None,
+        voting_authority_sole=None,
+        voting_authority_shared=None,
+        voting_authority_none=None,
+        reporting_owner_cik=None,
+        put_call=None,
+        other_manager=None,
+        figi=None,
+        accession=None,
+        filing_date=None,
+        
+        # API key handling
+        api_key=None,
+        
+        # Additional options
+        print_cost=True,
+        verbose=False
+    ):
+        """
+        Query the SEC BigQuery API for 13F-HR information table data and save to CSV.
+        
+        Parameters:
+        -----------
+        filepath : str
+            Path where to save the CSV file. If relative, it will be relative to the Sheet's path.
+        
+        table_type : str
+            The table to query (default is "INFORMATION_TABLE")
+        columns : List[str], optional
+            Specific columns to return. If None, all columns are returned.
+        
+        # Filter parameters
+        name_of_issuer, title_of_class, etc. : Various filters that can be:
+            - str: Exact match
+            - List[str]: Match any in list
+            - tuple: (min, max) range for numeric/date fields
+        
+        api_key : str, optional
+            SEC BigQuery API key. If None, looks for DATAMULE_API_KEY env variable.
+        print_cost : bool
+            Whether to print the query cost information
+        verbose : bool
+            Whether to print additional information about the query
+            
+        Returns:
+        --------
+        List[Dict]
+            A list of dictionaries containing the query results
+            
+        Raises:
+        -------
+        ValueError
+            If API key is missing or invalid
+        Exception
+            For API errors or other issues
+        """
+        # Get the data from the API
+        data = self.get_information_table(
+            table_type=table_type,
+            columns=columns,
+            name_of_issuer=name_of_issuer,
+            title_of_class=title_of_class,
+            cusip=cusip,
+            value=value,
+            ssh_prnamt=ssh_prnamt,
+            ssh_prnamt_type=ssh_prnamt_type,
+            investment_discretion=investment_discretion,
+            voting_authority_sole=voting_authority_sole,
+            voting_authority_shared=voting_authority_shared,
+            voting_authority_none=voting_authority_none,
+            reporting_owner_cik=reporting_owner_cik,
+            put_call=put_call,
+            other_manager=other_manager,
+            figi=figi,
+            accession=accession,
+            filing_date=filing_date,
+            api_key=api_key,
+            print_cost=print_cost,
+            verbose=verbose
+        )
+        
+        # If no data returned, nothing to save
+        if not data:
+            if verbose:
+                print("No data returned from API. No file was created.")
+            return data
+        
+        # Resolve filepath - if it's not absolute, make it relative to self.path
+        filepath_obj = Path(filepath)
+        if not filepath_obj.is_absolute():
+            filepath_obj = self.path / filepath_obj
+        
+        # Create directory if it doesn't exist
+        os.makedirs(filepath_obj.parent, exist_ok=True)
+        
+        # Get fieldnames from the first record
+        fieldnames = data[0].keys()
+        
+        # Write to CSV
+        with open(filepath_obj, 'w', newline='') as csvfile:
+            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(data)
+            
+        if verbose:
+            print(f"Saved {len(data)} records to {filepath_obj}")
+            
+        return data

datamule/submission.py

@@ -1,16 +1,38 @@
 from pathlib import Path
 import json
 from .document import Document
+from secsgml import parse_sgml_submission_into_memory
+from pathlib import Path
 
 class Submission:
-    def __init__(self, path):
-        self.path = Path(path)
-        self._load_metadata()
+    def __init__(self, path=None,sgml_content=None,keep_document_types=None):
+        if path is None and sgml_content is None:
+            raise ValueError("Either path or sgml_content must be provided")
+        if path is not None and sgml_content is not None:
+            raise ValueError("Only one of path or sgml_content must be provided")
+        
+        if sgml_content is not None:
+            self.path = None
+            self.metadata, raw_documents = parse_sgml_submission_into_memory(sgml_content)
+            self.documents = []
+
+            for idx,doc in enumerate(self.metadata['documents']):
+                type = doc.get('type')
+                
+                # Keep only specified types
+                if keep_document_types is not None and type not in keep_document_types:
+                    continue
+                filename = doc.get('filename')
+                extension = Path(filename).suffix
+                self.documents.append(Document(type=type, content=raw_documents[idx], extension=extension))
+
+
+        if path is not None:
+            self.path = Path(path)  
+            metadata_path = self.path / 'metadata.json'
+            with metadata_path.open('r') as f:
+                self.metadata = json.load(f)
     
-    def _load_metadata(self):
-        metadata_path = self.path / 'metadata.json'
-        with metadata_path.open('r') as f:
-            self.metadata = json.load(f)
 
     def document_type(self, document_type):
         # Convert single document type to list for consistent handling
@@ -19,20 +41,73 @@ class Submission:
         else:
             document_types = document_type
 
-        for doc in self.metadata['documents']:
+        for idx,doc in enumerate(self.metadata['documents']):
             if doc['type'] in document_types:
+                
+                # if loaded from path
+                if self.path is not None:
+                    filename = doc.get('filename')
+                    # oh we need handling here for sequences case
+                    if filename is None:
+                        filename = doc['sequence'] + '.txt'
+                        
+                    document_path = self.path / filename
+                    extension = document_path.suffix
+
+                    with document_path.open('r') as f:
+                        content = f.read()
+
+                    yield Document(type=doc['type'], content=content, extension=extension)
+                # if loaded from sgml_content
+                else:
+                    yield self.documents[idx]
+
+    
+    def __iter__(self):
+        for idx,doc in enumerate(self.metadata['documents']):
+            # if loaded from path
+            if self.path is not None:
                 filename = doc.get('filename')
+
+                # oh we need handling here for sequences case
                 if filename is None:
-                    continue
+                    filename = doc['sequence'] + '.txt'
                     
                 document_path = self.path / filename
-                yield Document(doc['type'], document_path)
-    
-    def __iter__(self):
-        for doc in self.metadata['documents']:
-            filename = doc.get('filename')
-            if filename is None:
-                continue
-                
-            document_path = self.path / filename
-            yield Document(doc['type'], document_path)
+                extension = document_path.suffix
+
+                # check if the file exists
+                if document_path.exists():
+                    with document_path.open('r') as f:
+                        content = f.read()
+
+                    yield Document(type=doc['type'], content=content, extension=extension)
+                else:
+                    print(f"Warning: File {document_path} does not exist likely due to keep types in downloading.")
+
+            # if loaded from sgml_content
+            else:
+                yield self.documents[idx]
+
+    # keep documents by document type
+    def keep(self, document_type):
+        # Convert single document type to list for consistent handling
+        if isinstance(document_type, str):
+            document_types = [document_type]
+        else:
+            document_types = document_type
+
+        if self.path is not None:
+            for doc in self.metadata['documents']:
+                filename = doc.get('filename')
+                type = doc.get('type')
+                if type not in document_types:
+                    # oh we need handling here for sequences case
+                    if filename is None:
+                        filename = doc.sequence + '.txt'
+
+                    document_path = self.path / filename
+                    # delete the file
+                    document_path.unlink()
+        else:
+            print("Warning: keep() method is only available when loading from path.")

{datamule-1.1.6.dist-info → datamule-1.1.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 1.1.6
+Version: 1.1.8
 Summary: Making it easier to use SEC filings.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-1.1.6.dist-info → datamule-1.1.8.dist-info}/RECORD

@@ -1,11 +1,11 @@
 datamule/__init__.py,sha256=l6YlwT5EeRxPlCtO5Jd4I8l266rSRUJyfFe97cRtSCM,991
 datamule/config.py,sha256=Y--CVv7JcgrjJkMOSLrvm2S8B9ost6RMSkGviP-MKtg,883
-datamule/document.py,sha256=BC8jdVy9pMOA9ghIqV5N2XJidmVNThqbBohsuSAnVoY,10813
+datamule/document.py,sha256=qShyVKHQ1nSCNvSfrhAOMVXprOd1br1rFKLy52S9WnE,22007
 datamule/helper.py,sha256=xgOVnea-lUlQ5I-U0vYUp0VeKPNZehNhqjJvegA3lYE,3342
 datamule/index.py,sha256=0txvbzPcvY1GsdxA-wGdLzAByxSeE_1VyyBp9mZEQRM,2292
-datamule/portfolio.py,sha256=JmZlTrom_g7FXKXxWp_CiQTyC7p6_cDP08G0kFUja48,6982
-datamule/sheet.py,sha256=WwumRdniClGU7W3AXVLOpCdMnepLC7KMrRpQlA6_NUY,1022
-datamule/submission.py,sha256=JsxYlEz1Ywu6eC32OS15p4p-p8qB6SWd_rXuf2p5UfY,1247
+datamule/portfolio.py,sha256=yWt5gYTjV7rJsLiPUmhc6Vmr3lfvfCR5MSpLQ_6Gdp4,7104
+datamule/sheet.py,sha256=QaArtx7LpT7bwyteelJV67C-lK0RjQbGS3ka7ftdi8w,7978
+datamule/submission.py,sha256=LI7Zr60YbE_tU-v2N09k2dGjfztSgplKZACT3eRUkFE,4463
 datamule/mapping_dicts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/mapping_dicts/txt_mapping_dicts.py,sha256=DQPrGYbAPQxomRUtt4iiMGrwuF7BHc_LeFBQuYBzU9o,6311
 datamule/mapping_dicts/xml_mapping_dicts.py,sha256=Z22yDVwKYonUfM5foQP00dVDE8EHhhMKp0CLqVKV5OI,438
@@ -16,20 +16,21 @@ datamule/sec/infrastructure/submissions_metadata.py,sha256=f1KarzFSryKm0EV8DCDNs
 datamule/sec/rss/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/rss/monitor.py,sha256=6r4EYaSlGu6VYErlj9zXJsIMLVie1cfacSZU-ESfuBI,18231
 datamule/sec/submissions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamule/sec/submissions/downloader.py,sha256=HxbSkNotLLW6ROmU30rnXPlCo9gY3SoB1Z4ZWvj9FIY,2669
-datamule/sec/submissions/eftsquery.py,sha256=v6YMBZzksqweqHnNIllMFN-frWypAgvZPKx2FH1UrL4,22515
-datamule/sec/submissions/monitor.py,sha256=F24I9yn1k8ggbCJQ-Vk7go_qJHlpkBzVKFYKDs_CWLs,5287
-datamule/sec/submissions/streamer.py,sha256=hc61le7gGIIWp1KEaOv_PhriUxf7YYFkQrSKELlZ3pg,9748
-datamule/sec/submissions/textsearch.py,sha256=oEIUrcO3HW-4dcyPCiOTvM7UUimNEM4HNIb-Juvc1BQ,4642
+datamule/sec/submissions/downloader.py,sha256=IB08W8-lQD5Bb0LgzrTN4Xi4HsCw24DybRLHqE1AUrU,3290
+datamule/sec/submissions/eftsquery.py,sha256=mSZon8rlW8dxma7M49ZW5V02Fn-ENOdt9TNO6elBrhE,27983
+datamule/sec/submissions/monitor.py,sha256=Im2kgnUehhTgyY2Vq3uk07n4Vkj4PjII_SsRDi8ehAE,5384
+datamule/sec/submissions/streamer.py,sha256=EXyWNCD9N6mZmvm9lFSCFodF19zSQ8jfIbWPZNp0K5Y,11253
+datamule/sec/submissions/textsearch.py,sha256=-a5yIrrxxtaK10IJeywFmXuJmSndYL9VKm4SC4I9JAs,5808
 datamule/sec/xbrl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datamule/sec/xbrl/downloadcompanyfacts.py,sha256=rMWRiCF9ci_gNZMJ9MC2c_PGEd-yEthawQ0CtVwWTjM,3323
 datamule/sec/xbrl/filter_xbrl.py,sha256=g9OT4zrNS0tiUJeBIwbCs_zMisOBkpFnMR3tV4Tr39Q,1316
 datamule/sec/xbrl/streamcompanyfacts.py,sha256=WyJIwuy5mNMXWpx_IkhFzDMe9MOfQ-vNkWl_JzBzFmc,3323
 datamule/sec/xbrl/xbrlmonitor.py,sha256=TKFVfSyyUUfUgFQw4WxEVs4g8Nh-2C0tygNIRmTqW3Y,5848
 datamule/seclibrary/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datamule/seclibrary/bq.py,sha256=C6kafFXWtm-MUjf70H1wTtpwv1Rxpcbk-Kfy8fkBPfo,6469
 datamule/seclibrary/downloader.py,sha256=Zb1TxsIz887tO3MJVP66siYVtNus89ti-g9oZ6VywrM,11500
 datamule/seclibrary/query.py,sha256=qGuursTERRbOGfoDcYcpo4oWkW3PCBW6x1Qf1Puiak4,7352
-datamule-1.1.6.dist-info/METADATA,sha256=2kV79oqPrN2_HTcd4bxbJ0Q2_nP7Ta5Z79yskrST5S8,512
-datamule-1.1.6.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-datamule-1.1.6.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
-datamule-1.1.6.dist-info/RECORD,,
+datamule-1.1.8.dist-info/METADATA,sha256=8HRRMz6l928E5tuHXkPi1_Kf-8nfPSjWQnnfReSxdPM,512
+datamule-1.1.8.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+datamule-1.1.8.dist-info/top_level.txt,sha256=iOfgmtSMFVyr7JGl_bYSTDry79JbmsG4p8zKq89ktKk,9
+datamule-1.1.8.dist-info/RECORD,,