datamule 2.0.3__tar.gz → 2.0.5__tar.gz

{datamule-2.0.3 → datamule-2.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.0.3
+Version: 2.0.5
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.0.3 → datamule-2.0.5}/datamule/datamule/datamule_mysql_rds.py

@@ -77,8 +77,14 @@ class DatamuleMySQL:
             if value is None:
                 continue
                 
+            # *** HIGHLIGHTED CHANGE: Special logic for members in simple_xbrl table ***
+            if table == 'simple_xbrl' and key == 'members':
+                if isinstance(value, list):
+                    filters[key] = {"type": "find_in_set", "values": value}
+                else:
+                    filters[key] = {"type": "find_in_set", "values": [value]}
             # Special logic for cik
-            if key == 'cik':
+            elif key == 'cik':
                 if isinstance(value, list):
                     value = [int(val) for val in value]
                 else:
@@ -100,6 +106,12 @@ class DatamuleMySQL:
         for key, filter_obj in filters.items():
             if filter_obj["type"] == "range":
                 query_desc.append(f"{key}={filter_obj['values'][0]} to {filter_obj['values'][1]}")
+            # *** HIGHLIGHTED CHANGE: Display logic for find_in_set type ***
+            elif filter_obj["type"] == "find_in_set":
+                if len(filter_obj["values"]) == 1:
+                    query_desc.append(f"{key} contains {filter_obj['values'][0]}")
+                else:
+                    query_desc.append(f"{key} contains any of {filter_obj['values']}")
             elif len(filter_obj["values"]) == 1:
                 query_desc.append(f"{key}={filter_obj['values'][0]}")
             else:
@@ -177,6 +189,7 @@ def query_mysql_rds(table, api_key=None, **kwargs):
     Parameters:
     - table: Table name (e.g., 'simple_xbrl')
     - cik: Company CIK number(s), can be int, string, or list
+    - members: For simple_xbrl table, search within comma-separated member strings
     - Any other filter parameters as keyword arguments
     - page_size: Number of records per page (max 25000, default 25000)
     - quiet: Boolean, whether to suppress progress output and summary (default False)
@@ -186,6 +199,7 @@ def query_mysql_rds(table, api_key=None, **kwargs):
     - Single value: Exact match
     - List: OR condition (any of the values)
     - Tuple: Range condition (between first and second values)
+    - members (simple_xbrl only): Searches within comma-separated strings using FIND_IN_SET
     
     Returns:
     - List of dictionaries containing the requested data (ready for pandas DataFrame)
@@ -224,8 +238,14 @@ def _query_mysql_rds_single(table, api_key=None, **kwargs):
         if value is None:
             continue
             
+        # *** HIGHLIGHTED CHANGE: Special logic for members in simple_xbrl table ***
+        if table == 'simple_xbrl' and key == 'members':
+            if isinstance(value, list):
+                filters[key] = {"type": "find_in_set", "values": value}
+            else:
+                filters[key] = {"type": "find_in_set", "values": [value]}
         # special logic for cik
-        if key == 'cik':
+        elif key == 'cik':
             if isinstance(value, list):
                 value = [int(val) for val in value]
             else:

{datamule-2.0.3 → datamule-2.0.5}/datamule/document/document.py

@@ -12,8 +12,7 @@ from .processing import process_tabular_data
 from pathlib import Path
 import webbrowser
 from secsgml.utils import bytes_to_str
-from secxbrl import parse_inline_xbrl
-from company_fundamentals import construct_fundamentals
+
 
 class Document:
     def __init__(self, type, content, extension,accession,filing_date,path=None):
@@ -35,8 +34,7 @@ class Document:
         self.extension = extension
         # this will be filled by parsed
         self.data = None
-        self.xbrl = None
-        self.fundamentals = None
+
 
     #_load_text_content
     def _preprocess_txt_content(self):
@@ -106,70 +104,6 @@ class Document:
             return bool(re.search(pattern, self.content))
         return False
     
-    # slated for removal
-    def parse_xbrl(self,type='inline'):
-        if self.xbrl:
-            return
-        if type =='inline':
-            if self.extension not in ['.htm','.html']:
-                return
-            self.xbrl = parse_inline_xbrl(self.content)
-        else:
-            raise ValueError("Only inline has been implemented so far.")
-        
-    def parse_fundamentals(self,categories=None):
-        self.parse_xbrl()
-        # Transform XBRL records into the format needed by construct_fundamentals
-        xbrl = []
-        
-        for xbrl_record in self.xbrl:
-            try:
-                # Extract basic fields
-                value = xbrl_record.get('_val', None)
-                taxonomy, name = xbrl_record['_attributes']['name'].split(':')
-                
-                # Handle scaling if present
-                if xbrl_record.get('_attributes', {}).get('scale') is not None:
-                    scale = int(xbrl_record['_attributes']['scale'])
-                    try:
-                        value = str(Decimal(value.replace(',', '')) * (Decimal(10) ** scale))
-                    except:
-                        pass
-                
-                # Extract period dates
-                period_start_date = None
-                period_end_date = None
-                
-                if xbrl_record.get('_context'):
-                    context = xbrl_record['_context']
-                    period_start_date = context.get('context_period_instant') or context.get('context_period_startdate')
-                    period_end_date = context.get('context_period_enddate')
-                
-                # Create record in the format expected by construct_fundamentals
-                record = {
-                    'taxonomy': taxonomy,
-                    'name': name,
-                    'value': value,
-                    'period_start_date': period_start_date,
-                    'period_end_date': period_end_date
-                }
-                
-                xbrl.append(record)
-                
-            except Exception as e:
-                # Skip malformed records
-                continue
-        
-        # Call construct_fundamentals with the transformed data
-        fundamentals = construct_fundamentals(xbrl, 
-                            taxonomy_key='taxonomy', 
-                            concept_key='name', 
-                            start_date_key='period_start_date', 
-                            end_date_key='period_end_date',
-                            categories=categories)
-        
-        self.fundamentals = fundamentals
-
     # Note: this method will be heavily modified in the future
     def parse(self):
         # check if we have already parsed the content

{datamule-2.0.3 → datamule-2.0.5}/datamule/submission.py

@@ -9,6 +9,10 @@ import tarfile
 import zstandard as zstd
 import gzip
 import urllib.request
+from secxbrl import parse_inline_xbrl
+from company_fundamentals import construct_fundamentals
+from decimal import Decimal
+
 
 class Submission:
     def __init__(self, path=None, sgml_content=None, keep_document_types=None,
@@ -17,6 +21,7 @@ class Submission:
 
         # declare vars to be filled later
         self.xbrl = None
+        self.fundamentals = None
         
         # Validate parameters
         param_count = sum(x is not None for x in [path, sgml_content, batch_tar_path,url])
@@ -242,18 +247,81 @@ class Submission:
             if doc['type'] in document_types:
                 yield self._load_document_by_index(idx)
 
-    # def parse_xbrl(self):
-    #     for idx, doc in enumerate(self.metadata.content['documents']):
-    #         if doc['type'] in ['EX-100.INS','EX-101.INS']:
-    #             document = self._load_document_by_index(idx)
-    #             break
+    def parse_xbrl(self):
+        if self.xbrl:
+            return
+        
+        for idx, doc in enumerate(self.metadata.content['documents']):
+            if doc['type'] in ['EX-100.INS','EX-101.INS']:
+                document = self._load_document_by_index(idx)
+                self.xbrl = parse_inline_xbrl(content=document.content,file_type='extracted_inline')
+                return
+
+            if doc['filename'].endswith('_htm.xml'):
+                document = self._load_document_by_index(idx)
+                self.xbrl = parse_inline_xbrl(content=document.content,file_type='extracted_inline')
+                return
+
+        
+    def parse_fundamentals(self,categories=None):
+        self.parse_xbrl()
+
+        # if no xbrl return
+        if not self.xbrl:
+            return
+        # Transform XBRL records into the format needed by construct_fundamentals
+        xbrl = []
+        
+        for xbrl_record in self.xbrl:
+            try:
+                # Extract basic fields
+                value = xbrl_record.get('_val', None)
+                
+                taxonomy, name = xbrl_record['_attributes']['name'].split(':')
+                
+
+                # Handle scaling if present
+                if xbrl_record.get('_attributes', {}).get('scale') is not None:
+                    scale = int(xbrl_record['_attributes']['scale'])
+                    try:
+                        value = str(Decimal(value.replace(',', '')) * (Decimal(10) ** scale))
+                    except:
+                        pass
+                
 
-    #         if doc['filename'].endswith('_htm.xml'):
-    #             document = self._load_document_by_index(idx)
-    #             break
+                # Extract period dates
+                period_start_date = None
+                period_end_date = None
+                
+                if xbrl_record.get('_context'):
+                    context = xbrl_record['_context']
+                    period_start_date = context.get('context_period_instant') or context.get('context_period_startdate')
+                    period_end_date = context.get('context_period_enddate')
+                
+                # Create record in the format expected by construct_fundamentals
+                record = {
+                    'taxonomy': taxonomy,
+                    'name': name,
+                    'value': value,
+                    'period_start_date': period_start_date,
+                    'period_end_date': period_end_date
+                }
+                
+                xbrl.append(record)
+                
+            except Exception as e:
+                # Skip malformed records
+                continue
+        
+  
+        # Call construct_fundamentals with the transformed data
+        fundamentals = construct_fundamentals(xbrl, 
+                            taxonomy_key='taxonomy', 
+                            concept_key='name', 
+                            start_date_key='period_start_date', 
+                            end_date_key='period_end_date',
+                            categories=categories)
+        
+        self.fundamentals = fundamentals
         
-    #     print(doc['type'])
-    #     if not document:
-    #         return
         
-    #     self.xbrl = document.parse_xbrl()

{datamule-2.0.3 → datamule-2.0.5}/datamule.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamule
-Version: 2.0.3
+Version: 2.0.5
 Summary: Work with SEC submissions at scale.
 Home-page: https://github.com/john-friedman/datamule-python
 Author: John Friedman

{datamule-2.0.3 → datamule-2.0.5}/setup.py

@@ -32,7 +32,7 @@ if not os.path.exists(file_path):
 setup(
     name="datamule",
     author="John Friedman",
-    version="2.0.3",
+    version="2.0.5",
     description="Work with SEC submissions at scale.",
     packages=find_packages(include=['datamule', 'datamule.*']),
     url="https://github.com/john-friedman/datamule-python",