dshield-parser 0.1.0__tar.gz → 0.1.3__tar.gz

{dshield_parser-0.1.0 → dshield_parser-0.1.3}/PKG-INFO

@@ -1,13 +1,13 @@
-Metadata-Version: 2.1
-Name: dshield_parser
-Version: 0.1.0
-Summary: This program has utilities to download, parse and enrich DSHield honeypot data
-Home-page: https://github.com/jslagrew/dshield-parser
-Author: Jesse La Grew
-Author-email: jlagrew.github@proton.me
-License: UNKNOWN
-Platform: UNKNOWN
-Requires-Python: >3.6
-
-UNKNOWN
-
+Metadata-Version: 2.4
+Name: dshield_parser
+Version: 0.1.3
+Summary: This program has utilities to download, parse and enrich DSHield honeypot data
+Home-page: https://github.com/jslagrew/dshield-parser
+Author: Jesse La Grew
+Author-email: jlagrew.github@proton.me
+Requires-Python: >3.12.7
+Dynamic: author
+Dynamic: author-email
+Dynamic: home-page
+Dynamic: requires-python
+Dynamic: summary

dshield_parser-0.1.3/README.md

@@ -0,0 +1,14 @@
+# dshield-parser
+Python scripts to parse DShield honeypot (cowrie, web honeypot, iptables firewall) logs
+
+# Installing
+The easiest way to install is using pip.
+```
+# it is recommended to use a virtual environment
+# python -m venv /path/to/new/venv
+# /path/to/new/venv/scripts/activate
+
+# depending on your system, you may need to replace "pip" with "pip3"
+pip install dshield-parser
+pip install -r https://raw.githubusercontent.com/jslagrew/dshield-parser/main/requirements.txt
+```

{dshield_parser-0.1.0 → dshield_parser-0.1.3}/dshield_parser/utils/enrichment.py

@@ -32,9 +32,9 @@ def get_spur_data(log_source, input, session, api):
     while True:
         try:
             response = session.get(url)
-        except:
-            print("Exception hit for SPUR query")
-            #time.sleep(10)
+        except Exception as e:
+            logging.error(f"Exception hit for SPUR query: '{e}'. Pausing for 15 seconds before trying again.")
+            time.sleep(15)
             continue
         break
     json_data = json.loads(response.text)
@@ -61,7 +61,15 @@ def get_vt_data(input, input_type, api):
     else:
         logging.error("Invalid input type given for VirusTotal to process.")
     
-    response = requests.get(url, headers=headers)
+    while True:
+        try:
+            response = requests.get(url, headers=headers)
+        except Exception as e:
+            logging.error(f"Exception hit for VirusTotal query: '{e}'. Pausing for 15 seconds before trying again.")
+            time.sleep(15)
+            continue
+        break
+    
     json_response = json.loads(response.text)
     logging.debug(response.text)
     if "error" in json_response:
@@ -165,7 +173,16 @@ def isc_cloudapis(email):
     headers = {
         'User-Agent': f'Request from {email}',
     }   
-    response = requests.get(url, headers=headers)
+
+    while True:
+        try:
+            response = requests.get(url, headers=headers)
+        except Exception as e:
+            logging.error(f"Exception hit for Could IP SANS ISC query: '{e}'. Pausing for 15 seconds before trying again.")
+            time.sleep(15)
+            continue
+        break    
+    
     xml =  response.text
     root = ET.fromstring(xml)
     ips = []
@@ -193,7 +210,15 @@ def isc_cloudcidrs(email):
     headers = {
         'User-Agent': f'Request from {email}',
     }   
-    response = requests.get(url, headers=headers)
+    while True:
+        try:
+            response = requests.get(url, headers=headers)
+        except Exception as e:
+            logging.error(f"Exception hit for Cloud CIDR SANS ISC query: '{e}'. Pausing for 15 seconds before trying again.")
+            time.sleep(15)
+            continue
+        break    
+    
     xml =  response.text
     root = ET.fromstring(xml)
     ips = []
@@ -217,7 +242,16 @@ def isc_intelfeed(email):
     headers = {
         f'User-Agent': 'Request from {email}',
     }   
-    response = requests.get(url, headers=headers)
+
+    while True:
+        try:
+            response = requests.get(url, headers=headers)
+        except Exception as e:
+            logging.error(f"Exception hit for SANS ISC Intel Feed query: '{e}'. Pausing for 15 seconds before trying again.")
+            time.sleep(15)
+            continue
+        break
+    
     xml =  response.text
     root = ET.fromstring(xml)
 
@@ -231,7 +265,16 @@ def isc_ipinfo(ip, email):
     headers = {
         'User-Agent': f'Request from {email}',
     }   
-    response = requests.get(url, headers=headers)    
+
+    while True:
+        try:
+            response = requests.get(url, headers=headers) 
+        except Exception as e:
+            logging.error(f"Exception hit for SANS ISC IP info query: '{e}'. Pausing for 15 seconds before trying again.")
+            time.sleep(15)
+            continue
+        break
+
     while response.status_code != 200:
         delay = 5
         if response.status_code == 429:
@@ -289,7 +332,16 @@ def isc_research_threats(email, start_date=None, end_date=None):
     headers = {
         'User-Agent': f'Request from {email}',
     }   
-    response = requests.get(url, headers=headers)    
+
+    while True:
+        try:
+            response = requests.get(url, headers=headers)
+        except Exception as e:
+            logging.error(f"Exception hit for SANS ISC Research Threats query: '{e}'. Pausing for 15 seconds before trying again.")
+            time.sleep(15)
+            continue
+        break
+        
     xml =  response.text
     logging.debug(f"XML Data: {xml}")
     root = ET.fromstring(xml)
@@ -307,12 +359,24 @@ def isc_research_threats(email, start_date=None, end_date=None):
     return pd.DataFrame(threat_intel)
 
 def binary_edge_minions(email):
-    url = f"https://api.binaryedge.io/krang/v1/minions"
+    # bad URL as of 7/6/2025
+    # url = f"https://api.binaryedge.io/krang/v1/minions"
+    url = f"https://api.binaryedge.io/v1/minions"
 
     headers = {
         'User-Agent': f'Request from {email}',
     }   
-    response = requests.get(url, headers=headers)    
+
+    while True:
+        try:
+            #this gave error 404 on 7/6/2025
+            response = requests.get(url, headers=headers)  
+        except Exception as e:
+            logging.error(f"Exception hit for Binary Edge query: '{e}'. Pausing for 15 seconds before trying again.")
+            time.sleep(15)
+            continue
+        break    
+      
     scanners =  json.loads(response.text)['scanners']
 
     #return a list of binary edge minions (scanning agents)
@@ -324,7 +388,16 @@ def censys_networks(email):
     headers = {
         'User-Agent': f'Request from {email}',
     }   
-    response = requests.get(url, headers=headers)    
+
+    while True:
+        try:
+            response = requests.get(url, headers=headers) 
+        except Exception as e:
+            logging.error(f"Exception hit for Censys query: '{e}'. Pausing for 15 seconds before trying again.")
+            time.sleep(15)
+            continue
+        break
+
     scanners =  response.text.split("\n")
 
     #return a list of binary edge minions (scanning agents)
@@ -336,7 +409,10 @@ def get_reverse_dns(ipaddress):
     try:
         response = socket.gethostbyaddr(f"{ipaddress}")
     except Exception as e:
-        logging.error(f"Lookup failed for {ipaddress}: {e}")
+        if "host not found" in e:
+            logging.info(f"Lookup failed for {ipaddress}: {e}")
+        else:
+            logging.error(f"Lookup failed for {ipaddress}: {e}")
     if len(response) == 0:
         return response
     else:
@@ -348,7 +424,15 @@ def isc_webhoneypot_useragents(user_agent, day, email):
     headers = {
         'User-Agent': f'Request from {email}',
     }   
-    response = requests.get(url, headers=headers)  
+    while True:
+        try:
+            response = requests.get(url, headers=headers)  
+        except Exception as e:
+            logging.error(f"Exception hit for SANS ISC web honeypot user agent query: '{e}'. Pausing for 15 seconds before trying again.")
+            time.sleep(15)
+            continue
+        break
+    
     return json.loads(response.text)    
     
 def extract_ips(data):
@@ -371,17 +455,4 @@ def extract_subnets(data):
             return_networks.append(each_network)
         except:
             logging.error(f"Error trying to create network object from {each_network}")
-    return return_networks   
-
-
-#cloudips = isc_cloudapis()
-#insert_df(f"{date}_iscdata.sqlite", cloudips, "cloudips")
-
-#cloudcidr = isc_cloudcidrs()
-#insert_df(f"{date}_iscdata.sqlite", cloudcidr, "cloudcidrs")
-
-#ipdata = isc_ipinfo("70.91.145.10")
-#insert_dict(f"{date}_iscdata.sqlite", ipdata, "70.91.145.10")  
-
-#research_data = isc_research_threats()
-#insert_df(f"{date}_iscdata.sqlite", research_data, "research_ip")  
+    return return_networks   

{dshield_parser-0.1.0 → dshield_parser-0.1.3}/dshield_parser/utils/file_io.py

@@ -32,7 +32,7 @@ def read_file_json_generator(filename):
     for line in file:
         if line[:1] =="{":
             try:
-                yield json.loads(line)
+                yield json.loads(line) #.replace('\x00','') # added .replace('\x00','') on 8/17/24 to replace null bytes for pandas merge issues
             except Exception as e:
                 logging.error(f"Exception hit for file '{filename}': {e}")
                 logging.error(f"Text of line with error (limted to first 40 characters): '{line[0:40]}'")
@@ -62,6 +62,8 @@ def get_file_lists(path):
             cowrie_files.append(each_item)
         elif "webhoneypot-" in str(each_item):
             web_honeypot_files.append(each_item)
+        elif "webhoneypot_" in str(each_item):
+            web_honeypot_files.append(each_item)            
     cowrie_files = sorted(cowrie_files, key=os.path.getmtime)
     web_honeypot_files = sorted(web_honeypot_files, key=os.path.getmtime)
     firewall_files = sorted(firewall_files, key=os.path.getmtime)

{dshield_parser-0.1.0 → dshield_parser-0.1.3}/dshield_parser/utils/json.py

@@ -11,6 +11,7 @@ import dshield_parser
 import dshield_parser.firewall_processor
 import dshield_parser.firewall_processor.reports
 import dshield_parser.utils.file_io as file_io
+import pandas as pd
 
 default_datespan={'start_date': '1970-01-01', 'end_date': '3000-01-01'}
 default_timespan={"start_time": 0, "end_time": 32516881838}
@@ -68,7 +69,18 @@ def print_list(list):
     for each_item in list:
         print(each_item)
 
-def get_json_values(key, incoming_data, timespan=default_timespan, exclusions=None):
+def sizeof_fmt(num, suffix='B'):
+    ''' by Fred Cirera,  https://stackoverflow.com/a/1094933/1870254, modified'''
+    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
+        if abs(num) < 1024.0:
+            return "%3.1f %s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f %s%s" % (num, 'Yi', suffix)
+
+def get_json_values(key, incoming_data, timespan=default_timespan, exclusions=None, compress_dates=False):
+    day_count = 0
+    df = pd.DataFrame()
+    #new_df = pd.DataFrame()
     if "start_time" in timespan and "end_time" in timespan:
         start_date = convert_to_date(timespan["start_time"])
         end_date = convert_to_date(timespan["end_time"])    
@@ -108,9 +120,50 @@ def get_json_values(key, incoming_data, timespan=default_timespan, exclusions=No
                                 else:
                                     if each_key in yielded_json:
                                         #values.append(yielded_json[each_key])    
-                                        log_data[each_key] = yielded_json[each_key]
+                                        if isinstance(yielded_json[each_key], str):
+                                            log_data[each_key] = yielded_json[each_key].replace('\x00','') # added .replace('\x00','') on 8/17/24 to replace null bytes for pandas merge issues
+                                        elif isinstance(yielded_json[each_key], dict):
+                                            log_data[each_key] = json.dumps(yielded_json[each_key])
+                                        else:
+                                            log_data[each_key] = yielded_json[each_key]
                             if len(log_data) > 0:
-                                values.append(log_data)
+                                if compress_dates == True:
+                                    #dates = [dshield_parser.utils.json.time_to_day(timestamp) for timestamp in df["time"]
+                                    try:
+                                        # different time field for cowrie honeypot data
+                                        log_data["dates"] = dshield_parser.utils.json.time_to_day(log_data["timestamp"])
+                                        log_data.pop("timestamp")
+                                    except Exception as e:
+                                        # different time field for web or firewall honeypot data
+                                        log_data["dates"] = dshield_parser.utils.json.time_to_day(log_data["time"])
+                                        log_data.pop("time")                                        
+                                    values.append(log_data)
+                                #df = pd.concat([df, pd.DataFrame([log_data])], ignore_index=True) 
+                                ##new_df = pd.DataFrame([log_data])
+                                #new_df = pd.concat([new_df, temp_df])
+                                ##new_df = new_df.groupby(new_df.columns.tolist(),as_index=False, dropna=False).size()
+                                ##df = pd.concat([df, new_df]).groupby(new_df.columns.tolist(),as_index=False, dropna=False)['size'].sum()
+                        #new_df = new_df.groupby(new_df.columns.tolist(),as_index=False, dropna=False).size()
+                        #df = pd.concat([df, new_df]).groupby(new_df.columns.tolist(),as_index=False, dropna=False)['size'].sum()
+                        #new_df = pd.DataFrame()
+
+                    #summarization will be run at the end of every file processed
+                    if compress_dates == True:
+                        if day_count % 50 == 0 or len(values) > 500000:   #compress if 50 days of data or more than 500k records
+                            logging.info(f"50 days of data processed or more than 500k records found. Collapsing dataframe by date (grouping by).")
+                            file_df = pd.DataFrame(values)
+                            file_df = file_df.groupby(file_df.columns.tolist(),as_index=False, dropna=False).size()
+                            df = pd.concat([df, file_df]).groupby(file_df.columns.tolist(),as_index=False, dropna=False)['size'].sum()
+                            values = []
+                        elif end_date in file.name or (file.name == incoming_data[-1].name):
+                            logging.info(f"Last day being processed: {date}")
+                            file_df = pd.DataFrame(values)
+                            file_df = file_df.groupby(file_df.columns.tolist(),as_index=False, dropna=False).size()
+                            df = pd.concat([df, file_df]).groupby(file_df.columns.tolist(),as_index=False, dropna=False)['size'].sum()     
+                            values = []                      
+                    day_count += 1
+                     
+                                
             else:
                 for yielded_json in file_io.read_file_json_generator(file):
                     if "logs" in yielded_json:
@@ -132,12 +185,51 @@ def get_json_values(key, incoming_data, timespan=default_timespan, exclusions=No
                                     else:
                                         logging.debug(f"Filtered out data since {each_item['time']} is not between {timespan['start_time']} and {timespan['end_time']}")  
                             if len(log_data) > 0:
-                                values.append(log_data)                                               
+                                if compress_dates == True:
+                                    #dates = [dshield_parser.utils.json.time_to_day(timestamp) for timestamp in df["time"]
+                                    try:
+                                        # different time field for cowrie honeypot data
+                                        log_data["dates"] = dshield_parser.utils.json.time_to_day(log_data["timestamp"])
+                                        log_data.pop("timestamp")
+                                    except Exception as e:
+                                        # different time field for web or firewall honeypot data
+                                        log_data["dates"] = dshield_parser.utils.json.time_to_day(log_data["time"])
+                                        log_data.pop("time")                                        
+                                    values.append(log_data)
+                #end of processing firewall log
+                logging.info(f"Undated data file finished processing: {file}")
+                file_df = pd.DataFrame(values)
+                file_df = file_df.groupby(file_df.columns.tolist(),as_index=False, dropna=False).size()
+                df = pd.concat([df, file_df]).groupby(file_df.columns.tolist(),as_index=False, dropna=False)['size'].sum()     
+                values = []    
+                                                        
         else:
             for each_item in file:
                 if key in each_item:
-                    values.append(each_item[key])
-    return values
+                    if compress_dates == True:
+                        #dates = [dshield_parser.utils.json.time_to_day(timestamp) for timestamp in df["time"]
+                        try:
+                            # different time field for cowrie honeypot data
+                            log_data["dates"] = dshield_parser.utils.json.time_to_day(log_data["timestamp"])
+                            log_data.pop("timestamp")
+                        except Exception as e:
+                            # different time field for web or firewall honeypot data
+                            log_data["dates"] = dshield_parser.utils.json.time_to_day(log_data["time"])
+                            log_data.pop("time")                                        
+                        values.append(log_data)
+                logging.info(f"Firewall log finished processing: {file}")
+                file_df = pd.DataFrame(values)
+                file_df = file_df.groupby(file_df.columns.tolist(),as_index=False, dropna=False).size()
+                df = pd.concat([df, file_df]).groupby(file_df.columns.tolist(),as_index=False, dropna=False)['size'].sum()     
+                values = []                
+
+    # print out variable memory usage
+    for name, size in sorted(((name, sys.getsizeof(value)) for name, value in list(
+                            locals().items())), key= lambda x: -x[1])[:10]:
+        print("{:>30}: {:>8}".format(name, sizeof_fmt(size)))
+
+    df = df.groupby(df.columns.tolist(),as_index=False, dropna=False)['size'].sum()
+    return df
 
 def summarize_values(list):
     summary = {}

{dshield_parser-0.1.0 → dshield_parser-0.1.3}/dshield_parser.egg-info/PKG-INFO

@@ -1,13 +1,13 @@
-Metadata-Version: 2.1
-Name: dshield-parser
-Version: 0.1.0
-Summary: This program has utilities to download, parse and enrich DSHield honeypot data
-Home-page: https://github.com/jslagrew/dshield-parser
-Author: Jesse La Grew
-Author-email: jlagrew.github@proton.me
-License: UNKNOWN
-Platform: UNKNOWN
-Requires-Python: >3.6
-
-UNKNOWN
-
+Metadata-Version: 2.4
+Name: dshield_parser
+Version: 0.1.3
+Summary: This program has utilities to download, parse and enrich DSHield honeypot data
+Home-page: https://github.com/jslagrew/dshield-parser
+Author: Jesse La Grew
+Author-email: jlagrew.github@proton.me
+Requires-Python: >3.12.7
+Dynamic: author
+Dynamic: author-email
+Dynamic: home-page
+Dynamic: requires-python
+Dynamic: summary

{dshield_parser-0.1.0 → dshield_parser-0.1.3}/dshield_parser.egg-info/SOURCES.txt

@@ -1,3 +1,5 @@
+README.md
+pyproject.toml
 setup.py
 dshield_parser/__init__.py
 dshield_parser/__main__.py

dshield_parser-0.1.3/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"

{dshield_parser-0.1.0 → dshield_parser-0.1.3}/setup.cfg

@@ -1,4 +1,4 @@
-[egg_info]
-tag_build = 
-tag_date = 0
-
+[egg_info]
+tag_build = 
+tag_date = 0
+

{dshield_parser-0.1.0 → dshield_parser-0.1.3}/setup.py

@@ -1,18 +1,18 @@
-import setuptools
-
-setuptools.setup(
-    name="dshield_parser",
-    version="0.1.0",
-    author="Jesse La Grew",
-    author_email="jlagrew.github@proton.me",
-    description="This program has utilities to download, parse and enrich DSHield honeypot data",
-    url="https://github.com/jslagrew/dshield-parser",
-    packages=setuptools.find_packages(),
-    #entry_points = {
-    #    'console_scripts':
-    #    ['summarize_urls=utils.json:summarize_urls',
-    #     'summarize_urls=utils.json:summarize_commands']
-    #},
-    python_requires='>3.6'
-    #install_requires=['matplotlib', 'numpy', 'pandas', 'Requests', 'scikit_learn', 'shodan', 'SQLAlchemy']
-)
+import setuptools
+
+setuptools.setup(
+    name="dshield_parser",
+    version="0.1.3",
+    author="Jesse La Grew",
+    author_email="jlagrew.github@proton.me",
+    description="This program has utilities to download, parse and enrich DSHield honeypot data",
+    url="https://github.com/jslagrew/dshield-parser",
+    packages=setuptools.find_packages(),
+    #entry_points = {
+    #    'console_scripts':
+    #    ['summarize_urls=utils.json:summarize_urls',
+    #     'summarize_urls=utils.json:summarize_commands']
+    #},
+    python_requires='>3.12.7'
+    #install_requires=['matplotlib', 'numpy', 'pandas', 'Requests', 'scikit_learn', 'shodan', 'SQLAlchemy']
+)