dshield-parser 0.0.0__py3-none-any.whl

dshield_parser/utils/sql.py

@@ -0,0 +1,204 @@
+import logging
+import sqlite3
+from io import StringIO
+import pandas as pd
+from sqlalchemy import create_engine, select
+
+def initialize_database(filename, table_name, column_names, column_types):
+    incrementer = 0
+    columns = ""
+    while incrementer < len(column_names):
+        columns += column_names[incrementer] + " " + column_types[incrementer]
+        incrementer += 1        
+        if incrementer != len(column_names):
+            columns += ", "
+
+    con = sqlite3.connect(filename)
+    cur = con.cursor()
+    sql = f"CREATE TABLE IF NOT EXISTS {table_name}({columns})"
+    logging.debug(sql)
+    logging.info(f"Creating table {table_name} in {filename} with columns {columns}")
+
+    cur.execute(sql)
+    con.commit()
+    con.close()
+
+def select(filename, columns, tables, conditionals=None):
+    con = sqlite3.connect(filename)
+    df = pd.read_sql_query(f'''SELECT {columns} from {tables}''', con)
+    logging.debug(df.head())
+    con.close()
+    return df
+
+def insert(filename, table_name, column_names, values):
+    if isinstance(column_names, list):
+        incrementer = 0
+        field_names = ""
+        value_placeholders
+        while incrementer < len(column_names):
+            field_names += column_names[incrementer]
+            value_placeholders += "?"
+            incrementer += 1   
+            if incrementer != len(column_names):
+                field_names += ", "
+                value_placeholders += ", "
+    else:
+        columns = column_names
+        values = [values]
+        value_placeholders = "?"
+
+    con = sqlite3.connect(filename)
+    cur = con.cursor()
+
+    #create table if it doesn't exist
+    sql = f"CREATE TABLE IF NOT EXISTS {table_name} ({columns})"
+    logging.debug(sql)
+    logging.debug(f"Creating table {table_name} in {filename} with columns {columns}")
+    try:
+        cur.execute(sql)
+    except Exception as e: 
+        logging.error(f"Issue with SQL statement '{sql}': {e}")
+    con.commit()
+
+    sql = f"INSERT INTO {table_name} ({columns}) VALUES ({value_placeholders})"
+    try:
+        cur.execute(sql, tuple(values))
+    except Exception as e: 
+        logging.error(f"Issue with SQL statement '{sql}': {e}")
+        if "no column named" in e:
+            add_column(filename, table_name, columns)
+    con.commit()
+
+def insert_dict(filename, dict, table_name):
+    df = pd.DataFrame(dict)
+    engine = create_engine(f'sqlite:///{filename}')
+    logging.info(f"Starting insert of dictionary into '{filename}'")
+
+    try:
+        df.to_sql(table_name, con=engine, if_exists='append')
+
+    except Exception as e:
+        logging.error(f"Issue while attempting to insert dictionary into database: {e}")
+        if "no column named" in str(e):
+            pass
+            #add_column(filename, table_name, column_name)
+            #df.to_sql(table_name, con=engine, if_exists='append')
+    logging.info(f"Completed insert of dictionary into '{filename}'")
+
+def insert_list(filename, dict, table_name):
+    df = pd.DataFrame(dict)
+    engine = create_engine(f'sqlite:///{filename}')
+    logging.info(f"Starting insert of dictionary into '{filename}'")
+
+    try:
+        df.to_sql(table_name, con=engine, if_exists='append')
+
+    except Exception as e:
+        logging.error(f"Issue while attempting to insert dictionary into database: {e}")
+        if "no column named" in str(e):
+            pass
+            #add_column(filename, table_name, column_name)
+            #df.to_sql(table_name, con=engine, if_exists='append')
+    logging.info(f"Completed insert of dictionary into '{filename}'")
+
+def insert_df(filename, df, table_name):
+    engine = create_engine(f'sqlite:///{filename}')
+    logging.info(f"Starting insert of dictionary into '{filename}'")
+
+    try:
+        df.to_sql(table_name, con=engine, if_exists='append', index=False)
+
+    except Exception as e:
+        logging.error(f"Issue while attempting to insert dictionary into database: {e}")
+        if "no column named" in str(e):
+            pass
+    logging.info(f"Completed insert of dictionary into '{filename}'")
+
+
+def update(filename, table_name, column_name, value):
+
+    #sql = '''UPDATE commands SET azurehp2=? WHERE command=? and quantity=?'''
+    #sql = f'''UPDATE malware SET {honeypot[incrementer]}_quantity=?, {honeypot[incrementer]}=? WHERE hash=?'''
+    #
+    pass
+
+#def select(filename, table_names, column_names, value_comparisons=None):
+#    incrementer = 0
+#    columns = ""
+#    while incrementer < len(column_names):
+#        columns += column_names[incrementer] + " " + column_names[incrementer]
+#        incrementer += 1        
+#        if incrementer != len(column_names):
+#            columns += ", "
+#
+#    incrementer = 0
+#    tables = ""
+#    while incrementer < len(table_names):
+#        tables += table_names[incrementer] + " " + table_names[incrementer]
+#        incrementer += 1        
+#        if incrementer != len(table_names):
+#            table += ", "
+
+#    con = sqlite3.connect(filename)
+#    cur = con.cursor()
+#    sql = f"SELECT {columns} from {tables}"
+#    if value_comparisons is not None:
+#        sql += "where {value_comparisons}"
+#    cur.execute(sql)
+#    con.commit()
+#    return cur.fetchall() # return rows of data
+
+def add_column(filename, table_name, column_name, column_type):
+    con = sqlite3.connect(filename)
+    cur = con.cursor()
+    sql = f"ALTER TABLE {table_name} ADD COLUMN {column_name} {column_type}"
+    cur.execute(sql)
+    con.commit()
+
+def value_exists(filename, table_names, column_names, value_comparisons):
+    incrementer = 0
+    columns = ""
+    while incrementer < len(column_names):
+        columns += column_names[incrementer] + " " + column_names[incrementer]
+        incrementer += 1        
+        if incrementer != len(column_names):
+            columns += ", "
+
+    incrementer = 0
+    tables = ""
+    while incrementer < len(table_names):
+        tables += table_names[incrementer] + " " + table_names[incrementer]
+        incrementer += 1        
+        if incrementer != len(table_names):
+            table += ", "
+
+    con = sqlite3.connect(filename)
+    cur = con.cursor()
+    sql = f"SELECT {columns} from {tables} where {value_comparisons}"
+    try:
+        cur.execute(sql)
+        rows = cur.fetchall()
+    except Exception as e:
+        logging.error(f"Issue running query '{sql}': {e}")
+        return False
+    
+    if len(rows) > 0:
+        return True
+    else:
+        return False
+
+def add_table(filename, table_name, columns):
+    con = sqlite3.connect(filename)
+    cur = con.cursor()
+    sql = f"CREATE TABLE IF NOT EXISTS {table_name}({columns})"
+    logging.debug(sql)
+    logging.info(f"Creating table {table_name} in {filename} with columns {columns}")
+    cur.execute(sql)
+    con.commit()
+
+def drop_table(filename, table_name):
+    con = sqlite3.connect(filename)
+    cur = con.cursor()
+    sql = f"DROP TABLE {table_name}"
+    cur.execute(sql)
+    con.commit()

dshield_parser/web_processor/__init__.py

@@ -0,0 +1 @@
+from dshield_parser.web_processor import *

dshield_parser/web_processor/reports.py

@@ -0,0 +1,76 @@
+from dshield_parser.utils import json as dshield_json
+import itertools
+
+default_datespan = {'start_date': '1970-01-01', 'end_date': '3000-01-01'}
+
+def get_top_10_ips(json_data, datespan=default_datespan):
+    return dshield_json.get_top_10("sip", json_data, datespan)
+
+def get_summary_ips(json_data, datespan=default_datespan, exclusions=None):
+    return dshield_json.get_summary_values("sip", json_data, datespan, exclusions)
+
+def get_top_10_headers(json_data, datespan=default_datespan):
+    return dshield_json.get_top_10("headers", json_data, datespan)
+
+def get_summary_headers(json_data, datespan=default_datespan, exclusions=None):
+    return dshield_json.get_summary_values("headers", json_data, datespan, exclusions)
+
+def get_top_10_method(json_data, datespan=default_datespan):
+    return dshield_json.get_top_10("method", json_data, datespan)
+
+def get_summary_methods(json_data, datespan=default_datespan, exclusions=None):
+    return dshield_json.get_summary_values("method", json_data, datespan, exclusions)
+
+def get_top_10_url(json_data, datespan=default_datespan):
+    return dshield_json.get_top_10("url", json_data, datespan)
+
+def get_summary_urls(json_data, datespan=default_datespan, exclusions=None):
+    return dshield_json.get_summary_values("url", json_data, datespan, exclusions)
+
+def get_all_urls(json_data, datespan=default_datespan, exclusions=None):
+    return dshield_json.get_json_values("url", json_data, datespan, exclusions)
+
+def get_top_10_data(json_data, datespan=default_datespan):
+    return dshield_json.get_top_10("data", json_data, datespan)
+
+def get_summary_data(json_data, datespan=default_datespan, exclusions=None):
+    return dshield_json.get_summary_values("data", json_data, datespan, exclusions)
+
+def get_top_10_useragent(json_data, datespan=default_datespan):
+    return dshield_json.get_top_10("useragent", json_data, datespan)
+
+def get_summary_useragents(json_data, datespan=default_datespan, exclusions=None):
+    return dshield_json.get_summary_values("useragent", json_data, datespan, exclusions)
+
+def get_top_10_version(json_data, datespan=default_datespan):
+    return dshield_json.get_top_10("version", json_data, datespan)
+
+def get_summary_versions(json_data, datespan=default_datespan, exclusions=None):
+    return dshield_json.get_summary_values("version", json_data, datespan, exclusions)
+
+def get_top_10_responseid(json_data, datespan=default_datespan):
+    return dshield_json.get_top_10("response_id", json_data, datespan)
+
+def get_summary_responseids(json_data, datespan=default_datespan, exclusions=None):
+    return dshield_json.get_summary_values("response_id", json_data, datespan, exclusions)
+
+def get_top_10_signatureid(json_data, datespan=default_datespan):
+    return dshield_json.get_top_10("signature_id", json_data, datespan)
+
+def get_summary_signatureids(json_data, datespan=default_datespan, exclusions=None):
+    return dshield_json.get_summary_values("signature_id", json_data, datespan, exclusions)
+
+def get_top_10_multiple(fields, json_data, datespan=default_datespan):
+    return dshield_json.get_top_10(fields, json_data, datespan)
+
+def get_summary_multiple(fields, json_data, datespan=default_datespan, exclusions=None):
+    return dshield_json.get_summary_values(fields, json_data, datespan, exclusions)
+
+def get_summary_all(json_data, datespan=default_datespan, exclusions=None):
+    keys = dshield_json.get_json_keys(json_data)
+    return keys, dshield_json.get_summary_values(keys, json_data, datespan, exclusions)
+
+def get_volume_over_time(json_data, datespan=None):
+    if datespan is None:
+        datespan = default_datespan    
+    return dshield_json.get_summary_values("time", json_data, datespan)

dshield_parser-0.0.0.dist-info/METADATA

@@ -0,0 +1,3 @@
+Metadata-Version: 2.4
+Name: dshield_parser
+Version: 0.0.0

dshield_parser-0.0.0.dist-info/RECORD

@@ -0,0 +1,26 @@
+dshield_parser/__init__.py,sha256=YvdQXG5JCiwHZ-vGJR4xkb8AMTBT5sqnpXSCIOlumJk,21
+dshield_parser/__main__.py,sha256=KS8XiLyk1SPk7fvSgTiFg2rphSBR3lOg3r09CIHRh2o,651
+dshield_parser/cowrie_processor/__init__.py,sha256=5Il9wz4Tp-ROf9RYO6vO6nbTRfUvUfxUwtLOgQz7aGk,45
+dshield_parser/cowrie_processor/__main__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dshield_parser/cowrie_processor/reports.py,sha256=-ImDweafXxUJLsTRkTRT7poAEhQ8Xw8wc3DTfBR8o5o,3895
+dshield_parser/firewall_processor/__init__.py,sha256=RvVuspWYcMlOk3586rzlmloni2-vByAK9MCam6ExOl4,47
+dshield_parser/firewall_processor/__main__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dshield_parser/firewall_processor/reports.py,sha256=kE1Vn9i0e9_kL0IL4jh7oN38Jl1NxSfznco6uDDGuv4,3571
+dshield_parser/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dshield_parser/test/test_json.py,sha256=Fi-IfrZwgRwG-S6JHZlyW8CqgN8qxQMEOCitto890tw,19041
+dshield_parser/utils/__init__.py,sha256=5xDI_zLzsLufARfazcSBdnTqzlwxVzaWB2p8k-Qi3LY,34
+dshield_parser/utils/__main__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dshield_parser/utils/enrichment.py,sha256=IEsj8LuV-8f0jr-xMuu4m_dKSY3xya0X9BYMFegW4JE,16274
+dshield_parser/utils/file_io.py,sha256=vinxM70ZVawA0Pm2nEFLGa89mc8LHACS9AMI4PLnJLI,3911
+dshield_parser/utils/graphs.py,sha256=_Qoj0hevZOyFJXa5WmJpu-X1PARb8RuBSEjNy-Jkvwo,4474
+dshield_parser/utils/json.py,sha256=fA_VNKvgvlQcLvVcdb-ahYmyDOoYVaQ_pPwuV97ANjk,27959
+dshield_parser/utils/ml.py,sha256=LrmUbyDqn_L-k5M2JOjryQ9_Y1HLNY4hegCZUqsG0L8,3235
+dshield_parser/utils/sql.py,sha256=hr-WcGwL816Nla3hLckjJ7p0xsELD-bjEyWY0Aco_Fc,7117
+dshield_parser/utils/ssh.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dshield_parser/web_processor/__init__.py,sha256=ib_dZ4okzZzvre7FY9JKUsFGuF2IqNAB3C3Udk22szg,42
+dshield_parser/web_processor/__main__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dshield_parser/web_processor/reports.py,sha256=P6bKqZTm3ndcduSmwvSK7xB81LIu5zua0ZD6891St88,3768
+dshield_parser-0.0.0.dist-info/METADATA,sha256=xEDT6-9HZFEgkyZITn_9xBAfBlJ_0iSZh6-HA69G0S8,58
+dshield_parser-0.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+dshield_parser-0.0.0.dist-info/top_level.txt,sha256=5fVrWGx44ZfX9aV-bPlXsKvwBhNm701Z27I7Baz71n0,15
+dshield_parser-0.0.0.dist-info/RECORD,,

dshield_parser-0.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dshield_parser-0.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+dshield_parser