dshield-parser 0.1.3__tar.gz → 0.1.5__tar.gz

{dshield_parser-0.1.3 → dshield_parser-0.1.5}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: dshield_parser
-Version: 0.1.3
+Version: 0.1.5
 Summary: This program has utilities to download, parse and enrich DSHield honeypot data
 Home-page: https://github.com/jslagrew/dshield-parser
 Author: Jesse La Grew
 Author-email: jlagrew.github@proton.me
-Requires-Python: >3.12.7
+Requires-Python: >=3.12.7
 Dynamic: author
 Dynamic: author-email
 Dynamic: home-page

{dshield_parser-0.1.3 → dshield_parser-0.1.5}/dshield_parser/utils/json.py

@@ -16,6 +16,19 @@ import pandas as pd
 default_datespan={'start_date': '1970-01-01', 'end_date': '3000-01-01'}
 default_timespan={"start_time": 0, "end_time": 32516881838}
 
+def _groupby_size(df):
+    """
+    Compatibility wrapper for groupby().size() that works with pandas 2.x and 3.x.
+    In pandas 3.0, 'size' became a reserved GroupBy method name and can no longer
+    be used as a column selector after groupby(). This helper renames the column
+    temporarily to avoid the conflict.
+    """
+    group_cols = df.columns.difference(['size']).tolist()
+    result = df.rename(columns={'size': '_count'})
+    result = result.groupby(result.columns.difference(['_count']).tolist(), as_index=False, dropna=False)['_count'].sum()
+    result = result.rename(columns={'_count': 'size'})
+    return result
+
 def print_json_pretty(incoming_data):
     for json_data in incoming_data:
         if os.path.exists(json_data):
@@ -153,13 +166,13 @@ def get_json_values(key, incoming_data, timespan=default_timespan, exclusions=No
                             logging.info(f"50 days of data processed or more than 500k records found. Collapsing dataframe by date (grouping by).")
                             file_df = pd.DataFrame(values)
                             file_df = file_df.groupby(file_df.columns.tolist(),as_index=False, dropna=False).size()
-                            df = pd.concat([df, file_df]).groupby(file_df.columns.tolist(),as_index=False, dropna=False)['size'].sum()
+                            df = _groupby_size(pd.concat([df, file_df]))
                             values = []
                         elif end_date in file.name or (file.name == incoming_data[-1].name):
                             logging.info(f"Last day being processed: {date}")
                             file_df = pd.DataFrame(values)
                             file_df = file_df.groupby(file_df.columns.tolist(),as_index=False, dropna=False).size()
-                            df = pd.concat([df, file_df]).groupby(file_df.columns.tolist(),as_index=False, dropna=False)['size'].sum()     
+                            df = _groupby_size(pd.concat([df, file_df]))
                             values = []                      
                     day_count += 1
                      
@@ -200,7 +213,7 @@ def get_json_values(key, incoming_data, timespan=default_timespan, exclusions=No
                 logging.info(f"Undated data file finished processing: {file}")
                 file_df = pd.DataFrame(values)
                 file_df = file_df.groupby(file_df.columns.tolist(),as_index=False, dropna=False).size()
-                df = pd.concat([df, file_df]).groupby(file_df.columns.tolist(),as_index=False, dropna=False)['size'].sum()     
+                df = _groupby_size(pd.concat([df, file_df]))
                 values = []    
                                                         
         else:
@@ -220,7 +233,7 @@ def get_json_values(key, incoming_data, timespan=default_timespan, exclusions=No
                 logging.info(f"Firewall log finished processing: {file}")
                 file_df = pd.DataFrame(values)
                 file_df = file_df.groupby(file_df.columns.tolist(),as_index=False, dropna=False).size()
-                df = pd.concat([df, file_df]).groupby(file_df.columns.tolist(),as_index=False, dropna=False)['size'].sum()     
+                df = _groupby_size(pd.concat([df, file_df]))
                 values = []                
 
     # print out variable memory usage
@@ -228,7 +241,7 @@ def get_json_values(key, incoming_data, timespan=default_timespan, exclusions=No
                             locals().items())), key= lambda x: -x[1])[:10]:
         print("{:>30}: {:>8}".format(name, sizeof_fmt(size)))
 
-    df = df.groupby(df.columns.tolist(),as_index=False, dropna=False)['size'].sum()
+    df = _groupby_size(df)
     return df
 
 def summarize_values(list):
@@ -482,7 +495,7 @@ def convert_to_datetime(dictionary):
     return new_dates
 
 def convert_to_epoch(date):
-    return datetime.strptime(date, "%Y-%m-%d")
+    return datetime.strptime(date, "%Y-%m-%d").timestamp()
 
 def convert_to_date(timestamp):
     return datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d')

{dshield_parser-0.1.3 → dshield_parser-0.1.5}/dshield_parser.egg-info/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: dshield_parser
-Version: 0.1.3
+Version: 0.1.5
 Summary: This program has utilities to download, parse and enrich DSHield honeypot data
 Home-page: https://github.com/jslagrew/dshield-parser
 Author: Jesse La Grew
 Author-email: jlagrew.github@proton.me
-Requires-Python: >3.12.7
+Requires-Python: >=3.12.7
 Dynamic: author
 Dynamic: author-email
 Dynamic: home-page

{dshield_parser-0.1.3 → dshield_parser-0.1.5}/setup.py

@@ -2,7 +2,7 @@ import setuptools
 
 setuptools.setup(
     name="dshield_parser",
-    version="0.1.3",
+    version="0.1.5",
     author="Jesse La Grew",
     author_email="jlagrew.github@proton.me",
     description="This program has utilities to download, parse and enrich DSHield honeypot data",
@@ -13,6 +13,6 @@ setuptools.setup(
     #    ['summarize_urls=utils.json:summarize_urls',
     #     'summarize_urls=utils.json:summarize_commands']
     #},
-    python_requires='>3.12.7'
+    python_requires='>=3.12.7'
     #install_requires=['matplotlib', 'numpy', 'pandas', 'Requests', 'scikit_learn', 'shodan', 'SQLAlchemy']
 )