skilleter-modules 0.0.1__py3-none-any.whl

skilleter_modules/tidy.py

@@ -0,0 +1,177 @@
+#! /usr/bin/env python3
+
+################################################################################
+""" Functions for making log files and similar more easily viewed and/or compared
+
+    Defaults to making things readable on a light background but has the
+    option to make things readable on a dark background or for removing
+    colours altogether.
+
+    Has functions for removing times, AWS ID values, colours, SHA1 values,
+    Also function for making ANSI codes more readable on light or dark backgrounds
+
+    Currently this is very cude and needs to be a lot more cleverer.
+
+    TODO: Handle multiple colour changes in a single ANSI sequence
+    TODO: Take account of the background colour when determining how to change the foreground colour
+    TODO: More colour conversions (currently only converts bright yellow on white to dark yellow)
+    TODO: Handle 256 colour codes
+    TODO: More time formats
+"""
+################################################################################
+
+import re
+
+################################################################################
+# Regular expressions
+
+# Match an ANSI colour control sequence
+
+ANSI_REGEXES = [
+    re.compile(r'\x1b\[([0-9][0-9;]*)*m'),
+    re.compile(r'\x1b\[m'),
+    re.compile(r'\x1b\[[0-9][A-Z]'),
+]
+
+# Match and ANSI colour control sequence and capture the colour bit
+
+RE_DEBUG = re.compile(r'\x1b(\[[0-9][0-9;]*m)')
+
+# Colour conversions for light backgrounds
+
+LIGHT_TABLE = [['1;33', '0;33']]
+
+# Colour conversions for dark backgrounds
+
+DARK_TABLE = []
+
+# Common time formats found in log files
+
+RE_TIME = [
+    {'regex': re.compile(r'[1-9][0-9]{3}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[.][0-9]+Z?'), 'replace': '{DATE+TIME}'},
+    {'regex': re.compile(r'[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}(Z|,[0-9]+)'), 'replace': '{TIME}'},
+    {'regex': re.compile(r'[0-9]{1,2}:[0-9]{2}:[0-9]{2}(\.[0-9]{1,6})?([+][0-9]{2}:[0-9]{2})?'), 'replace': '{TIME}'},
+
+    {'regex': re.compile(r'[1-9][0-9]{3}/[0-9][0-9]?/[1-9][0-9]'), 'replace': '{DATE}'},
+    {'regex': re.compile(r'[1-9][0-9]/[0-9][0-9]?/[1-9][0-9]{3}'), 'replace': '{DATE}'},
+    {'regex': re.compile(r'[0-9]{4}-[0-9]{2}-[0-9]{2}'), 'replace': '{DATE}'},
+    {'regex': re.compile(r'[0-9]{2}-[0-9]{2}-[0-9]{4}'), 'replace': '{DATE}'},
+
+    {'regex': re.compile(r'[0-9]([.][0-9]*)*\s*(second[s]?)'), 'replace': '{ELAPSED}'},
+
+    {'find': '{DATE} {TIME}', 'replace': '{DATE+TIME}'},
+    {'regex': re.compile(r'[0-9]+m *[0-9]+s'), 'replace': '{ELAPSED}'},
+]
+
+# SHA values
+
+RE_SHA256 = [
+    {'regex': re.compile(r'[0-9a-f]{64}'), 'replace': '{SHA256}'},
+]
+
+RE_SHA1 = [
+    {'regex': re.compile(r'[0-9a-f]{40}'), 'replace': '{SHA1}'},
+]
+
+# AWS ids
+
+RE_AWS = \
+    [
+        {'regex': re.compile(r'eni-0[0-9a-f]{16}'), 'replace': '{ENI-ID}'},
+        {'regex': re.compile(r'ami-0[0-9a-f]{16}'), 'replace': '{AMI-ID}'},
+        {'regex': re.compile(r'snap-0[0-9a-f]{16}'), 'replace': '{AMI-SNAP}'},
+        {'regex': re.compile(r'vol-0[0-9a-f]{16}'), 'replace': '{AMI-VOL}'},
+        {'regex': re.compile(r'sir-[0-9a-z]{8}'), 'replace': '{SPOT-INSTANCE}'},
+        {'regex': re.compile(r'i-0[0-9a-f]{16}'), 'replace': '{EC2-ID}'},
+        {'regex': re.compile(r'request id: [0-0a-f]{8}-[0-0a-f]{4}-[0-0a-f]{4}-[0-0a-f]{4}-[0-0a-f]{12}'),
+         'replace': 'request id: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'},
+    ]
+
+# Data transfer speeds
+
+RE_SPEED = \
+    [
+        {'regex': re.compile(r'[0-9.]+ *MB/s'), 'replace': '{SPEED}'},
+        {'regex': re.compile(r'[0-9.]+ *MiB/s'), 'replace': '{SPEED}'},
+    ]
+
+################################################################################
+
+def regex_replace(data, regexes):
+    """ Do a set of regex replacements """
+
+    for regex in regexes:
+        if 'find' in regex:
+            data = data.replace(regex['find'], regex['replace'])
+        elif 'regex' in regex:
+            data = regex['regex'].sub(regex['replace'], data)
+        else:
+            assert False
+
+    return data
+
+################################################################################
+
+def debug_format(data):
+    """ Make the ANSI colour codes in the specified string human-readable """
+
+    return RE_DEBUG.sub('{ESC\\1}', data)
+
+################################################################################
+
+def convert_ansi(data, light=True):
+    """ Use the conversion table to convert ANSI codes in the string for display
+        on a light or dark background """
+
+    table = LIGHT_TABLE if light else DARK_TABLE
+
+    for entry in table:
+        data = data.replace('\x1b[%s' % entry[0], '\x1b[%s' % entry[1])
+
+    return data
+
+################################################################################
+
+def remove_times(data):
+    """ Attempt to remove obvious time and duration references from a string """
+
+    return regex_replace(data, RE_TIME)
+
+################################################################################
+
+def remove_sha1(data):
+    """ Attempt to remove SHA1 references from a string """
+
+    return regex_replace(data, RE_SHA1)
+
+################################################################################
+
+def remove_sha256(data):
+    """ Attempt to remove SHA256 references from a string """
+
+    return regex_replace(data, RE_SHA256)
+
+################################################################################
+
+def remove_aws_ids(data):
+    """ Attempt to remove a variety of AWS ID references from a string """
+
+    return regex_replace(data, RE_AWS)
+
+################################################################################
+
+def remove_speeds(data):
+    """ Attempt to remove data transfer speed references from a string """
+
+    return regex_replace(data, RE_SPEED)
+
+################################################################################
+
+def remove_ansi(text):
+    """ Remove ANSI codes from a string """
+
+    if '\x1b' in text:
+        for regex in ANSI_REGEXES:
+            text = regex.sub('', text)
+
+    return text

skilleter_modules/venv_template.py

@@ -0,0 +1,47 @@
+TEMPLATE = \
+r"""#!/usr/bin/env bash
+
+set -e
+
+################################################################################
+
+VENV_NAME=$(basename "$0")
+VENV_DIR=__venv__
+
+GREEN="\e[42m"
+NORMAL="\e[0m"
+
+################################################################################
+
+function box()
+{
+   echo -e "${GREEN}################################################################################${NORMAL}"
+   echo -e "${GREEN}# $@${NORMAL}"
+   echo -e "${GREEN}################################################################################${NORMAL}"
+}
+
+################################################################################
+
+box "Creating & activating $VENV_NAME virtual environment"
+
+python3 -m venv $VENV_DIR
+
+source $VENV_DIR/bin/activate
+
+if [[ -f requirements.txt ]]
+then
+   box "Installing/Upgrading packages"
+
+   python3 -m pip install -r requirements.txt
+fi
+
+if [[ -f ${VENV_NAME} ]]
+then
+    box "Running ${VENV_NAME} script"
+
+    python3 ./${VENV_NAME}.py "$@"
+
+    deactivate
+fi
+"""
+

skilleter_modules-0.0.1.dist-info/METADATA

@@ -0,0 +1,61 @@
+Metadata-Version: 2.4
+Name: skilleter_modules
+Version: 0.0.1
+Summary: Modules used by my various Python projects (and hopefully useful to other people)
+Author-email: John Skilleter <john@skilleter.org.uk>
+Project-URL: Home, https://skilleter.org.uk
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests
+Requires-Dist: pygit2
+Dynamic: license-file
+
+# skilleter-modules
+
+A collection of Python modules used in my various other Python projects (readable, skilleter-thingy, skilleter-extras)
+
+## colour.py
+
+Functions for writing coloured text using ANSI control code sequences
+
+## dc_curses.py, dc_defaults.py, dc_util.py, dircolors.py
+
+Colour filesystem objects similar to ls.
+
+Large parts of this code are Copyright 2019 Allen Wild <allenwild93@gmail.com> released under the Apache-2.0 licence.
+
+## files.py
+
+Miscellaneous file-related functions - backup creation, file-type detection, etc.
+
+## git.py
+
+A git library
+
+## gitlab.py
+
+GitLab access module - due to be deprecated as I no longer support GitLab
+
+## path.py
+
+Filesystem path-related functions
+
+## popup.py
+
+Popup window class for curses
+
+## run.py
+
+Wrapper around the subprocess module, providing additional functionality
+
+## tfm_pane.py
+
+Console-pane-handling code for TFM - will probably be moved into the TFM project when that is split out from skilleter-thingy
+
+## tidy.py
+
+Functions for tidying log files - this may be moved into the skilleter-readable project at some point

skilleter_modules-0.0.1.dist-info/RECORD

@@ -0,0 +1,20 @@
+skilleter_modules/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+skilleter_modules/colour.py,sha256=8fWsuMvqUrC8IUOj1e4x7xt_4WFAgq_gzcc5CXorydo,11201
+skilleter_modules/dc_curses.py,sha256=fuuQPR11zV_akAhygL_cAhVLC5YAgKgowzlITVbETE8,8539
+skilleter_modules/dc_defaults.py,sha256=ahcteQvoWZrO5iTU68zkIY1Zex6iX5uR5ubwI4CCYBk,6170
+skilleter_modules/dc_util.py,sha256=Df73imXhHx3HzcPHiRcHAoea0e3HURdLcrolUsMhOFs,1783
+skilleter_modules/dircolors.py,sha256=oGj08wJ512q8c08KSaTbahbtW01HYDDpBnsXUGwLGMQ,12252
+skilleter_modules/files.py,sha256=YmmYgUoG3G1ZPrHmTuuTJqedLdVxyVsKvpFllLMVjdc,4703
+skilleter_modules/git.py,sha256=wPov5Q6p-17gD5yi2olz43RV1GNqpa4HaCwraHnD_iQ,43284
+skilleter_modules/gitlab.py,sha256=uXAF918xnPk6qQyiwPQDbMZfqtJzhiRqDS7yEtJEIAg,6079
+skilleter_modules/path.py,sha256=8uM2Q9zFRWv_SaVOX49PeecQXttl7J6lsmBuRXWsXKY,4732
+skilleter_modules/popup.py,sha256=TY9rpj4q8uZxerSt641LGUTy0TZgUjgfEX-CkRMuyek,2540
+skilleter_modules/run.py,sha256=yFFUNqVHOMFvs8bIZSufYQhW1nukOuB0FGH5xFW6GzE,10578
+skilleter_modules/tfm_pane.py,sha256=p-F_P7NObRni9MGB1RcEhcOG0NhnXM9yEf_VY9HHKBI,19741
+skilleter_modules/tidy.py,sha256=AQ2RawsZJg6WHrgayi_ZptFL9occ7suSdCHbU3P-cys,5971
+skilleter_modules/venv_template.py,sha256=ZfUvi8qFNGrk7J030Zy57xjwMtfIArJyqa-MqafyjVk,1016
+skilleter_modules-0.0.1.dist-info/licenses/LICENSE,sha256=ljOS4DjXvqEo5VzGfdaRwgRZPbNScGBmfwyC8PChvmQ,32422
+skilleter_modules-0.0.1.dist-info/METADATA,sha256=yS3_SIUpT7n3sOM39FwnuGZQ1illk2xDRG3OSJ0czks,1675
+skilleter_modules-0.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+skilleter_modules-0.0.1.dist-info/top_level.txt,sha256=DMa0AkGOjoDHiHDG6Nw1jtdUylaMm5WkF6uODT9yxJc,18
+skilleter_modules-0.0.1.dist-info/RECORD,,

skilleter_modules-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+