bleanser 0.1.20250119__tar.gz

bleanser-0.1.20250119/.ci/release

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+'''
+Run [[file:scripts/release][scripts/release]] to deploy Python package onto [[https://pypi.org][PyPi]] and [[https://test.pypi.org][test PyPi]].
+
+The script expects =TWINE_PASSWORD= environment variable to contain the [[https://pypi.org/help/#apitoken][PyPi token]] (not the password!).
+
+The script can be run manually.
+It's also running as =pypi= job in [[file:.github/workflows/main.yml][Github Actions config]]. Packages are deployed on:
+- every master commit, onto test pypi
+- every new tag, onto production pypi
+
+You'll need to set =TWINE_PASSWORD= and =TWINE_PASSWORD_TEST= in [[https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets#creating-encrypted-secrets][secrets]]
+for Github Actions deployment to work.
+'''
+
+import os
+import sys
+from pathlib import Path
+from subprocess import check_call
+import shutil
+
+is_ci = os.environ.get('CI') is not None
+
+def main() -> None:
+    import argparse
+    p = argparse.ArgumentParser()
+    p.add_argument('--test', action='store_true', help='use test pypi')
+    args = p.parse_args()
+
+    extra = []
+    if args.test:
+        extra.extend(['--repository', 'testpypi'])
+
+    root = Path(__file__).absolute().parent.parent
+    os.chdir(root) # just in case
+
+    if is_ci:
+        # see https://github.com/actions/checkout/issues/217
+        check_call('git fetch --prune --unshallow'.split())
+
+    dist = root / 'dist'
+    if dist.exists():
+        shutil.rmtree(dist)
+
+    check_call(['python3', '-m', 'build'])
+
+    TP = 'TWINE_PASSWORD'
+    password = os.environ.get(TP)
+    if password is None:
+        print(f"WARNING: no {TP} passed", file=sys.stderr)
+        import pip_secrets
+        password = pip_secrets.token_test if args.test else pip_secrets.token # meh
+
+    check_call([
+        'python3', '-m', 'twine',
+        'upload', *dist.iterdir(),
+        *extra,
+    ], env={
+        'TWINE_USERNAME': '__token__',
+        TP: password,
+        **os.environ,
+    })
+
+
+if __name__ == '__main__':
+    main()

bleanser-0.1.20250119/.ci/release-uv

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+'''
+Deploys Python package onto [[https://pypi.org][PyPi]] or [[https://test.pypi.org][test PyPi]].
+
+- running manually
+
+  You'll need =UV_PUBLISH_TOKEN= env variable
+
+- running on Github Actions
+
+  Instead of env variable, relies on configuring github as Trusted publisher (https://docs.pypi.org/trusted-publishers/) -- both for test and regular pypi
+
+  It's running as =pypi= job in [[file:.github/workflows/main.yml][Github Actions config]].
+  Packages are deployed on:
+  - every master commit, onto test pypi
+  - every new tag, onto production pypi
+'''
+
+UV_PUBLISH_TOKEN = 'UV_PUBLISH_TOKEN'
+
+import argparse
+import os
+import shutil
+from pathlib import Path
+from subprocess import check_call
+
+is_ci = os.environ.get('CI') is not None
+
+def main() -> None:
+    p = argparse.ArgumentParser()
+    p.add_argument('--use-test-pypi', action='store_true')
+    args = p.parse_args()
+
+    publish_url = ['--publish-url', 'https://test.pypi.org/legacy/'] if args.use_test_pypi else []
+
+    root = Path(__file__).absolute().parent.parent
+    os.chdir(root) # just in case
+
+    if is_ci:
+        # see https://github.com/actions/checkout/issues/217
+        check_call('git fetch --prune --unshallow'.split())
+
+    # TODO ok, for now uv won't remove dist dir if it already exists
+    #  https://github.com/astral-sh/uv/issues/10293
+    dist = root / 'dist'
+    if dist.exists():
+        shutil.rmtree(dist)
+
+    # todo what is --force-pep517?
+    check_call(['uv', 'build'])
+
+    if not is_ci:
+        # CI relies on trusted publishers so doesn't need env variable
+        assert UV_PUBLISH_TOKEN in os.environ, f'no {UV_PUBLISH_TOKEN} passed'
+
+    check_call(['uv', 'publish', *publish_url])
+
+
+if __name__ == '__main__':
+    main()

bleanser-0.1.20250119/.ci/run

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -eu
+
+cd "$(dirname "$0")"
+cd .. # git root
+
+if ! command -v sudo; then
+    # CI or Docker sometimes doesn't have it, so useful to have a dummy
+    function sudo {
+        "$@"
+    }
+fi
+
+# --parallel-live to show outputs while it's running
+tox_cmd='run-parallel --parallel-live'
+if [ -n "${CI-}" ]; then
+    # install OS specific stuff here
+    case "$OSTYPE" in
+    darwin*) 
+        # macos
+        brew install libmagic # for python-magic
+        brew install diffutils # for GNU diff
+        ;;
+    cygwin* | msys* | win*)
+        # windows
+        # ugh. parallel stuff seems super flaky under windows, some random failures, "file used by other process" and crap like that
+        tox_cmd='run'
+        ;;
+    *)
+        # must be linux?
+        :
+        ;;
+    esac
+fi
+
+# NOTE: expects uv installed
+uv tool run --with tox-uv tox $tox_cmd "$@"

bleanser-0.1.20250119/.github/workflows/main.yml

@@ -0,0 +1,99 @@
+# see https://github.com/karlicoss/pymplate for up-to-date reference
+
+name: CI
+on:
+  push:
+    branches: '*'
+    tags: 'v[0-9]+.*' # only trigger on 'release' tags for PyPi
+    # Ideally I would put this in the pypi job... but github syntax doesn't allow for regexes there :shrug:
+    # P.S. fuck made up yaml DSLs.
+  pull_request: # needed to trigger on others' PRs
+  # Note that people who fork it need to go to "Actions" tab on their fork and click "I understand my workflows, go ahead and enable them".
+  workflow_dispatch: # needed to trigger workflows manually
+    # todo cron?
+    inputs:
+      debug_enabled:
+        type: boolean
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
+
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: [ubuntu-latest, macos-latest] # , windows-latest]
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
+        exclude: [
+          # min and max version is enough for osx running (it's kinda slow)
+          {platform: macos-latest, python-version: '3.10'},
+          {platform: macos-latest, python-version: '3.11'},
+          {platform: macos-latest, python-version: '3.12'},
+        ]
+    runs-on: ${{ matrix.platform }}
+
+    steps:
+    # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
+    - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+      
+    - uses: astral-sh/setup-uv@v5
+      with:
+        enable-cache: false  # we don't have lock files, so can't use them as cache key
+
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+        fetch-depth: 0  # nicer to have all git history when debugging/for tests
+
+    - uses: mxschmitt/action-tmate@v3
+      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+
+    # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd...
+    - run: bash .ci/run
+
+    - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
+      uses: actions/upload-artifact@v4
+      with:
+        include-hidden-files: true
+        name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }}
+        path: .coverage.mypy/
+
+
+  pypi:
+    runs-on: ubuntu-latest
+    needs: [build] # add all other jobs here
+    permissions:
+      # necessary for Trusted Publishing
+      id-token: write
+    steps:
+    # ugh https://github.com/actions/toolkit/blob/main/docs/commands.md#path-manipulation
+    - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+
+    - uses: astral-sh/setup-uv@v5
+      with:
+        enable-cache: false  # we don't have lock files, so can't use them as cache key
+
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: 'release to test pypi'
+      # always deploy merged master to test pypi
+      if: github.event_name != 'pull_request' && github.event.ref == 'refs/heads/master'
+      run: .ci/release-uv --use-test-pypi
+
+    - name: 'release to pypi'
+      # always deploy tags to release pypi
+      # NOTE: release tags are guarded by on: push: tags on the top
+      if: github.event_name != 'pull_request' && startsWith(github.event.ref, 'refs/tags')
+      run: .ci/release-uv

bleanser-0.1.20250119/.gitignore

@@ -0,0 +1,160 @@
+
+# Created by https://www.gitignore.io/api/python,emacs
+# Edit at https://www.gitignore.io/?templates=python,emacs
+
+### Emacs ###
+# -*- mode: gitignore; -*-
+*~
+\#*\#
+/.emacs.desktop
+/.emacs.desktop.lock
+*.elc
+auto-save-list
+tramp
+.\#*
+
+# Org-mode
+.org-id-locations
+*_archive
+
+# flymake-mode
+*_flymake.*
+
+# eshell files
+/eshell/history
+/eshell/lastdir
+
+# elpa packages
+/elpa/
+
+# reftex files
+*.rel
+
+# AUCTeX auto folder
+/auto/
+
+# cask packages
+.cask/
+dist/
+
+# Flycheck
+flycheck_*.el
+
+# server auth directory
+/server/
+
+# projectiles files
+.projectile
+
+# directory configuration
+.dir-locals.el
+
+# network security
+/network-security.data
+
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# End of https://www.gitignore.io/api/python,emacs

bleanser-0.1.20250119/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Dmitrii Gerasimov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

bleanser-0.1.20250119/PKG-INFO

@@ -0,0 +1,45 @@
+Metadata-Version: 2.2
+Name: bleanser
+Version: 0.1.20250119
+Author-email: "Dima Gerasimov (@karlicoss)" <karlicoss@gmail.com>
+Maintainer-email: "Dima Gerasimov (@karlicoss)" <karlicoss@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2021 Dmitrii Gerasimov
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/karlicoss/bleanser
+Requires-Python: >=3.9
+Requires-Dist: more-itertools
+Requires-Dist: typing-extensions
+Requires-Dist: click
+Requires-Dist: plumbum
+Requires-Dist: kompress
+Provides-Extra: extra
+Requires-Dist: python-magic; extra == "extra"
+Requires-Dist: logzero; extra == "extra"
+Provides-Extra: json
+Requires-Dist: orjson; extra == "json"
+Provides-Extra: xml
+Requires-Dist: lxml; extra == "xml"
+Provides-Extra: zstd
+Requires-Dist: kompress[zstd]; extra == "zstd"
+Provides-Extra: hpi
+Requires-Dist: HPI; extra == "hpi"

bleanser-0.1.20250119/README.md

@@ -0,0 +1,197 @@
+bleanser or 'backup cleanser' is a tool for cleaning old and redundant backups
+
+## Installing
+
+To install, run: `pip install bleanser`.
+
+There are also extra install options. You can use none or multiple depending on your needs, i.e. `pip install bleanser[flavor1,flavor2]`:
+
+- `bleanser[extra]` : some recommended but optional extras
+- `bleanser[json]` : dependencies for JSON based modules
+- `bleanser[xml]` : dependencies for XML based modules
+- `bleanser[HPI]` : dependencies for [HPI](https://github.com/karlicoss/HPI) based modules
+
+See `optional-dependencies` section in [pyproject.toml](pyproject.toml) if you're curious what libraries these extras pull.
+
+## What bleanser does
+
+In this context, backup typically means something like a GDPR export, an XML or JSON file which includes your data from some website/API, or a sqlite database from an application
+
+<https://beepb00p.xyz/exobrain/projects/bleanser.html>
+
+This is used to find 'redundant backups'. As an example, say you save your data to a JSON file by making API requests to some API service once a day. If your export of the data you exported today is a [superset](https://en.wikipedia.org/wiki/Subset) of the export yesterday, you know you can safely delete the old file and still have a complete backup of your data. This helps:
+
+- save on disk space
+- save of data access time; how long it takes to parse all your input files (see [data access layer](https://beepb00p.xyz/exports.html#dal))
+
+This works for both [full](https://beepb00p.xyz/exports.html#full) (you're able to get all your data from a service) and [incremental](https://beepb00p.xyz/exports.html#incremental) exports.
+
+This is especially relevant for incremental data exports, as they're harder to reason about. So, this handles the complex bits of diffing adjacent backups.
+
+As an example of an incremental export, imagine the service you were using only gave you access to the latest 3 items in your history (a real example of this is the [github activity feed](https://github.com/karlicoss/ghexport))
+
+| Day 1 | Day 2 | Day 3 |
+| ----- | ----- | ----- |
+| A     | B     | C     |
+| B     | C     | D     |
+| C     | D     | E     |
+
+To parse this in your [data access layer](https://beepb00p.xyz/exports.html#dal), you could imagine something like this:
+
+```python
+events = set()
+for file in inputs:
+    for line in file:
+        events.add(line)
+# events is now {'A', 'B', 'C', 'D', 'E'}
+```
+
+You might notice that if you removed 'Day 2', you'd still have an accurate backup, and we'd still have all 5 items, but its not obvious you can remove it since none of these are supersets of each other.
+
+`bleanser` is meant to solve this problem in a data agnostic way, so any export can be converted to a normalised representation, and those can be compared against each other to find redundant data
+
+Sidenote: in particular this is describing how `--multiway` finds redundant files, see [`options.md`](./doc/options.md) for more info
+
+## How it works
+
+This has `Normaliser`s for different data sources (see [modules](src/bleanser/modules)), and generally follows a pattern like this:
+
+```python
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Iterator
+
+from bleanser.core.processor import BaseNormaliser, unique_file_in_tempdir
+
+class Normaliser(BaseNormaliser):
+
+    @contextmanager
+    def normalise(self, *, path: Path) -> Iterator[Path]:
+        # if the input file was compressed, the "path" you receive here will be decompressed
+
+        # a temporary file we write 'normalised' data to, that can be easily diffed/compared
+        normalised = unique_file_in_tempdir(input_filepath=path, dir=self.tmp_dir)
+
+        # some custom code here per-module that writes to 'normalised'
+
+        yield normalised
+
+
+# this script should be run as a module like
+# python3 -m bleanser.modules.smscalls --glob ...
+if __name__ == "__main__":
+    Normaliser.main()
+```
+
+This is **always** acting on the data loaded into memory/temporary files, it is not modifying the files itself. Once it determines an input file can be pruned, it will warn you by default, and you can specify `--move` or `--remove` with the CLI (see below) to remove it.
+
+There are particular normalisers for different filetypes, e.g. [`json`](./src/bleanser/core/modules/json.py), [`xml`](./src/bleanser/core/modules/xml.py), [`sqlite`](./src/bleanser/core/modules/sqlite.py) which might work if your data is especially basic, but typically this requires subclassing one of those and writing some custom code to 'cleanup' the data, so it can be properly compared/diffed.
+
+### normalise
+
+There are two ways you can think about `normalise` (creating a 'cleaned'/normalised representation of an input file) -- by specifying an 'upper' or 'lower' bound:
+
+- upper: specify which data you want to drop, dumping everything else to `normalised`
+- lower: specify which keys/data you want to keep, e.g. only returning a few keys which uniquely identify events in the data
+
+As an example say you had a JSON export:
+
+```json
+[
+  { "id": 5, "images": [{}], "href": "..." },
+  { "id": 6, "images": [{}], "href": "..." },
+  { "id": 7, "images": [{}], "href": "..." }
+]
+```
+
+When comparing this, you could possibly:
+
+1. Just write the `id` to the file. This is somewhat risky as you don't know if the `href` will always remain the same, so you may be losing data
+2. Write the `id` and the `href`, by specifying those two keys you're interested in
+3. Write the `id` and the `href`, by deleting the `images` key (this is different from 2!)
+
+There is a trade-off to be made here. For especially noisy exports with lots of metadata that might change over time that you're not interested in, number 3 means every couple months you might have to check and add new keys to delete (as an example see [spotify](./src/bleanser/modules/spotify.py)). This could be seen as a positive as well, as it means when the schema for the API/data changes underneath you, you may notice it quicker
+
+With option 2, you are more likely to remove redundant data files if additional metadata fields are added, and if you only really care about the `id` and `href` and you don't think the export format will change often, this is fine.
+
+Option 3. is generally the safest, but most verbose/tedious, it makes sure you're not removing files that may possibly contain new fields you want to preserve/parse.
+
+Ideally you meet somewhere in the middle, it depends a lot on the specific export data you're comparing.
+
+As it can be a bit difficult to follow, generally this is doing something like:
+
+- Decompress file if its a known compressed format into a `cleaned` file (`unpacked` in [`BaseNormaliser`](./src/bleanser/core/processor.py)), see [`kompress`](https://github.com/karlicoss/kompress/) for supported compression formats
+- Creating a temporary file to write data to (`unique_file_in_tempdir` in [`BaseNormaliser`](./src/bleanser/core/processor.py))
+- Parse the `cleaned` file into python objects (`JsonNormaliser`, `XmlNormaliser`, or something custom)
+- Let the user `cleanup` the data to remove noisy keys/data (specific modules, e.g. [spotify](./src/bleanser/modules/spotify.py))
+- Diff those against each other to find and/or remove files which dont contribute new data (module agnostic, run in `main`)
+
+### Subclassing
+
+For example, the JSON normaliser calls a `cleanup` function before it starts processing the data. If you wanted to remove the `images` key like shown above, you could do so like:
+
+```python
+from bleanser.core.modules.json import JsonNormaliser, delkeys, Json
+
+
+class Normaliser(JsonNormaliser):
+    # here, j is a dict, each file that this gets passed from the CLI call
+    # below is pre-processed by the cleanup function
+    def cleanup(self, j: Json) -> Json:
+        delkeys(j, keys={
+            'images',
+        })
+
+        return j
+
+
+if __name__ == '__main__':
+    Normaliser.main()
+```
+
+For common formats, the helper classes handle all the tedious bits like loading/parsing data, managing the temporary files. The `Normaliser.main` calls the CLI, which looks like this:
+
+```
+ $ python3 -m bleanser.core.modules.json prune --help
+Usage: python -m bleanser.core.modules.json prune [OPTIONS] PATH
+
+Options:
+  --glob                 Treat the path as glob (in the glob.glob sense)
+  --sort-by [size|name]  how to sort input files  [default: name]
+  --dry                  Do not prune the input files, just print what would happen after pruning.
+  --remove               Prune the input files by REMOVING them (be careful!)
+  --move PATH            Prune the input files by MOVING them to the specified path. A bit safer than --remove mode.
+  --yes                  Do not prompt before pruning files (useful for cron etc)
+  --threads INTEGER      Number of threads (processes) to use. Without the flag won't use any, with the flag will try
+                         using all available, can also take a specific value. Passed down to PoolExecutor.
+  --from INTEGER
+  --to INTEGER
+  --multiway             force "multiway" cleanup
+  --prune-dominated
+  --help                 Show this message and exit.
+```
+
+You'd provide input paths/globs to this file, and possibly `--remove` or `--move /tmp/removed` to remove/move files
+
+If you're not able to subclass one of the those, you might be able to subclass [extract](./src/bleanser/core/modules/extract.py), which lets you just yield any sort of string-afiable data, which is then used to diff/compare the input files. For example, if you only wanted to return the `id` and `href` in the JSON example above, you could just return a tuple:
+
+```python
+import json
+from pathlib import Path
+from typing import Iterator, Any
+
+from bleanser.core.modules.extract import ExtractObjectsNormaliser
+
+
+class Normaliser(ExtractObjectsNormaliser):
+    def extract_objects(self, path: Path) -> Iterator[Any]:
+        data = json.loads(path.read_text())
+        for blob in data:
+            yield (blob["id"], blob["href"])
+
+
+if __name__ == "__main__":
+    Normaliser.main()
+```
+
+Otherwise if you have some complex data source you need to handle yourself, you can override `do_normalise` and `unpacked` (how the data gets uncompressed/pre-processed) methods yourself, see handling the [discord zip files](https://github.com/purarue/bleanser/blob/master/src/bleanser_pura/modules/discord.py) as an example.