django-grouper 0.1.0__tar.gz

django_grouper-0.1.0/PKG-INFO

@@ -0,0 +1,32 @@
+Metadata-Version: 2.1
+Name: django-grouper
+Version: 0.1.0
+Summary: 
+Author: Birger Schacht
+Requires-Python: >=3.10,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: django (>=3)
+Requires-Dist: pandas (>=2.2.2,<3.0.0)
+Requires-Dist: scikit-learn (>=1.5.0,<2.0.0)
+Requires-Dist: sparse-dot-topn (>=1.1.1,<2.0.0)
+Description-Content-Type: text/markdown
+
+Group Django model instances on similar fields
+
+Based on [this tutorial](https://towardsdatascience.com/group-thousands-of-similar-spreadsheet-text-cells-in-seconds-2493b3ce6d8d)
+
+# Installation
+
+1. add `django_grouper` to your `INSTALLED_APPS`
+2. add `path("", include("django_grouper.urls"))` to your `urlpatterns`
+3. go to `/grouper` and add the param `group_content_type` to specify which ContentType to group and the params `group_fields` to specify which fields to group by
+
+# Configuration
+
+If you define a `GROUP_FILTER` in you settings, the queryset in the `/grouper` view will be passed through this filter. The filter is expected to
+be a callable and will receive the queryset and the request as parameters. This means you can pass the queryset together with the request to a
+django-filter filter.
+

django_grouper-0.1.0/README.md

@@ -0,0 +1,15 @@
+Group Django model instances on similar fields
+
+Based on [this tutorial](https://towardsdatascience.com/group-thousands-of-similar-spreadsheet-text-cells-in-seconds-2493b3ce6d8d)
+
+# Installation
+
+1. add `django_grouper` to your `INSTALLED_APPS`
+2. add `path("", include("django_grouper.urls"))` to your `urlpatterns`
+3. go to `/grouper` and add the param `group_content_type` to specify which ContentType to group and the params `group_fields` to specify which fields to group by
+
+# Configuration
+
+If you define a `GROUP_FILTER` in you settings, the queryset in the `/grouper` view will be passed through this filter. The filter is expected to
+be a callable and will receive the queryset and the request as parameters. This means you can pass the queryset together with the request to a
+django-filter filter.

django_grouper-0.1.0/django_grouper/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class DjangoGrouperConfig(AppConfig):
+    default_auto_field = "django.db.models.AutoField"
+    name = "django_grouper"

django_grouper-0.1.0/django_grouper/templates/django_grouper/group.html

@@ -0,0 +1,52 @@
+{% extends "webpage/base.html" %}
+{% block content %}
+    {% if merged_ids %}
+        {% for merge_id in merged_ids %}Merged {{ merge_id }}{% endfor %}
+    {% else %}
+        <style>
+.checkbox:checked ~ .card{
+  background: #ffc107;
+}
+.checkbox {
+    position: absolute;
+    top: 0;
+    left: 0;
+    opacity: 0;
+    width: 100%;
+    height: 100%;
+    z-index: 999;
+    cursor: pointer;
+}
+.card_area{
+  position: relative;
+  margin-bottom: 30px;
+}
+        </style>
+        <div class="container">
+            <form class="mt-5" method="post">
+                {% csrf_token %}
+                <button type="submit" class="btn btn-primary">Merge selected</button>
+                <input type="hidden"
+                       name="content_type"
+                       value="{{ content_type.app_label }}.{{ content_type.model }}">
+                <div class="card-columns mt-5">
+                    {% for object in object_list %}
+                        <div class="card_area">
+                            <input class="checkbox"
+                                   type="checkbox"
+                                   id="to_merge"
+                                   name="to_merge"
+                                   value="{{ object.pk }}" />
+                            <div class="card border" style="width: 18rem;">
+                                <div class="card-body">
+                                    <h5 class="card-title">{{ object }}</h5>
+                                    <p class="card-text">Details</p>
+                                </div>
+                            </div>
+                        </div>
+                    {% endfor %}
+                </div>
+            </form>
+        </div>
+    {% endif %}
+{% endblock content %}

django_grouper-0.1.0/django_grouper/templates/django_grouper/grouper.html

@@ -0,0 +1,18 @@
+{% extends "webpage/base.html" %}
+{% block content %}
+    <div class="container mt-3">
+        <ul>
+            {% for group, ids in groups.items %}
+                <li>
+                    <a href="{% url "group" %}?group_content_type={{ content_type.app_label }}.{{ content_type.model }}{% for id in ids %}&ids={{ id }}{% endfor %}">{{ group }}</a>:
+                    <details style="display: inline;">
+                        <summary>{{ ids|length }}</summary>
+                        {% for id in ids %}
+                            <a href="{% url "apis_core:apis_entities:generic_entities_detail_view" content_type.model id %}">{{ id }}</a>,
+                        {% endfor %}
+                    </details>
+                </li>
+            {% endfor %}
+        </ul>
+    </div>
+{% endblock content %}

django_grouper-0.1.0/django_grouper/urls.py

@@ -0,0 +1,8 @@
+from django.urls import path
+
+from . import views
+
+urlpatterns = [
+    path("grouper/", views.Grouper.as_view(), name="grouper"),
+    path("group/", views.Group.as_view(), name="group"),
+]

django_grouper-0.1.0/django_grouper/utils.py

@@ -0,0 +1,79 @@
+import re
+import pandas as pd
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sparse_dot_topn import awesome_cossim_topn
+
+
+def group_queryset(queryset, fields=[]) -> dict:
+    # Instaniate our lookup hash table
+    group_lookup = {}
+
+    # Write a function for cleaning strings and returning an array of ngrams
+    def ngrams_analyzer(string):
+        string = re.sub(r"[,-./]", r"", string)
+        ngrams = zip(*[string[i:] for i in range(5)])  # N-Gram length is 5
+        return ["".join(ngram) for ngram in ngrams]
+
+    def find_group(row, col):
+        # If either the row or the col string have already been given
+        # a group, return that group. Otherwise return none
+        if row in group_lookup:
+            return group_lookup[row]
+        elif col in group_lookup:
+            return group_lookup[col]
+        else:
+            return None
+
+    def add_vals_to_lookup(group, row, col):
+        # Once we know the group name, set it as the value
+        # for both strings in the group_lookup
+        group_lookup[row] = group
+        group_lookup[col] = group
+
+    def add_pair_to_lookup(row, col):
+        # in this function we'll add both the row and the col to the lookup
+        group = find_group(row, col)  # first, see if one has already been added
+        if group is not None:
+            # if we already know the group, make sure both row and col are in lookup
+            add_vals_to_lookup(group, row, col)
+        else:
+            # if we get here, we need to add a new group.
+            # The name is arbitrary, so just make it the row
+            add_vals_to_lookup(row, row, col)
+
+    # Construct your vectorizer for building the TF-IDF matrix
+    vectorizer = TfidfVectorizer(analyzer=ngrams_analyzer)
+
+    if fields:
+        allfields = fields + ["pk"]
+        values = queryset.values_list(*allfields)
+        df = pd.DataFrame(list(values), columns=allfields)
+
+        df["grouper"] = df[fields.pop(0)].astype(str).str.cat(df[fields].astype(str))
+
+        # Grab the column you'd like to group, filter out duplicate values
+        # and make sure the values are Unicode
+        vals = df["grouper"].unique().astype("U")
+
+        # Build the matrix!!!
+        tfidf_matrix = vectorizer.fit_transform(vals)
+
+        cosine_matrix = awesome_cossim_topn(
+            tfidf_matrix, tfidf_matrix.transpose(), vals.size, 0.8
+        )
+
+        # Build a coordinate matrix
+        coo_matrix = cosine_matrix.tocoo()
+
+        # for each row and column in coo_matrix
+        # if they're not the same string add them to the group lookup
+        for row, col in zip(coo_matrix.row, coo_matrix.col):
+            if row != col:
+                add_pair_to_lookup(vals[row], vals[col])
+
+        df["Group"] = df["grouper"].map(group_lookup).fillna(df["grouper"])
+
+        d = df.groupby("Group")["pk"].apply(list).to_dict()
+        ret = {key: value for key, value in d.items() if len(value) > 1}
+        return dict(sorted(ret.items(), key=lambda item: len(item[1]), reverse=True))
+    return {}

django_grouper-0.1.0/django_grouper/views.py

@@ -0,0 +1,63 @@
+from django.conf import settings
+from django.contrib.contenttypes.models import ContentType
+from django.shortcuts import get_object_or_404
+from django.views.generic.base import TemplateView
+from django.http import HttpResponseRedirect
+
+
+from .utils import group_queryset
+
+
+class BaseView(TemplateView):
+    def dispatch(self, request, *args, **kwargs):
+        app_label, model = request.GET.get("group_content_type", ".").split(".")
+        self.django_content_type = get_object_or_404(
+            ContentType, app_label=app_label, model=model
+        )
+        return super().dispatch(request, *args, **kwargs)
+
+
+class Grouper(BaseView):
+    template_name = "django_grouper/grouper.html"
+
+    def get_context_data(self, *args, **kwargs):
+        fields = self.request.GET.getlist("group_fields", [])
+
+        ctx = super().get_context_data(*args, **kwargs)
+        ctx["content_type"] = self.django_content_type
+        ctx["groups"] = group_queryset(self.get_queryset(), fields)
+        return ctx
+
+    def get_queryset(self):
+        qs = self.django_content_type.model_class().objects.all()
+        if hasattr(settings, "GROUP_FILTER"):
+            qs = settings.GROUP_FILTER(qs, self.request)
+        return qs
+
+
+class Group(BaseView):
+    template_name = "django_grouper/group.html"
+
+    def get_context_data(self, *args, **kwargs):
+        ids = self.request.GET.getlist("ids", [])
+
+        ctx = super().get_context_data(*args, **kwargs)
+        ctx["content_type"] = self.django_content_type
+        ctx["title"] = self.request.GET.get("group_title")
+        ctx["object_list"] = self.django_content_type.model_class().objects.filter(
+            pk__in=ids
+        )
+        return ctx
+
+    def post(self, request, *args, **kwargs):
+        ctx = self.get_context_data()
+        ctx["merged_ids"] = []
+        newinstance = self.django_content_type.model_class().objects.create()
+        for merge_id in request.POST.getlist("to_merge"):
+            mergeobject = get_object_or_404(
+                self.django_content_type.model_class(), pk=merge_id
+            )
+            mergeobject.grouped_into = newinstance
+            mergeobject.save()
+            ctx["merged_ids"].append(merge_id)
+        return HttpResponseRedirect(newinstance.get_absolute_url())

django_grouper-0.1.0/pyproject.toml

@@ -0,0 +1,21 @@
+[tool.poetry]
+name = "django-grouper"
+version = "0.1.0"
+description = ""
+authors = ["Birger Schacht"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+pandas = "^2.2.2"
+scikit-learn = "^1.5.0"
+sparse-dot-topn = "^1.1.1"
+django = ">=3"
+
+[tool.poetry.group.dev.dependencies]
+ruff = "^0.4.8"
+djlint = "^1.31.1"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"