localcosmos-app-kit 0.9.17__py3-none-any.whl → 0.9.18__py3-none-any.whl

app_kit/taxonomy/sources/custom/management/commands/import_custom_species_csv.py

@@ -0,0 +1,177 @@
+import csv
+import os
+from typing import Optional, List
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+
+from taxonomy.models import TaxonomyModelRouter
+
+
+class Command(BaseCommand):
+    help = (
+        'Importiert die Arten-/Taxa-CSV (|-getrennt). Erkennt Headerzeile automatisch und überspringt sie.\n'
+        'Unterstütztes Layout (aktuell):\n'
+        '  0: scientific_name | 1: rank | 2: parent_lat | 3: parent_code | 4: de | 5: en | 6: nl | 7: da'
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument('csv_path', type=str, help='Pfad zur Arten-CSV (|-getrennt).')
+        parser.add_argument('--delimiter', type=str, default='|', help='Trennzeichen (Standard: |).')
+        parser.add_argument('--encoding', type=str, default='utf-8', help='Datei-Kodierung (Standard: utf-8).')
+        parser.add_argument('--dry-run', action='store_true', help='Nur prüfen, am Ende zurückrollen.')
+
+    def handle(self, *args, **options):
+        csv_path = options['csv_path']
+        delimiter = options['delimiter'] or '|'
+        encoding = options['encoding']
+        dry_run = options['dry_run']
+
+        if not os.path.exists(csv_path):
+            raise CommandError(f'Datei nicht gefunden: {csv_path}')
+
+        models = TaxonomyModelRouter('taxonomy.sources.custom')
+        self.stdout.write(self.style.NOTICE('Quelle: taxonomy.sources.custom'))
+        self.stdout.write(self.style.NOTICE(f'Trennzeichen: {repr(delimiter)}'))
+
+        created = 0
+        skipped = 0
+        existing = 0
+        rows = 0
+        self._missing_parents = {}
+        self._header_skipped = False
+        self._existing_names = []
+
+        with open(csv_path, 'r', encoding=encoding, newline='') as f:
+            reader = csv.reader(f, delimiter=delimiter)
+
+            with transaction.atomic():
+                for parts in reader:
+                    rows += 1
+                    if not parts:
+                        continue
+                    # Skip header line if present
+                    if not self._header_skipped and self._looks_like_header(parts):
+                        self._header_skipped = True
+                        continue
+                    c, e, s, existing_name = self._import_species_row(models, parts)
+                    created += c
+                    existing += e
+                    skipped += s
+                    if e and existing_name:
+                        # preserve insertion order, avoid duplicates
+                        if existing_name not in self._existing_names:
+                            self._existing_names.append(existing_name)
+
+                if dry_run:
+                    # Rollback alle DB-Änderungen, aber dennoch Auswertung anzeigen
+                    transaction.set_rollback(True)
+
+        self.stdout.write(self.style.SUCCESS(
+            f'Fertig. Erstellt: {created}, vorhanden: {existing}, übersprungen: {skipped}, Zeilen: {rows}'))
+        if dry_run:
+            self.stdout.write(self.style.WARNING('Dry-Run: Alle Änderungen wurden zurückgerollt.'))
+
+        if existing and self._existing_names:
+            self.stdout.write(self.style.NOTICE('Bereits vorhandene Taxa (exakt):'))
+            for nm in self._existing_names:
+                self.stdout.write(f'- {nm}')
+
+        if skipped and self._missing_parents:
+            self.stdout.write(self.style.WARNING('Übersprungene Zeilen wegen fehlendem Eltern-Taxon (Auszug):'))
+            # Zeige bis zu 25 fehlende Eltern mit Häufigkeit
+            shown = 0
+            for parent_name, count in sorted(self._missing_parents.items(), key=lambda x: -x[1]):
+                self.stdout.write(f'- {parent_name or "<leer>"}: {count}')
+                shown += 1
+                if shown >= 25:
+                    break
+
+    def _looks_like_header(self, parts: List[str]) -> bool:
+        if not parts:
+            return False
+        p0 = (parts[0] or '').strip().lower()
+        p1 = (parts[1] or '').strip().lower() if len(parts) > 1 else ''
+        # Typical header tokens
+        header_tokens = {'scientific name', 'name', 'latname', 'parent taxon', 'rank', 'de', 'en', 'nl', 'dk'}
+        return (p0 in header_tokens) or (p1 in header_tokens)
+
+    def _get_parent(self, models, parent_latname: str):
+        if not parent_latname:
+            return None
+        # exakter Treffer zuerst
+        parent = models.TaxonTreeModel.objects.filter(taxon_latname=parent_latname).first()
+        if parent:
+            return parent
+        # Fallback: case-insensitive Vergleich
+        return models.TaxonTreeModel.objects.filter(taxon_latname__iexact=parent_latname).first()
+
+    def _get_or_create_taxon(self, models, parent, latname: str, rank: str):
+        if not latname:
+            return None, False
+
+        r = (rank or 'species').strip().lower()
+        qs = models.TaxonTreeModel.objects.filter(taxon_latname=latname, rank=r)
+        if parent is None:
+            qs = qs.filter(is_root_taxon=True)
+        else:
+            qs = qs.filter(parent=parent)
+
+        instance = qs.first()
+        created = False
+        if not instance:
+            extra = {'rank': r}
+            if parent is None:
+                extra['is_root_taxon'] = True
+            else:
+                extra['parent'] = parent
+            instance = models.TaxonTreeModel.objects.create(latname, None, **extra)
+            created = True
+        return instance, created
+
+    def _add_locale_if_present(self, models, taxon, name: Optional[str], language: str):
+        if not taxon or not name:
+            return False
+        exists = models.TaxonLocaleModel.objects.filter(taxon=taxon, name=name, language=language).exists()
+        if not exists:
+            models.TaxonLocaleModel.objects.create(taxon, name, language, preferred=True)
+            return True
+        return False
+
+    def _import_species_row(self, models, parts: List[str]):
+        # Unterstütztes Layout (aktuell):
+        # 0: scientific_name | 1: rank | 2: parent_lat | 3: parent_code | 4: de | 5: en | 6: nl | 7: da
+
+        # Normalize parts
+        parts = [(p or '').strip() for p in parts]
+
+        # Minimalprüfung: mindestens 4 Spalten (bis parent_code) erwartet
+        if len(parts) < 4:
+            return 0, 0, 1, None
+
+        name = parts[0]
+        rank = (parts[1] or 'species').strip().lower()
+        parent_latname = parts[2]
+        # parent_code aktuell informativ, optional verwendbar
+        # parent_code = parts[3]
+        locale_start_idx = 4
+
+        parent = self._get_parent(models, parent_latname) if parent_latname else None
+        # If no parent and this should be a root-level taxon, allow creation with no parent
+        if not parent and parent_latname:
+            key = parent_latname
+            self._missing_parents[key] = self._missing_parents.get(key, 0) + 1
+            return 0, 0, 1, None
+
+        taxon, created = self._get_or_create_taxon(models, parent, name, rank)
+        if not taxon:
+            return 0, 0, 1, None
+
+        # Attach vernacular locales if present (de, en, nl, da)
+        locale_langs = ['de', 'en', 'nl', 'da']
+        for offset, lang in enumerate(locale_langs):
+            idx = locale_start_idx + offset
+            val = parts[idx] if idx < len(parts) else ''
+            self._add_locale_if_present(models, taxon, val, lang)
+
+        return (1, 0, 0, None) if created else (0, 1, 0, name)

app_kit/taxonomy/sources/custom/management/commands/import_custom_taxonomy_csv.py

@@ -0,0 +1,177 @@
+# temporary script for a one-time import of BeachExplorer's higher taxonom
+import csv
+import os
+import re
+from typing import List, Optional
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+
+from taxonomy.models import TaxonomyModelRouter
+
+RANK_SET = set([
+    'domain','kingdom','phylum','division','class','order','family','genus','species',
+    'subkingdom','infrakingdom','superphylum','subphylum','infraphylum',
+    'superclass','subclass','infraclass','superorder','suborder',
+    'infraorder','parvorder','superfamily','subfamily','tribe','subtribe',
+    'clade'
+])
+
+
+def is_code_like(val: str) -> bool:
+    return bool(re.fullmatch(r'[a-z]{2,6}', val))
+
+def _non_empty_indices(parts: List[str]) -> List[int]:
+    return [i for i, p in enumerate(parts) if p]
+
+
+class Command(BaseCommand):
+    help = 'Import a custom taxonomy tree from a GROK-Rang CSV (headerlos, |-getrennt).'
+
+    def add_arguments(self, parser):
+        parser.add_argument('csv_path', type=str, help='Pfad zur GROK-Rang CSV-Datei (ohne Header).')
+        parser.add_argument('--language', type=str, default=None,
+                            help='Sprachcode für Bezeichnungen aus dem Pfad (z. B. de, en).')
+        parser.add_argument('--delimiter', type=str, default='|',
+                            help='Trennzeichen (Standard: |).')
+        parser.add_argument('--encoding', type=str, default='utf-8',
+                            help='Datei-Kodierung (Standard: utf-8).')
+        parser.add_argument('--dry-run', action='store_true', help='Nur prüfen, am Ende zurückrollen.')
+        parser.add_argument('--report-limit', type=int, default=10,
+                            help='Maximale Anzahl Beispiel-Einträge in der Dry-Run-Übersicht (Standard: 10).')
+
+    def handle(self, *args, **options):
+        csv_path = options['csv_path']
+        language = options['language']
+        delimiter = options['delimiter'] or '|'
+        encoding = options['encoding']
+        dry_run = options['dry_run']
+
+        if not os.path.exists(csv_path):
+            raise CommandError(f'Datei nicht gefunden: {csv_path}')
+
+        models = TaxonomyModelRouter('taxonomy.sources.custom')
+        self.stdout.write(self.style.NOTICE('Quelle: taxonomy.sources.custom'))
+        self.stdout.write(self.style.NOTICE(f'Trennzeichen: {repr(delimiter)}'))
+
+        # track last Latin parent per depth to build a Latin-only hierarchy
+        self.depth_taxon = {}
+
+        with open(csv_path, 'r', encoding=encoding, newline='') as f:
+            reader = csv.reader(f, delimiter=delimiter)
+            created_count = 0
+            existing_count = 0
+            skipped_count = 0
+            skip_reasons = {}
+            skipped_examples = []
+            row_index = 0
+
+            try:
+                with transaction.atomic():
+                    for parts in reader:
+                        row_index += 1
+                        if not parts:
+                            # Treat truly empty lines as skipped for reporting
+                            skipped_count += 1
+                            skip_reasons['empty line'] = skip_reasons.get('empty line', 0) + 1
+                            if len(skipped_examples) < (options.get('report_limit') or 10):
+                                skipped_examples.append({'row': row_index, 'reason': 'empty line', 'raw': []})
+                            continue
+                        c, e, s, reason = self._import_grok_row(models, parts, language)
+                        created_count += c
+                        existing_count += e
+                        if s:
+                            skipped_count += s
+                            if reason:
+                                skip_reasons[reason] = skip_reasons.get(reason, 0) + 1
+                                if len(skipped_examples) < (options.get('report_limit') or 10):
+                                    skipped_examples.append({'row': row_index, 'reason': reason, 'raw': parts})
+
+                    if dry_run:
+                        # Print a detailed dry-run report before rolling back
+                        self.stdout.write(self.style.WARNING('Dry-Run Bericht:'))
+                        self.stdout.write(f'  Zeilen gelesen: {row_index}')
+                        self.stdout.write(f'  Würde erstellen: {created_count}')
+                        self.stdout.write(f'  Bereits vorhanden: {existing_count}')
+                        self.stdout.write(f'  Übersprungen: {skipped_count}')
+                        if skip_reasons:
+                            self.stdout.write('  Gründe für Überspringen:')
+                            for r, cnt in sorted(skip_reasons.items(), key=lambda x: (-x[1], x[0])):
+                                self.stdout.write(f'    - {r}: {cnt}')
+                        if skipped_examples:
+                            self.stdout.write('  Beispiele übersprungener Einträge:')
+                            for ex in skipped_examples:
+                                preview = '|'.join([str(p) for p in ex.get('raw', [])])
+                                self.stdout.write(f"    - Zeile {ex['row']}: {ex['reason']} -> {preview}")
+                        # Trigger rollback
+                        raise CommandError('Dry-Run abgeschlossen: Änderungen werden zurückgerollt.')
+
+            except CommandError as e:
+                if 'Dry-Run abgeschlossen' in str(e):
+                    self.stdout.write(self.style.WARNING(str(e)))
+                    return
+                raise
+
+        self.stdout.write(self.style.SUCCESS(
+            f'Fertig. Erstellt: {created_count}, vorhanden: {existing_count}, Zeilen: {row_index}'))
+
+    def _get_or_create(self, models, parent, latname, author, rank):
+        existing_qs = models.TaxonTreeModel.objects.filter(taxon_latname=latname, rank=rank)
+        if parent is None:
+            existing_qs = existing_qs.filter(is_root_taxon=True)
+        else:
+            existing_qs = existing_qs.filter(parent=parent)
+
+        instance = existing_qs.first()
+        created = False
+        if not instance:
+            extra = {'rank': rank}
+            if parent is None:
+                extra['is_root_taxon'] = True
+            else:
+                extra['parent'] = parent
+            instance = models.TaxonTreeModel.objects.create(latname, None, **extra)
+            created = True
+        return instance, created
+
+    def _clear_deeper_depth(self, depth: int):
+        # drop any cached parents deeper than current depth
+        for d in list(self.depth_taxon.keys()):
+            if d > depth:
+                del self.depth_taxon[d]
+
+    def _import_grok_row(self, models, parts: List[str], language: Optional[str]):
+        # Normalize tokens but keep positional layout
+        parts = [(p or '').strip() for p in parts]
+        if not parts or len(parts) < 3:
+            return 0, 0, 1, 'empty or too few columns'
+
+        # Fixed GROK layout: [...path..., latin, code, rank]
+        rank = parts[-1]
+        if not rank:
+            return 0, 0, 1, 'missing rank'
+        latin = parts[-3] if len(parts) >= 3 else ''
+        if not latin:
+            return 0, 0, 1, 'missing latin name'
+
+        # Path segments are all tokens before latin
+        path_segments = [p for p in parts[:-3] if p]
+        depth = len(path_segments)
+        parent = self.depth_taxon.get(depth - 1) if depth > 0 else None
+
+        taxon, created = self._get_or_create(models, parent, latin, author=None, rank=rank)
+
+        # Store this taxon as the current parent at its depth and clear deeper cache
+        self.depth_taxon[depth] = taxon
+        self._clear_deeper_depth(depth)
+
+        # Locale: letzter Pfadteil als Bezeichnung (attach vernacular to Latin taxon)
+        if path_segments and language:
+            vernacular = path_segments[-1]
+            exists = models.TaxonLocaleModel.objects.filter(
+                taxon=taxon, name=vernacular, language=language
+            ).exists()
+            if not exists:
+                models.TaxonLocaleModel.objects.create(taxon, vernacular, language, preferred=True)
+
+        return (1, 0, 0, None) if created else (0, 1, 0, None)

{localcosmos_app_kit-0.9.17.dist-info → localcosmos_app_kit-0.9.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: localcosmos_app_kit
-Version: 0.9.17
+Version: 0.9.18
 Summary: LocalCosmos App Kit. Web Portal to build Android and iOS apps
 Home-page: https://github.com/localcosmos/app-kit
 Author: Thomas Uher

{localcosmos_app_kit-0.9.17.dist-info → localcosmos_app_kit-0.9.18.dist-info}/RECORD

@@ -1797,6 +1797,10 @@ app_kit/taxonomy/sources/custom/models.py,sha256=4ExGyy2l_mYnGlsBhLndS7lQL5QSFGb
 app_kit/taxonomy/sources/custom/tests.py,sha256=mrbGGRNg5jwbTJtWWa7zSKdDyeB4vmgZCRc2nk6VY-g,60
 app_kit/taxonomy/sources/custom/urls.py,sha256=0P71XKmzS7mLeYIMcn0EE6Vi0pRV1fu-l_d317Kpnug,1032
 app_kit/taxonomy/sources/custom/views.py,sha256=ZFkqba9mGgKdbaZx4GNGI0hKoviyaEOnjf4CGvTTr8E,9780
+app_kit/taxonomy/sources/custom/management/commands/import_custom_species_csv.py,sha256=Ng3-FKfJpxXijxb8EzTEKXgjM8ivJiM92FAIG4ngrug,7434
+app_kit/taxonomy/sources/custom/management/commands/import_custom_taxonomy_csv.py,sha256=Y3QHClrZdvz5Fz-UgyTYhxtcCc6UxBa4BXpyL47PN54,8202
+app_kit/taxonomy/sources/custom/management/commands/__pycache__/import_custom_species_csv.cpython-313.pyc,sha256=AxUh7xot7wo__NBggEI9PEqH9lDX0GHBQ9ZkmTwLjNs,9725
+app_kit/taxonomy/sources/custom/management/commands/__pycache__/import_custom_taxonomy_csv.cpython-313.pyc,sha256=nU0sag8bvHUMSb6KBbs7EF8sZbdp5zwDdSiERW_PAQg,10567
 app_kit/taxonomy/sources/custom/migrations/0001_initial.py,sha256=FLCqXqGQcG0-cMeyxgSt-BuE-5uNq7HfCvYXBDni4wc,4246
 app_kit/taxonomy/sources/custom/migrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 app_kit/taxonomy/sources/custom/templates/custom_taxonomy/manage_custom_taxon.html,sha256=SSXwvvZflxkKAK1GrkZM15ZD6JUfjEWDmiI78TGhmKg,1348
@@ -1935,8 +1939,8 @@ app_kit/tests/__pycache__/test_models.cpython-313.pyc,sha256=Lmv3BfjLs5Fg-olJeMl
 app_kit/tests/__pycache__/test_utils.cpython-313.pyc,sha256=GX3REqZygi2eO_A2F2_KtYi7hg54X5QPtCTCGWuxGpM,14054
 app_kit/tests/__pycache__/test_views.cpython-311.pyc,sha256=NDJR40TcMm-bXXC-wV7OgH1sGR3N7psSWYiUirkkrjU,133242
 app_kit/tests/__pycache__/test_views.cpython-313.pyc,sha256=q851UqIZFCCTfQb1lF4SVxV1j_Vu1hJdOlpckmrGX28,125363
-localcosmos_app_kit-0.9.17.dist-info/licenses/LICENCE,sha256=VnxALPSxXoU59rlNeRdJtwS_nU79IFpVWsZZCQUM4Mw,1086
-localcosmos_app_kit-0.9.17.dist-info/METADATA,sha256=e8ksAKCmQ153iA3vmv2B_1ILF-cHvCiN4JbKU19a1XA,1388
-localcosmos_app_kit-0.9.17.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-localcosmos_app_kit-0.9.17.dist-info/top_level.txt,sha256=F6H4pEBkCvUR_iwQHIy4K1iby-jzfWg3CTym5XJKeys,8
-localcosmos_app_kit-0.9.17.dist-info/RECORD,,
+localcosmos_app_kit-0.9.18.dist-info/licenses/LICENCE,sha256=VnxALPSxXoU59rlNeRdJtwS_nU79IFpVWsZZCQUM4Mw,1086
+localcosmos_app_kit-0.9.18.dist-info/METADATA,sha256=wC25Ke7ekb_Rf0l-tcIN2Ih3__GSe64Z0SNNQwGQVSg,1388
+localcosmos_app_kit-0.9.18.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+localcosmos_app_kit-0.9.18.dist-info/top_level.txt,sha256=F6H4pEBkCvUR_iwQHIy4K1iby-jzfWg3CTym5XJKeys,8
+localcosmos_app_kit-0.9.18.dist-info/RECORD,,