aaa-ml-datasets-course 1.0.0__tar.gz

aaa_ml_datasets_course-1.0.0/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

aaa_ml_datasets_course-1.0.0/PKG-INFO

@@ -0,0 +1,30 @@
+Metadata-Version: 2.3
+Name: aaa-ml-datasets-course
+Version: 1.0.0
+Summary: Репозиторий курса по созданию датасетов Академии Аналитиков Авито
+Author: Avito
+Requires-Python: >=3.10,<3.13
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: dotenv (==0.9.9)
+Requires-Dist: ipykernel (==6.29.4)
+Requires-Dist: ipywidgets (==8.1.6)
+Requires-Dist: matplotlib (==3.8.4)
+Requires-Dist: nltk (==3.8.1)
+Requires-Dist: notebook (==7.4.1)
+Requires-Dist: numpy (==1.26.4)
+Requires-Dist: opencv-python (==4.9.0.80)
+Requires-Dist: pandas (==2.2.3)
+Requires-Dist: protobuf (==4.25.3)
+Requires-Dist: scikit-learn (==1.4.2)
+Requires-Dist: scipy (==1.13.1)
+Requires-Dist: seaborn (==0.13.2)
+Requires-Dist: tensorboard (==2.16.2)
+Requires-Dist: together (==1.4.6)
+Requires-Dist: toloka-kit (==1.2.1)
+Requires-Dist: torch (==2.2.2)
+Requires-Dist: torchvision (==0.17.2)
+Requires-Dist: tqdm (==4.67.1)
+Requires-Dist: transformers (==4.39.3)

aaa_ml_datasets_course-1.0.0/heuristics/active_learning.py

@@ -0,0 +1,111 @@
+from collections import Counter
+import numpy as np
+from typing import Tuple, List, Dict
+
+
+def get_num_samples_per_class(class_counts: Dict, sample_size: int) -> Dict[int, int]:
+    """
+    Принимает на вход словарь class_id -> class_count
+    Возвращает словарь class_id -> sample_size
+    так как не во всех классах есть хотя бы  sample_size // n_classes примеров,
+    то основной смысл функции сделать выборку равномернее и добрать
+    примеров из более многочисленных классов
+    Args:
+        class_counts: словарь с кол-во примеров в выборке по каждому
+                    классу {0:100, 1: 234, ..., 19: 33}
+        sample_size: итоговый размер семпла который хочется получить
+
+    Returns:
+
+    """
+    class_counts_rest = class_counts.copy()
+
+    n_classes = len(class_counts)
+    sample_size_per_target = sample_size // n_classes
+
+    class_to_sample_size = {}
+    current_sample_size = sample_size_per_target
+    ready_targets = []
+
+    while True:
+        values = [target for target, count in class_counts_rest.items() if
+                  count <= current_sample_size]
+        if len(values) > 0:
+            ready_targets += values
+            for value in values:
+                class_to_sample_size[value] = class_counts_rest.pop(value)
+            if n_classes == len(ready_targets):
+                break
+            current_sample_size = (sample_size - sum(class_to_sample_size.values())) // (
+                    n_classes - len(ready_targets)
+            )
+        else:
+            for target in class_counts_rest.keys():
+                class_to_sample_size[target] = current_sample_size
+                class_counts_rest[target] -= current_sample_size
+
+            break
+
+    s = sum(class_to_sample_size.values())
+    for target, count in class_counts_rest.items():
+        if s >= sample_size:
+            break
+        if count > 0:
+            class_to_sample_size[target] += 1
+            s += 1
+
+    return class_to_sample_size
+
+
+def max_min_with_diversity(probas: np.ndarray, sample_size: int) -> Tuple[List, Dict]:
+    """
+    Семплирование примеров, где самый вероятный класс имеет
+    наименьшую вероятность среди остальных примеров
+    Args:
+        probas: матрица NxC - вероятности по всем классам C -(кол-во классов)
+        sample_size: размер итогового семпла
+
+    Returns:
+
+    """
+    class_counts = Counter(probas.argmax(axis=1))
+    sample_size_per_target = get_num_samples_per_class(class_counts, sample_size)
+
+    indexes = []
+
+    for target, sample_size in sample_size_per_target.items():
+        target_indexes = np.argwhere(probas.argmax(axis=1) == target)[:, 0]
+        max_probas = probas[target_indexes, :].max(axis=1)
+
+        max_indexes = np.argsort(max_probas)[:sample_size]
+
+        indexes += list(target_indexes[max_indexes])
+
+    return indexes, sample_size_per_target
+
+
+def margin_with_diversity(probas: np.ndarray, sample_size: int) -> Tuple[List, Dict]:
+    """
+    Семплирование примеров где модель сомневается между 2 классами
+    Args:
+        probas: матрица NxC - вероятности по всем классам C -(кол-во классов)
+        sample_size: размер итогового семпла
+
+    Returns:
+
+    """
+    class_counts = Counter(probas.argmax(axis=1))
+    sample_size_per_target = get_num_samples_per_class(class_counts, sample_size)
+
+    indexes = []
+
+    for target, sample_size in sample_size_per_target.items():
+        target_indexes = np.argwhere(probas.argmax(axis=1) == target)[:, 0]
+        probas_sorted = np.sort(probas[target_indexes, :], axis=1)
+        margin = probas_sorted[:, -1] - probas_sorted[:, -2]
+
+        margin_indexes = np.argsort(margin)[:sample_size]
+
+        indexes += list(target_indexes[margin_indexes])
+
+    return indexes, sample_size_per_target

aaa_ml_datasets_course-1.0.0/heuristics/heuristics.py

@@ -0,0 +1,110 @@
+import numpy as np
+from typing import List, Tuple, Union
+import cv2
+import pandas as pd
+import re
+from PIL import Image
+
+
+def order_points(points: np.ndarray) -> np.ndarray:
+    """
+    Упорядочивает точки по часовой стрелке.
+
+    Args:
+        points: набор точек (n, 2)
+
+    Returns:
+        Упорядоченный набор точек (n, 2)
+
+    """
+    if len(points.shape) != 2 and points.shape[0] == 0 and points.shape[1] != 2:
+        raise ValueError('points: набор точек (n, 2)')
+
+    center = points.mean(axis=0)
+    points = np.array(
+        sorted(
+            points,
+            key=lambda point: np.arctan2(point[1] - center[1], point[0] - center[0]),
+        ),
+        dtype=np.float32,
+    )
+
+    points_sort_x = sorted(points, key=lambda point: point[0])[:2]
+    point_sort_x_y = sorted(points_sort_x, key=lambda point: point[1])[0]
+    ind = [
+        ind for ind, point in enumerate(points) \
+        if point[0] == point_sort_x_y[0] and point[1] == point_sort_x_y[1]
+    ][0]
+
+    points = np.concatenate([points[ind:], points[:ind]]).astype(np.float32)
+
+    return points
+
+
+def image_average_color(image: np.array) -> np.array:
+    average_color_row = np.average(image, axis=0)
+    average_color = np.average(average_color_row, axis=0)
+
+    return average_color
+
+
+def catalog_flag(
+        image: Union[np.array, str],
+        crop_coord: List[float],
+        fill_color: Tuple[int, int, int] = (255, 255, 255),
+        threshold: float = 247.0,
+):
+    if isinstance(image, str):
+        image = Image.open(image)
+        image = np.array(image)
+
+    points = order_points(np.array(crop_coord).reshape(-1, 2)) * np.array(
+        [image.shape[1], image.shape[0]]
+    )
+    result_image = cv2.rectangle(
+        image,
+        (int(points[0][0]), int(points[0][1])),
+        (int(points[1][0] + 0.5), int(points[1][1] + 0.5)),
+        fill_color,
+        -1,
+    )
+    average_color = image_average_color(result_image)
+
+    return all(average_color >= threshold)
+
+
+def find_text_patterns(
+    text: str, include_re: List[re.Pattern], exclude_re: List[re.Pattern] = None
+):
+    text = text.lower()
+    if not exclude_re:
+        return int(any([pattern.findall(text) for pattern in include_re]))
+    return int(
+        all(
+            [
+                any([pattern.findall(text) for pattern in include_re]),
+                not any([pattern.findall(text) for pattern in exclude_re]),
+            ]
+        )
+    )
+
+
+def calculate_text_heuristics(
+    title_desc_df: pd.DataFrame, include_re: List[re.Pattern], exclude_re: List[re.Pattern] = None
+):
+    title_desc_df['title_match'] = title_desc_df['title'].apply(
+        lambda x: find_text_patterns(
+            text=x,
+            include_re=include_re,
+            exclude_re=exclude_re,
+        )
+    )
+    title_desc_df['description_match'] = title_desc_df['description'].apply(
+        lambda x: find_text_patterns(
+            text=x,
+            include_re=include_re,
+            exclude_re=exclude_re,
+        )
+    )
+
+    return title_desc_df

aaa_ml_datasets_course-1.0.0/heuristics/model/classifier.py

@@ -0,0 +1,59 @@
+import os
+from typing import Dict, Optional
+
+import torch
+from torch import nn
+from torchvision.models import (
+    resnet18,
+    resnet34,
+    resnet101,
+)
+
+from .settings import NUM_CLASSES
+
+
+class BaseModel(nn.Module):
+    def __init__(self, num_classes=NUM_CLASSES):
+        super().__init__()
+        self.feature_extractor = None
+        self.clf = None
+        self.num_classes = num_classes
+
+    def get_model_saving_params(self):
+        model_params = {'num_classes': self.num_classes}
+        return model_params
+
+    def save_pretrained(self, model_path: str, meta_info: Optional[Dict] = None):
+        self.cpu()
+        meta_info = meta_info or {}
+        meta_info['model_state'] = self.feature_extractor.state_dict()
+        meta_info['model_params'] = self.get_model_saving_params()
+
+        os.makedirs(os.path.dirname(model_path), exist_ok=True)
+        torch.save(meta_info, model_path)
+
+    @classmethod
+    def from_pretrained(cls, model_path):
+        model_state = torch.load(model_path)
+        instance = cls(**model_state.pop('model_params'))
+        instance.feature_extractor.load_state_dict(model_state.pop('model_state'))
+
+        instance.eval()
+
+        return instance, model_state
+
+
+class RoomModel(BaseModel):
+    def __init__(self, num_classes=NUM_CLASSES):
+        super().__init__(num_classes)
+        self.feature_extractor = resnet18(pretrained=True)
+        # без послденего слоя
+        # self.feature_extractor = torch.nn.Sequential(*(list(feature_extractor.children())[:-1]))
+        self.feature_extractor.fc = nn.Linear(
+            self.feature_extractor.fc.in_features, out_features=num_classes
+        )
+
+    def forward(self, batch):
+        out = self.feature_extractor(batch)
+
+        return out

aaa_ml_datasets_course-1.0.0/heuristics/model/dataset.py

@@ -0,0 +1,81 @@
+import os
+from typing import Dict, Optional
+
+import pandas as pd
+from PIL import Image
+from torch.utils.data import Dataset
+from torchvision import transforms
+
+
+class TorchDataset(Dataset):
+    image_id_column: str = 'image_id_ext'
+    label_column: str = 'result'
+    img_path_column: str = 'img_path'
+    sample_weight_column: str = 'sample_weight'
+    img_extension: str = 'jpg'
+
+    def __init__(
+        self,
+        df: pd.DataFrame,
+        image_dir: str = None,
+        transformer: Optional[transforms.transforms.Compose] = None,
+        normalize_sample_weights: bool = True,
+        with_cache=True,
+    ):
+        super(TorchDataset, self).__init__()
+        self.image_dir = image_dir
+        self.df = df.reset_index(drop=True)
+
+        if normalize_sample_weights and (self.sample_weight_column in self.df.columns):
+            self.df[self.sample_weight_column] /= self.df[self.sample_weight_column].mean()
+        # self.df = self.df[self.df['result'] != -1]
+        self.transformer = transformer
+        self.image_cache = {}
+        self.with_cache = with_cache
+
+    def img_path_from_id(self, image_id):
+        path = os.path.join(self.image_dir, f'{image_id}.{self.img_extension}')
+
+        return path
+
+    def get_image_path(self, item: pd.Series):
+        if item.get(self.img_path_column) is None and item.get(self.image_id_column) is None:
+            raise ValueError('image_id or image_path should be specified')
+        image_path = item.get(self.img_path_column) or self.img_path_from_id(
+            item.get(self.image_id_column)
+        )
+
+        return image_path
+
+    def load_img(self, idx: int, item: pd.Series):
+        # img = self.img_cache.get(idx)
+        # if img is None:
+        path = self.get_image_path(item)
+        img = Image.open(path)
+            # self.img_cache[idx] = img
+
+        return img
+
+    def __getitem__(self, idx):
+        item = self.df.iloc[idx]
+        label = int(item.get(self.label_column, 0))
+        sample_weight = float(item.get(self.sample_weight_column, 1))
+
+        img = self.image_cache.get(idx)
+        if img is None:
+            img = self.load_img(idx, item)
+            if self.transformer is not None:
+                img = self.transformer(img)
+            if self.with_cache:
+                self.image_cache[idx] = img
+
+        # item = {
+        #     'img': img,
+        #     'label': item_series['result'],
+        #     'label_name': item_series['label']
+        # }
+
+        return img, label, sample_weight
+
+    def __len__(self):
+        return len(self.df)

aaa_ml_datasets_course-1.0.0/heuristics/model/metrics.py

@@ -0,0 +1,43 @@
+from collections import Counter
+import pandas as pd
+
+from sklearn.metrics import (
+    ConfusionMatrixDisplay,
+    accuracy_score,
+    classification_report,
+    confusion_matrix,
+    f1_score,
+)
+
+import matplotlib.pyplot as plt
+
+
+class Metrics:
+    def __init__(self, class_mapping):
+        self.class_mapping = class_mapping
+
+    def plot_cf_matrix(self, targets, predictions):
+        labels_ordered = [item[0] for item in (Counter(targets).most_common())]
+        cm = confusion_matrix(
+            targets, predictions, labels=labels_ordered, normalize='true'
+        )
+        cm_disp = ConfusionMatrixDisplay(
+            confusion_matrix=cm,
+            display_labels=[self.class_mapping.get(label, 'None') for label in labels_ordered],
+        )
+        fig, ax = plt.subplots(figsize=(15, 15))
+        cm_disp.plot(ax=ax, xticks_rotation='vertical')
+
+    def get_accuracies_df(self, targets, predictions):
+        df = pd.DataFrame(
+            classification_report(
+                targets,
+                predictions,
+                labels=list(self.class_mapping.keys()),
+                target_names=list(self.class_mapping.values()),
+                output_dict=True,
+                zero_division=0,
+            )
+        ).T.reset_index().sort_values('support', ascending=False)
+
+        return df

aaa_ml_datasets_course-1.0.0/heuristics/model/settings.py

@@ -0,0 +1,39 @@
+from collections import namedtuple
+
+MAX_IMG_SIZE = (1280, 960)
+MAX_IMG_SIZE_STR = '1280x960'
+NUM_CLASSES = 20
+IMAGES_DIR = '/data/images/'
+LABELS_PATH = '/app/data/labels.csv'
+
+
+ROOM_TYPE = namedtuple('ROOM_TYPE', ['name', 'value_id', 'exclude'])
+
+ROOM_TYPES = (
+    ROOM_TYPE('ошибка загрузки изображения', -1, True),
+    ROOM_TYPE('кухня / столовая', 0, False),
+    ROOM_TYPE('кухня-гостиная', 1, False),
+    ROOM_TYPE('универсальная комната', 2, False),
+    ROOM_TYPE('гостиная', 3, False),
+    ROOM_TYPE('спальня', 4, False),
+    ROOM_TYPE('кабинет', 5, False),
+    ROOM_TYPE('детская', 6, False),
+    ROOM_TYPE('ванная комната', 7, False),
+    ROOM_TYPE('туалет', 8, False),
+    ROOM_TYPE('совмещенный санузел', 9, False),
+    ROOM_TYPE('коридор / прихожая', 10, False),
+    ROOM_TYPE('гардеробная / кладовая / постирочная', 11, False),
+    ROOM_TYPE('балкон / лоджия', 12, False),
+    ROOM_TYPE('вид из окна / с балкона', 13, False),
+    ROOM_TYPE('дом снаружи / двор', 14, False),
+    ROOM_TYPE('подъезд / лестничная площадка', 15, False),
+    ROOM_TYPE('другое', 16, False),
+    ROOM_TYPE('предметы интерьера / быт.техника', 17, False),
+    ROOM_TYPE('не могу дать ответ / не ясно', 18, True),
+    ROOM_TYPE ('комната без мебели', 19, False),
+)
+
+
+VALID_ROOM_TYPES = tuple(filter(lambda x: not x.exclude, ROOM_TYPES))
+
+CLASS_NAME_MAPPING = {room_type.value_id: room_type.name for room_type in VALID_ROOM_TYPES}

aaa_ml_datasets_course-1.0.0/heuristics/model/trainer.py

@@ -0,0 +1,258 @@
+import logging
+import os
+from typing import (
+    Dict,
+    List,
+    Optional,
+    Tuple,
+)
+
+import numpy as np
+import pandas as pd
+import torch
+from sklearn.metrics import accuracy_score
+from torch.nn.functional import softmax as torch_softmax
+from torch.utils.data import DataLoader
+from torch.utils.tensorboard import SummaryWriter
+from tqdm import tqdm
+from tqdm.notebook import tqdm as tqdm_notebook
+
+from .classifier import RoomModel
+from .dataset import TorchDataset
+from .utils import get_preprocessor
+
+
+class TrainerUtils:
+    def __init__(
+        self,
+        device='cpu',
+        tensorboard_dir: Optional[str] = None,
+        experiment_tag: Optional[str] = None,
+        run_in_notebook=True,
+        model_checkpoint_path: str = '/app/data/models/',
+    ):
+        self.device: str = device
+        self.loss_fn = torch.nn.CrossEntropyLoss(reduction='none')
+        self._global_steps_num = 0
+        self.tensorboard: Optional[SummaryWriter] = None
+        if tensorboard_dir is not None:
+            if experiment_tag is not None:
+                tensorboard_dir = os.path.join(tensorboard_dir, experiment_tag)
+            self.tensorboard = SummaryWriter(tensorboard_dir)
+        self.run_in_notebook = run_in_notebook
+        self.best_score: float = 0
+        self.best_epoch: int = -1
+        self.accuracies: List[float] = []
+        self.checkpoint_path: str = os.path.join(model_checkpoint_path, f'model_{experiment_tag}')
+
+    def get_progress_bar(self, epoch_len):
+        if self.run_in_notebook:
+            pbar = tqdm_notebook(total=epoch_len)
+        else:
+            pbar = tqdm(total=epoch_len)
+
+        return pbar
+
+    def log_metrics_loss(self, loss, name, step):
+        self.tensorboard.add_scalar(f'Loss/{name}', loss, step)
+
+    def log_metrics_accuracy(self, accuracy, name, step):
+        self.tensorboard.add_scalar(f'Accuracy/{name}', accuracy, step)
+
+    def do_checkpoint(
+        self,
+        epoch: int,
+        score: float,
+        model,
+        accuracy_df: pd.DataFrame = None
+    ):
+        self.accuracies.append(score)
+        better_than_previous = score > self.best_score
+
+        if accuracy_df is not None:
+            f1_score = accuracy_df.loc[accuracy_df['index'] == 'macro avg']['f1-score'].values[0]
+            better_than_previous = f1_score > self.best_f1
+
+        if better_than_previous:
+            self.best_score = score
+
+            self.best_epoch = epoch
+            model_state = {
+                'epoch': epoch,
+                'score': score,
+            }
+            model.save_pretrained(self.checkpoint_path, model_state)
+
+    def forward_step(self, batch, model) -> Tuple[torch.Tensor, float, list, list, torch.Tensor]:
+        labels, sample_weights = batch[1], batch[2]
+        logits = model(batch[0])
+
+        loss_batch = self.loss_fn(logits, labels)
+        if sample_weights is not None:
+            loss_batch = loss_batch * sample_weights
+
+        loss = loss_batch.mean()
+
+        # loss = self.loss_fn(logits, labels)
+        logits = logits.detach().cpu()
+        labels_predicted = logits.argmax(dim=1).tolist()
+        labels = labels.detach().cpu().tolist()
+
+        accuracy = accuracy_score(labels, labels_predicted)
+
+        return loss, accuracy, labels_predicted, labels, logits
+
+    def training_step(
+        self, model, batch: List[torch.Tensor], optimizer, scheduler, grad_clipping_norm
+    ):
+        model.train()
+        batch[0] = batch[0].to(self.device)
+        batch[1] = batch[1].to(self.device)
+        batch[2] = batch[2].to(self.device)
+        optimizer.zero_grad()
+
+        loss, accuracy, labels_predicted, labels, logits = self.forward_step(batch, model)
+
+        loss.backward()
+        if grad_clipping_norm is not None:
+            torch.nn.utils.clip_grad_norm_(model.parameters(), grad_clipping_norm)
+        optimizer.step()
+        if scheduler is not None:
+            scheduler.step()
+
+        return loss.item(), accuracy
+
+    def test_step(
+        self, model, batch: List[torch.Tensor]
+    ) -> Tuple[float, float, list, list, torch.Tensor]:
+        model.eval()
+        batch[0] = batch[0].to(self.device)
+        batch[1] = batch[1].to(self.device)
+        batch[2] = batch[2].to(self.device)
+        with torch.no_grad():
+            loss, accuracy, labels_predicted, targets, logits = self.forward_step(batch, model)
+
+        return loss.item(), accuracy, labels_predicted, targets, logits
+
+    def training_loop(
+        self,
+        model,
+        train_dataloader,
+        val_dataloader,
+        optimizer,
+        epoch_num=1,
+        scheduler=None,
+        grad_clipping_norm=None,
+        validate_every=10,
+        verbose=True,
+        metrics_scorer=None,
+    ):
+        val_loss = np.inf
+        val_accuracy = 0
+
+        epoch_len = len(train_dataloader)
+
+        for epoch in range(epoch_num):
+            iter_loader = iter(train_dataloader)
+            pbar = self.get_progress_bar(epoch_len)
+            model.to(self.device)
+            for step_number in range(epoch_len):
+                batch = next(iter_loader)
+                self._global_steps_num += 1
+                train_loss, train_accuracy = self.training_step(
+                    model, batch, optimizer, scheduler, grad_clipping_norm
+                )
+
+                if step_number % validate_every == 0:
+                    batch = next(iter(val_dataloader))
+                    val_loss, val_accuracy, _, _, _ = self.test_step(model, batch)
+                if verbose:
+                    pbar.update(1)
+                    pbar.set_description(
+                        f'train loss: {round(train_loss, 3)}, ' f'val loss: {round(val_loss, 3)}'
+                    )
+                if self.tensorboard is not None and step_number % validate_every == 0:
+                    self.log_metrics_loss(train_loss, 'train', self._global_steps_num)
+                    self.log_metrics_loss(train_loss, 'val', self._global_steps_num)
+                    self.log_metrics_accuracy(train_accuracy, 'train', self._global_steps_num)
+                    self.log_metrics_accuracy(val_accuracy, 'val', self._global_steps_num)
+
+            # прогоняем модель на тесте/вале и сохраняем если она лучше по accuracy
+            predictions, _, targets, _ = self.predict(model, val_dataloader, verbose=verbose)
+
+            accuracy_df = None
+            f1_score = None
+            if metrics_scorer is not None:
+                accuracy_df = metrics_scorer.get_accuracies_df(targets, predictions)
+                f1_score = accuracy_df.loc[accuracy_df['index'] == 'macro avg'
+                                           ]['f1-score'].values[0]
+
+            accuracy = accuracy_score(targets, predictions)
+            logging.info(f'val accuracy after epoch {epoch}: {round(accuracy, 4)}')
+            if f1_score:
+                logging.info(f'val f1_score after epoch {epoch}: {round(f1_score, 4)}')
+
+            self.do_checkpoint(epoch,  f1_score or accuracy, model)
+
+    def predict(
+        self, model, dataloader, num_iterations=-1, with_all_probas=False, verbose=True,
+    ) -> Tuple[List, List, List, List]:
+        predictions_all = []
+        probas_all = []
+        targets_all = []
+        all_class_probas = []
+        model.to(self.device)
+        epoch_len = len(dataloader)
+        pbar = self.get_progress_bar(epoch_len)
+        for i, batch in enumerate(dataloader):
+            _, _, predictions, targets, logits = self.test_step(model, batch)
+            soft_probas = torch_softmax(logits, dim=1)
+            probas = soft_probas.max(dim=1).values.tolist()
+            if with_all_probas:
+                all_class_probas.extend(soft_probas.tolist())
+            predictions_all.extend(predictions)
+            probas_all.extend(probas)
+            targets_all.extend(targets)
+            if verbose:
+                pbar.update(1)
+                pbar.set_description('testing')
+            if i == num_iterations:
+                break
+
+        return predictions_all, probas_all, targets_all, all_class_probas
+
+
+def predict_img_batch(
+    image_paths: Optional[List[str]] = None,
+    image_dir: Optional[str] = None,
+    batch_size: int = 32,
+    model_path: str = '/app/data/models/model_resnet18',
+    labels: Optional[List[int]] = None,
+    with_all_probas: bool = False,
+    device='cuda:0',
+):
+    if (image_paths is None and image_dir is None) or (
+        image_paths is not None and image_dir is not None
+    ):
+        raise ValueError('only one from image_paths or image_dir should be specified')
+
+    if image_dir is not None:
+        image_paths = [os.path.join(image_dir, filename) for filename in os.listdir(image_dir)]
+
+    test_df = pd.DataFrame({TorchDataset.img_path_column: image_paths})
+
+    if labels is not None:
+        test_df[TorchDataset.label_column] = labels
+
+    dataset = TorchDataset(df=test_df, transformer=get_preprocessor(), with_cache=False)
+
+    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False)
+
+    room_clf, _ = RoomModel.from_pretrained(model_path)
+
+    trainer = TrainerUtils(device=device)
+    predictions_all, probas_all, targets_all, all_class_probas = trainer.predict(
+        room_clf, dataloader, with_all_probas=with_all_probas
+    )
+
+    return predictions_all, probas_all, targets_all, all_class_probas

aaa_ml_datasets_course-1.0.0/heuristics/model/utils.py

@@ -0,0 +1,144 @@
+import os
+from typing import (
+    Any,
+    List,
+    Optional,
+    Tuple,
+    Union,
+)
+
+import threading
+import urllib
+
+
+from collections import Counter
+import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.image as mpimg
+
+import torch
+from PIL import Image
+from torchvision import transforms
+from torchvision.transforms import functional as F
+
+
+from .settings import MAX_IMG_SIZE
+
+
+class PadCustom(transforms.Pad):
+    """
+    принимает на вход до какого размера нужно заппадить,
+    а transforms.Pad на сколько нужно западдить
+    """
+
+    def forward(self, img: Union[Image.Image, torch.Tensor]):
+        """
+        Args:
+            img (PIL Image or Tensor): Image to be padded.
+        Returns:
+            PIL Image or Tensor: Padded image.
+        """
+        if isinstance(img, torch.Tensor):
+            img_size = img.size()[1:]
+        else:
+            img_size = img.size
+
+        height_diff = self.padding[0] - img_size[0]
+        width_diff = self.padding[1] - img_size[1]
+
+        padding = (
+            height_diff // 2,
+            width_diff // 2,
+            height_diff // 2 + height_diff % 2,
+            width_diff // 2 + width_diff % 2,
+        )
+
+        return F.pad(img, padding, self.fill, self.padding_mode)
+
+
+def get_preprocessor(pad_size=MAX_IMG_SIZE):
+    preprocessor = transforms.Compose(
+        [
+            PadCustom(pad_size),
+            transforms.Resize(256),
+            transforms.CenterCrop(224),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        ]
+    )
+
+    return preprocessor
+
+
+def plot_imgs_with_labels(
+    image_paths: List[str],
+    labels_1: Tuple[List[Any], str] = None,
+    labels_2: Tuple[List[Any], str] = None,
+    labels_3: Tuple[List[Any], str] = None,
+):
+    plt.figure(figsize=(20, 20))
+
+    n_subplots = len(image_paths)
+    n_lines = int((n_subplots + 2) // 3)
+
+    for i in range(n_subplots):
+        plt.subplot(n_lines, 3, i+1)
+        img = mpimg.imread(image_paths[i])
+        title = ''
+        if labels_1 is not None:
+            title += f'{labels_1[1]}: {labels_1[0][i]}'
+        if labels_2 is not None:
+            if isinstance(labels_2[0][i], float):
+                labels_2[0][i] = round(labels_2[0][i], 4)
+            title += f'\n{labels_2[1]}: {labels_2[0][i]}'
+
+        if labels_3 is not None:
+            if isinstance(labels_3[0][i], float):
+                labels_3[0][i] = round(labels_3[0][i], 4)
+            title += f'\n{labels_3[1]}: {labels_3[0][i]}'
+
+        plt.title(title)
+        plt.imshow(img)
+
+
+def plot_sample(test_df, value='детская', column='label_pred', size=10):
+    plot_df = test_df[test_df[column] == value]
+    sample_size = min(plot_df.shape[0], size)
+
+    sample_df = plot_df.sample(sample_size).reset_index()
+
+    plot_imgs_with_labels(sample_df['img_path'],
+                          (sample_df['label'], 'label'),
+                          (sample_df['label_pred'], 'label_pred'),
+                          (sample_df['proba'], 'proba'))
+
+
+def _load_images(urls, save_dir):
+    results = []
+
+    def getter(url, dest):
+        results.append(urllib.request.urlretrieve(url, dest))
+
+    os.makedirs(save_dir, exist_ok=True)
+
+    threads = []
+    for url in urls:
+        filename = os.path.split(url)[-1]
+        t = threading.Thread(target=getter, args=(url, os.path.join(save_dir, filename)))
+        t.start()
+        threads.append(t)
+
+    map(lambda t: t.join(), threads)
+
+    image_paths = [os.path.join(save_dir, os.path.split(url)[-1]) for url in urls]
+
+    return image_paths
+
+
+def load_images(urls, save_dir, max_threads_num=50):
+    image_paths = []
+    for i in range(0, len(urls), max_threads_num):
+        new_image_paths = _load_images(urls[i:i+max_threads_num], save_dir)
+        image_paths.extend(new_image_paths)
+
+    return image_paths

aaa_ml_datasets_course-1.0.0/pyproject.toml

@@ -0,0 +1,41 @@
+[tool.poetry]
+name = "aaa-ml-datasets-course"
+version = "1.0.0"
+description = "Репозиторий курса по созданию датасетов Академии Аналитиков Авито"
+authors = ["Avito"]
+
+packages = [
+    { include = "heuristics", from = "."},
+]
+exclude = [
+    "**/*.ipynb",    
+]
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.13"
+pandas = "2.2.3"
+matplotlib = "3.8.4"
+seaborn = "0.13.2"
+scipy = "1.13.1"
+numpy = "1.26.4"
+scikit-learn = "1.4.2"
+transformers = "4.39.3"
+tensorboard = "2.16.2"
+nltk = "3.8.1"
+torch = "2.2.2"
+torchvision = "0.17.2"
+protobuf = "4.25.3"
+opencv-python = "4.9.0.80"
+
+toloka-kit = "1.2.1"
+tqdm = "4.67.1"
+ipykernel = "6.29.4"
+notebook = "7.4.1"
+ipywidgets = "8.1.6"
+
+together = "1.4.6"
+dotenv = "0.9.9"
+
+[build-system]
+requires = ["poetry-core>=1.2.0"]
+build-backend = "poetry.core.masonry.api"