leadguru-jobs 0.404.0__py3-none-any.whl → 0.405.0__py3-none-any.whl

lgt_jobs/lgt_data/analytics.py

@@ -1,723 +0,0 @@
-import os
-from typing import Optional, List, Dict, Union
-from collections import OrderedDict
-import datetime
-from datetime import timedelta
-from bson import ObjectId
-from dateutil import tz
-from lgt_jobs.lgt_data.mongo_repository import to_object_id
-from pymongo import MongoClient
-
-client = MongoClient(os.environ.get('MONGO_CONNECTION_STRING', 'mongodb://127.0.0.1:27017/'))
-db = client.lgt_analytics
-
-
-def _build_date_aggregated_analytics_pipeline(source_id=None,
-                                              email=None,
-                                              started_at: datetime.datetime = None,
-                                              ended_at: datetime.datetime = None,
-                                              bots_ids: [str] = None):
-    pipeline = [
-        {
-            "$sort": {"created_at": 1}
-        },
-        {
-            "$project": {
-                "created_at": {"$dateToString": {"format": "%Y-%m-%d", "date": "$created_at"}},
-                "name": "$name"
-            }
-        },
-        {
-            "$group":
-                {
-                    "_id": "$created_at",
-                    "count": {"$sum": 1}
-                }
-        },
-        {
-            "$project": {
-                "_id": {"$dateFromString": {"format": "%Y-%m-%d", "dateString": "$_id"}},
-                "count": "$count"
-            }
-        },
-        {
-            "$sort": {"_id": 1}
-        },
-        {"$limit": 1000}
-    ]
-
-    if source_id:
-        pipeline.insert(0, {"$match": {"source.source_id": source_id}})
-
-    if email:
-        pipeline.insert(0, {"$match": {"name": email}})
-
-    if started_at:
-        beginning_of_the_day = datetime.datetime(started_at.year, started_at.month, started_at.day, 0, 0, 0, 0)
-        pipeline.insert(0, {"$match": {"created_at": {"$gte": beginning_of_the_day}}})
-
-    if ended_at:
-        end_of_the_day = datetime.datetime(ended_at.year, ended_at.month, ended_at.day, 23, 59, 59, 999)
-        pipeline.insert(0, {"$match": {"created_at": {"$lte": end_of_the_day}}})
-
-    if bots_ids is not None:
-        pipeline.insert(0, {"$match": {'extra_ids': {'$in': bots_ids}}})
-
-    return pipeline
-
-
-def _create_result_dic(started_at: datetime.datetime = None, ended_at: datetime.datetime = None):
-    analytics_dict = OrderedDict()
-
-    if started_at and ended_at:
-        days_range = range(0, (ended_at - started_at).days + 1)
-        for day in days_range:
-            cur_date = started_at + datetime.timedelta(days=day)
-            str_date = f'{cur_date.year}-{cur_date.month:02d}-{cur_date.day:02d}'
-            analytics_dict[str_date] = 0
-
-    return analytics_dict
-
-
-def _prepare_date_analytics_doc(doc, ordered_result_dict: Dict[str, int]):
-    for item in doc:
-        str_date = f'{item["_id"].year}-{item["_id"].month:02d}-{item["_id"].day:02d}'
-        ordered_result_dict[str_date] = item["count"]
-    return ordered_result_dict
-
-
-def get_aggregated_user_leads(user_id: Union[ObjectId, str],
-                              from_date: datetime.datetime,
-                              to_date: datetime.datetime = None):
-    pipeline = [
-        {
-            '$match': {
-                'created_at': {'$gte': from_date}
-            }
-        }, {
-            '$group': {
-                '_id': {
-                    '$dateFromParts': {
-                        'day': {
-                            '$dayOfMonth': '$created_at'
-                        },
-                        'month': {
-                            '$month': '$created_at'
-                        },
-                        'year': {
-                            '$year': '$created_at'
-                        }
-                    }
-                },
-                'count': {
-                    '$sum': 1
-                },
-                'leads': {
-                    '$push': {
-                        'id': '$id',
-                        'created_at': '$created_at'
-                    }
-                }
-            }
-        }, {
-            '$sort': {
-                '_id': 1
-            }
-        }
-    ]
-
-    if user_id:
-        pipeline[0]["$match"]["user_id"] = to_object_id(user_id)
-
-    if to_date:
-        end = datetime.datetime(to_date.year, to_date.month, to_date.day, 23, 59, 59, tzinfo=tz.tzutc())
-        pipeline[0]["$match"]["created_at"]['$lte'] = end
-
-    user_leads_data = list(client.lgt_admin.user_leads.aggregate(pipeline))
-    result = _create_result_dic(from_date, to_date)
-
-    for item in user_leads_data:
-        str_date = f'{item["_id"].year}-{item["_id"].month:02d}-{item["_id"].day:02d}'
-        result[str_date] = item["count"]
-    return result
-
-
-def get_register_users_analytics(from_date: datetime = None, to_date: datetime = None):
-    pipeline = [
-        {
-            '$addFields': {
-                'created_at_formatted': {
-                    '$dateToString': {
-                        'format': '%Y-%m-%d',
-                        'date': '$created_at'
-                    }
-                }
-            }
-        }, {
-            '$group': {
-                '_id': '$created_at_formatted',
-                'count': {
-                    '$sum': 1
-                }
-            }
-        }
-    ]
-    if from_date:
-        beginning_of_the_day = datetime.datetime(from_date.year, from_date.month, from_date.day, 0, 0, 0, 0)
-        pipeline.insert(0, {"$match": {"created_at": {"$gte": beginning_of_the_day}}})
-
-    if to_date:
-        end_of_the_day = datetime.datetime(to_date.year, to_date.month, to_date.day, 23, 59, 59, 999)
-        pipeline.insert(0, {"$match": {"created_at": {"$lte": end_of_the_day}}})
-
-    users = list(client.lgt_admin.users.aggregate(pipeline))
-    users_dic = OrderedDict()
-
-    for item in users:
-        users_dic[item["_id"]] = item["count"]
-
-    return users_dic
-
-
-def get_global_saved_leads_analytics(from_date: datetime = None, to_date: datetime = None):
-    pipeline = [
-        {
-            '$addFields': {
-                'created_at_formatted': {
-                    '$dateToString': {
-                        'format': '%Y-%m-%d',
-                        'date': '$created_at'
-                    }
-                }
-            }
-        }, {
-            '$group': {
-                '_id': '$created_at_formatted',
-                'count': {
-                    '$sum': 1
-                }
-            }
-        }
-    ]
-
-    if from_date:
-        beginning_of_the_day = datetime.datetime(from_date.year, from_date.month, from_date.day, 0, 0, 0, 0)
-        pipeline.insert(0, {"$match": {"created_at": {"$gte": beginning_of_the_day}}})
-
-    if to_date:
-        end_of_the_day = datetime.datetime(to_date.year, to_date.month, to_date.day, 23, 59, 59, 999)
-        pipeline.insert(0, {"$match": {"created_at": {"$lte": end_of_the_day}}})
-
-    saved_leads = list(client.lgt_admin.user_leads.aggregate(pipeline))
-    saved_leads_dic = OrderedDict()
-    for item in saved_leads:
-        saved_leads_dic[str(item["_id"])] = item["count"]
-    return saved_leads_dic
-
-
-def get_global_uniq_leads_analytics(from_date: datetime = None, to_date: datetime = None):
-    pipeline = [
-        {
-            '$addFields': {
-                'created_at_formatted': {
-                    '$dateToString': {
-                        'format': '%Y-%m-%d',
-                        'date': '$created_at'
-                    }
-                }
-            }
-        }, {
-            '$group': {
-                '_id': {
-                    'created_at': '$created_at_formatted',
-                    'message': '$message.message_id'
-                },
-                'uniq_leads': {
-                    '$addToSet': '$message.message_id'
-                }
-            }
-        }, {
-            '$group': {
-                '_id': '$_id.created_at',
-                'uniq_leads_count': {
-                    '$sum': {
-                        '$size': '$uniq_leads'
-                    }
-                }
-            }
-        }
-    ]
-
-    if from_date:
-        beginning_of_the_day = datetime.datetime(from_date.year, from_date.month, from_date.day, 0, 0, 0, 0)
-        pipeline.insert(0, {"$match": {"created_at": {"$gte": beginning_of_the_day}}})
-
-    if to_date:
-        end_of_the_day = datetime.datetime(to_date.year, to_date.month, to_date.day, 23, 59, 59, 999)
-        pipeline.insert(0, {"$match": {"created_at": {"$lte": end_of_the_day}}})
-
-    uniq_leads = list(client.lgt_admin.user_leads.aggregate(pipeline))
-    uniq_leads_dic = OrderedDict()
-
-    for item in uniq_leads:
-        uniq_leads_dic[item["_id"]] = item["uniq_leads_count"]
-    return uniq_leads_dic
-
-
-def get_bots_global_analytics(from_date: datetime = None, to_date: datetime = None):
-    pipeline = [
-        {
-            '$addFields': {
-                'created_at_formatted': {
-                    '$dateToString': {
-                        'format': '%Y-%m-%d',
-                        'date': '$created_at'
-                    }
-                }
-            }
-        },
-        {
-            '$group': {
-                '_id': {
-                    'source_id': '$source.source_id',
-                    'created_at': '$created_at_formatted'
-                },
-                'count': {
-                    '$sum': 1
-                }
-            }
-        }, {
-            '$group': {
-                '_id': '$_id.created_at',
-                'bots': {
-                    '$sum': '$count'
-                },
-                'sources': {
-                    '$sum': {
-                        '$cond': [
-                            {
-                                '$eq': [
-                                    '$count', 1
-                                ]
-                            }, 1, 0
-                        ]
-                    }
-                }
-            }
-        }
-    ]
-    if from_date:
-        beginning_of_the_day = datetime.datetime(from_date.year, from_date.month, from_date.day, 0, 0, 0, 0)
-        pipeline.insert(0, {"$match": {"created_at": {"$gte": beginning_of_the_day}}})
-
-    if to_date:
-        end_of_the_day = datetime.datetime(to_date.year, to_date.month, to_date.day, 23, 59, 59, 999)
-        pipeline.insert(0, {"$match": {"created_at": {"$lte": end_of_the_day}}})
-
-    bots = list(client.lgt_admin.dedicated_bots.aggregate(pipeline))
-    result = {}
-    for bot in bots:
-        result.update({bot['_id']: {'bots': bot['bots'], 'sources': bot['sources']}})
-    return result
-
-
-def get_date_aggregated_analytics(source_id=None, started_at: datetime.datetime = None,
-                                  ended_at: datetime.datetime = None, bots_ids: [str] = None, configs: [str] = None):
-    pipeline: list = _build_date_aggregated_analytics_pipeline(source_id=source_id, started_at=started_at,
-                                                               ended_at=ended_at, bots_ids=bots_ids)
-
-    received_messages = list(db[f'received_messages'].aggregate(pipeline))
-
-    if configs is not None:
-        pipeline.insert(0, {"$match": {"attributes": {'$in': configs}}})
-
-    filtered_messages = list(db[f'filtered_messages'].aggregate(pipeline))
-
-    received_messages_dic = _create_result_dic(started_at, ended_at)
-    filtered_messages_dic = _create_result_dic(started_at, ended_at)
-
-    return (_prepare_date_analytics_doc(received_messages, received_messages_dic),
-            _prepare_date_analytics_doc(filtered_messages, filtered_messages_dic))
-
-
-def get_leads_aggregated_analytics(from_date: datetime.datetime = None, to_date: datetime.datetime = None):
-    pipeline = [
-        {
-            '$unwind': {
-                'path': '$message.configs',
-                'preserveNullAndEmptyArrays': False
-            }
-        },
-        {
-            '$group': {
-                '_id': '$message.configs.id',
-                'count': {
-                    '$sum': 1
-                }
-            }
-        }
-    ]
-
-    if from_date:
-        beginning_of_the_day = datetime.datetime(from_date.year, from_date.month, from_date.day, 0, 0, 0, 0)
-        pipeline.insert(0, {"$match": {"created_at": {"$gte": beginning_of_the_day}}})
-
-    if to_date:
-        end_of_the_day = datetime.datetime(to_date.year, to_date.month, to_date.day, 23, 59, 59, 999)
-        pipeline.insert(0, {"$match": {"created_at": {"$lte": end_of_the_day}}})
-
-    filtered_messages = list(client.lgt_admin.general_leads.aggregate(pipeline))
-    filtered_messages_dic = OrderedDict()
-
-    for item in filtered_messages:
-        filtered_messages_dic[str(item["_id"])] = item["count"]
-
-    return filtered_messages_dic
-
-
-def get_contacts_aggregated_analytics(from_date: datetime.datetime = None, to_date: datetime.datetime = None):
-    pipeline = [
-        {
-            '$group': {
-                '_id': {
-                    'user_id': '$user_id',
-                    'sender_id': '$message.sender_id'
-                },
-                'count': {
-                    '$sum': 1
-                }
-            }
-        }, {
-            '$project': {
-                'user_id': '$_id.user_id',
-                'sender_id': '$_id.sender_id',
-                'count': 1,
-                '_id': 0
-            }
-        }, {
-            '$group': {
-                '_id': '$user_id',
-                'contacts_count': {
-                    '$sum': 1
-                }
-            }
-        }
-    ]
-
-    if from_date:
-        beginning_of_the_day = datetime.datetime(from_date.year, from_date.month, from_date.day, 0, 0, 0, 0)
-        pipeline.insert(0, {"$match": {"created_at": {"$gte": beginning_of_the_day}}})
-
-    if to_date:
-        end_of_the_day = datetime.datetime(to_date.year, to_date.month, to_date.day, 23, 59, 59, 999)
-        pipeline.insert(0, {"$match": {"created_at": {"$lte": end_of_the_day}}})
-
-    contact_saved = list(client.lgt_admin.user_leads.aggregate(pipeline))
-    contact_saved_dic = OrderedDict()
-    for item in contact_saved:
-        contact_saved_dic[str(item["_id"])] = item["contacts_count"]
-
-    return contact_saved_dic
-
-
-def get_total_active_bots_global_analytics(from_date: datetime.datetime = None, to_date: datetime.datetime = None):
-    active_bot_dates = []
-    while from_date <= to_date:
-        active_bot_dates.append(from_date)
-        from_date += timedelta(days=1)
-    pipeline = [
-        {
-            '$match': {
-                'created_at': {
-                    '$lte': to_date
-                },
-                'deleted': False
-            }
-        }, {
-            '$addFields': {
-                'created_at_formatted': {
-                    '$dateToString': {
-                        'format': '%Y-%m-%d',
-                        'date': '$created_at'
-                    }
-                }
-            }
-        }, {
-            '$group': {
-                '_id': {
-                    'source': '$source.source_id',
-                    'created_at': '$created_at_formatted'
-                },
-                'count': {
-                    '$sum': 1
-                }
-            }
-        }, {
-            '$sort': {
-                '_id': 1
-            }
-        }
-    ]
-    bots = list(client.lgt_admin.dedicated_bots.aggregate(pipeline))
-    response = {}
-    count = 0
-    name = ""
-    for date in active_bot_dates:
-        for bot in bots:
-            bot_date = datetime.datetime.strptime(bot["_id"]["created_at"], "%Y-%m-%d")
-            if bot_date <= date:
-                if bot["_id"]["source"] == name:
-                    continue
-                else:
-                    count += 1
-            name = bot["_id"]["source"]
-        response[str(date.date())] = count
-        count = 0
-
-    return response
-
-
-def get_bots_aggregated_analytics(from_date: datetime.datetime = None,
-                                  to_date: datetime.datetime = None,
-                                  bot_ids: Optional[List[str]] = None,
-                                  configs: Optional[List[str]] = None):
-    pipeline = [
-        {
-            "$group": {
-                "_id": "$source.source_id",
-                "count": {"$sum": 1}
-            }
-        },
-        {"$limit": 1000}
-    ]
-
-    if from_date:
-        beginning_of_the_day = datetime.datetime(from_date.year, from_date.month, from_date.day, 0, 0, 0, 0)
-        pipeline.insert(0, {"$match": {"created_at": {"$gte": beginning_of_the_day}}})
-
-    if to_date:
-        end_of_the_day = datetime.datetime(to_date.year, to_date.month, to_date.day, 23, 59, 59, 999)
-        pipeline.insert(0, {"$match": {"created_at": {"$lte": end_of_the_day}}})
-
-    if bot_ids is not None:
-        pipeline.insert(0, {"$match": {"extra_ids": {"$in": bot_ids}}})
-
-    received_messages = list(db[f'received_messages'].aggregate(pipeline))
-
-    if configs is not None:
-        pipeline.insert(0, {"$match": {"attributes": {"$in": configs}}})
-
-    filtered_messages = list(db[f'filtered_messages'].aggregate(pipeline))
-    received_messages_dic = OrderedDict()
-    filtered_messages_dic = OrderedDict()
-
-    for item in received_messages:
-        received_messages_dic[item["_id"]] = item["count"]
-
-    for item in filtered_messages:
-        filtered_messages_dic[item["_id"]] = item["count"]
-
-    return received_messages_dic, filtered_messages_dic
-
-
-def get_channel_aggregated_analytics(from_date: datetime.datetime = None, to_date: datetime.datetime = None,
-                                     bot_id: Optional[str | ObjectId] = None):
-    pipeline = [
-        {
-            '$match': {
-                'extra_ids': {"$in": [str(bot_id)]}
-            }
-        },
-        {
-            "$group": {
-                '_id': {
-                    '$arrayElemAt': [
-                        '$attributes', 0
-                    ]
-                },
-                'count': {"$sum": 1}
-            }
-        },
-        {
-            '$limit': 1000
-        }
-    ]
-
-    if from_date:
-        beginning_of_the_day = datetime.datetime(from_date.year, from_date.month, from_date.day, 0, 0, 0, 0)
-        pipeline.insert(0, {"$match": {"created_at": {"$gte": beginning_of_the_day}}})
-
-    if to_date:
-        end_of_the_day = datetime.datetime(to_date.year, to_date.month, to_date.day, 23, 59, 59, 999)
-        pipeline.insert(0, {"$match": {"created_at": {"$lte": end_of_the_day}}})
-
-    received_messages = list(db[f'received_messages'].aggregate(pipeline))
-    filtered_messages = list(db[f'filtered_messages'].aggregate(pipeline))
-
-    received_messages_dic = OrderedDict()
-    filtered_messages_dic = OrderedDict()
-
-    for item in received_messages:
-        received_messages_dic[item["_id"]] = item["count"]
-
-    for item in filtered_messages:
-        filtered_messages_dic[item["_id"]] = item["count"]
-
-    return received_messages_dic, filtered_messages_dic
-
-
-def get_users_aggregated_analytics(event_type: str = 'user-lead-extended',
-                                   from_date: datetime.datetime = None,
-                                   to_date: datetime.datetime = None,
-                                   email: str = None):
-    pipeline = [
-        {
-            "$group": {
-                "_id": "$name",
-                "count": {"$sum": 1}
-            }
-        },
-        {"$limit": 1000}
-    ]
-
-    if email:
-        pipeline.insert(0, {"$match": {"name": email}})
-
-    if from_date:
-        beginning_of_the_day = datetime.datetime(from_date.year, from_date.month, from_date.day, 0, 0, 0, 0)
-        pipeline.insert(0, {"$match": {"created_at": {"$gte": beginning_of_the_day}}})
-
-    if to_date:
-        end_of_the_day = datetime.datetime(to_date.year, to_date.month, to_date.day, 23, 59, 59, 999)
-        pipeline.insert(0, {"$match": {"created_at": {"$lte": end_of_the_day}}})
-
-    read_messages = list(db[event_type].aggregate(pipeline))
-    read_messages_dic = OrderedDict()
-
-    for item in read_messages:
-        read_messages_dic[item["_id"]] = item["count"]
-
-    return read_messages_dic
-
-
-def get_user_date_aggregated_analytics(email=None, event_type: str = 'user-lead-extended',
-                                       started_at: datetime.datetime = None,
-                                       ended_at: datetime.datetime = None):
-    pipeline = _build_date_aggregated_analytics_pipeline(email=email, started_at=started_at, ended_at=ended_at)
-
-    messages = list(db[event_type].aggregate(pipeline))
-    messages_dic = _create_result_dic(started_at, ended_at)
-
-    for item in messages:
-        str_date = f'{item["_id"].year}-{item["_id"].month:02d}-{item["_id"].day:02d}'
-        messages_dic[str_date] = item["count"]
-
-    return messages_dic
-
-
-def get_user_read_count(lead_ids: [str]):
-    pipeline = [
-        {
-            "$group":
-                {
-                    "_id": "$data",
-                    "count": {"$sum": 1}
-                }
-        },
-        {
-            "$match": {
-                "_id": {"$in": lead_ids}
-            }
-        }
-    ]
-    messages = list(db['user-lead-extended'].aggregate(pipeline))
-    result = dict()
-
-    for message in messages:
-        result[message['_id']] = message['count']
-
-    return result
-
-
-def get_events_leads(email, event, from_date, to_date=None):
-    pipeline = {
-        'name': email,
-        'created_at': {'$gte': from_date}
-    }
-
-    if to_date:
-        end = datetime.datetime(to_date.year, to_date.month, to_date.day, 23, 59, 59, tzinfo=tz.tzutc())
-        pipeline['created_at']['$lte'] = end
-
-    return list(db[event].find(pipeline))
-
-
-def get_leads_aggregation(email, event, from_date, to_date=None):
-    pipeline = get_event_pipeline(email, from_date, to_date)
-    return list(db[event].aggregate(pipeline))
-
-
-def get_event_pipeline(email, from_date, to_date=None):
-    pipeline = [
-        {
-            "$match": {
-                'name': email,
-                'created_at': {'$gte': from_date}
-            }
-        },
-        {
-            '$group': {
-                '_id': {
-                    '$dateFromParts': {
-                        'day': {
-                            '$dayOfMonth': '$created_at'
-                        },
-                        'month': {
-                            '$month': '$created_at'
-                        },
-                        'year': {
-                            '$year': '$created_at'
-                        }
-                    }
-                },
-                'count': {
-                    '$sum': 1
-                }
-            }
-        },
-        {
-            "$sort": {"_id": 1}
-        }
-    ]
-
-    if to_date:
-        end = datetime.datetime(to_date.year, to_date.month, to_date.day, 23, 59, 59, tzinfo=tz.tzutc())
-        pipeline[0]["$match"]["created_at"]['$lte'] = end
-
-    return pipeline
-
-
-def get_total_analytic_followup_up_date(from_date, to_date=None):
-    beginning_of_the_day = datetime.datetime(from_date.year, from_date.month, from_date.day, 0, 0, 0, 0)
-    end_of_the_day = datetime.datetime(to_date.year, to_date.month, to_date.day, 23, 59, 59, 999)
-    pipeline = [
-        {
-            "$match": {
-                "followup_date": {
-                    "$gte": beginning_of_the_day,
-                    "$lte": end_of_the_day
-                }
-            }
-        },
-        {
-            "$group": {
-                "_id": "$_id",
-                "count": {"$sum": 1}
-            }
-        }
-    ]
-    followup_analytic = list(client.lgt_admin.user_leads.aggregate(pipeline))
-
-    followup_analytic_dic = {item["_id"]: item["count"] for item in followup_analytic}
-
-    return followup_analytic_dic