xync-client 0.0.57.dev23__tar.gz → 0.0.57.dev25__tar.gz

{xync_client-0.0.57.dev23/xync_client.egg-info → xync_client-0.0.57.dev25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xync-client
-Version: 0.0.57.dev23
+Version: 0.0.57.dev25
 Author-email: Mike Artemiev <mixartemev@gmail.com>
 Project-URL: Homepage, https://gitlab.com/XyncNet/client
 Project-URL: Repository, https://gitlab.com/XyncNet/client

{xync_client-0.0.57.dev23 → xync_client-0.0.57.dev25}/xync_client/Bybit/agent.py

@@ -79,9 +79,10 @@ class AgentClient(BaseAgentClient):  # Bybit client
         "actionType": "MODIFY",
         "securityRiskToken": "",
     }
-    all_conds: dict[int, tuple[str, set[str]]] = {}
-    cond_sims: dict[int, tuple[int, int]] = {}
-    sim_conds: dict[int, set[int]] = defaultdict(set)  # backward
+    all_conds: dict[int, tuple[str, set[int]]] = {}
+    cond_sims: dict[int, int] = defaultdict(set)
+    rcond_sims: dict[int, set[int]] = defaultdict(set)  # backward
+    tree: dict = {}
 
     def __init__(self, actor: Actor, bot: FileClient, **kwargs):
         super().__init__(actor, bot, **kwargs)
@@ -395,7 +396,7 @@ class AgentClient(BaseAgentClient):  # Bybit client
                 size=30,
                 # status=status,  # 50 - завершено
                 # tokenId=token_id,
-                # beginTime=begin_time,
+                beginTime=begin_time,
                 # endTime=end_time,
                 # side=side,  # 1 - продажа, 0 - покупка
             )
@@ -713,7 +714,8 @@ class AgentClient(BaseAgentClient):  # Bybit client
                 if rest_uids := {ra.maker_id for ra in rest_ads} - {ad_db.maker_id}:
                     # создадим новое условие и присвоим его только текущей объяве
                     new_cond = await Cond.create(raw_txt=cleaned)
-                    await self.sim_new(new_cond.id, get_sim(cleaned, ad_db.cond.raw_txt), ad_db.cond_id)
+                    if sim := get_sim(cleaned, ad_db.cond.raw_txt):
+                        await CondSim.create(cond_id=new_cond.id, similarity=sim, cond_rel_id=ad_db.cond_id)
                     ad_db.cond_id = new_cond.id
                     self.all_conds[ad_db.cond_id] = cleaned, {ad_db.maker.exid}
                     await ad_db.save()
@@ -729,36 +731,71 @@ class AgentClient(BaseAgentClient):  # Bybit client
             ad_db.cond.raw_txt = cleaned
             self.all_conds[ad_db.cond_id] = cleaned, {ad_db.maker.exid}
             await ad_db.cond.save()
-            await self.sim_upd(ad_db.cond_id, cleaned)
+            await self.sim_check_rels(ad_db.cond_id, cleaned)
             return ad_db, False
 
-        # находим все старые тексты похожие на 90% и более
-        if _sims := {
-            old_cid: (txt, sim)
-            for old_cid, (txt, uids) in self.all_conds.items()
-            if len(cleaned) > 15 and ad.userId not in uids and (sim := get_sim(cleaned, txt))
-        }:
-            # если есть, берем самый похожий из них
-            old_cid, (txt, sim) = max(_sims.items(), key=lambda x: x[1][1])
-            old_ads = await models.Ad.filter(cond_id=old_cid, maker__exid=int(ad.userId)).prefetch_related("cond")
-            for old_ad in old_ads:
-                # и у этого чела есть объява с почти таким же текстом
-                if old_ad.exid == int(ad.id):  # и он изменил текст как раз в ней
-                    # заменяем текст без создания нового cond
-                    await old_ad.cond.update_or_create(raw_txt=cleaned)
-                    await old_ad.fetch_related("cond")
-                    return old_ad, False
-                # но это не она, значит у него есть другая объява с похожим, но чуть отличающимся текстом
-                logging.warning(f"ad#{ad.id}-cond#{old_cid} txt updated:\n{txt}\n|\nV\n{cleaned}")
-
         new_cond = await Cond.create(raw_txt=cleaned)
-        self.all_conds[new_cond.id] = new_cond.raw_txt, {ad.userId}
+        # находим все старые тексты похожие на 90% и более
+        old_cid, sim = await self.cond_get_max_sim(new_cond.id, new_cond.raw_txt, {int(ad.userId)})
+        # old_ads = await models.Ad.filter(cond_id=old_cid, maker__exid=int(ad.userId)).prefetch_related("cond")
+        # for old_ad in old_ads:
+        #     # и у этого чела есть объява с почти таким же текстом
+        #     if old_ad.exid == int(ad.id):  # и он изменил текст как раз в ней
+        #         # заменяем текст без создания нового cond
+        #         await old_ad.cond.update_or_create(raw_txt=cleaned)
+        #         await old_ad.fetch_related("cond")
+        #         return old_ad, False
+        #     # но это не она, значит у него есть другая объява с похожим, но чуть отличающимся текстом
+        #     logging.warning(f"ad#{ad.id}-cond#{old_cid} txt updated:\n{txt}\n|\nV\n{cleaned}")
+
+        self.all_conds[new_cond.id] = new_cond.raw_txt, {int(ad.userId)}
         # если нашелся похожий текст у другого юзера, добавим связь с % похожести
         if sim:
-            await self.sim_new(new_cond.id, sim, old_cid)
+            await CondSim.create(cond_id=new_cond.id, similarity=sim, cond_rel_id=old_cid)
 
         return await self.ad_create(ad, new_cond.id), True
 
+    def find_in_tree(self, cid: int, old_cid: int) -> bool:
+        if p := self.cond_sims.get(old_cid):
+            if p == cid:
+                return True
+            return self.find_in_tree(cid, p)
+        return False
+
+    async def cond_get_max_sim(self, cid: int, txt: str, uids: set[int]) -> tuple[int | None, int | None]:
+        # находим все старые тексты похожие на 90% и более
+        if len(txt) < 15:
+            return None, None
+        sims: dict[int, int] = {}
+        for old_cid, (old_txt, old_uids) in self.all_conds.items():
+            if len(old_txt) < 15 or uids == old_uids:
+                continue
+            elif not self.can_add_sim(cid, old_cid):
+                continue
+            if sim := get_sim(txt, old_txt):
+                sims[old_cid] = sim
+        # если есть, берем самый похожий из них
+        if sims:
+            old_cid, sim = max(sims.items(), key=lambda x: x[1])
+            await sleep(0.3)
+            return old_cid, sim
+        return None, None
+
+    def can_add_sim(self, cid: int, old_cid: int) -> bool:
+        if cid == old_cid:
+            return False
+        elif self.cond_sims.get(cid) == old_cid:
+            return False
+        elif self.find_in_tree(cid, old_cid):
+            return False
+        elif self.cond_sims.get(old_cid) == cid:
+            return False
+        elif cid in self.rcond_sims.get(old_cid, {}):
+            return False
+        elif old_cid in self.rcond_sims.get(cid, {}):
+            return False
+        return True
+
     async def person_upsert(self, name: str, exid: int) -> models.Person:
         if actor := await models.Actor.get_or_none(exid=exid, ex=self.ex_client.ex).prefetch_related("person"):
             if not actor.person:
@@ -793,29 +830,75 @@ class AgentClient(BaseAgentClient):  # Bybit client
         await ad_db.pms.add(*(await models.Pm.filter(pmexs__ex=self.ex_client.ex, pmexs__exid__in=ad.payments)))
         return ad_db
 
-    async def sim_new(self, new_cid: int, sim: int, old_cid: int):
-        if not sim:
-            return None
-        return await CondSim.create(cond_id=new_cid, similarity=sim, cond_rel_id=old_cid)
-
-    async def sim_upd(self, cid: int, new_txt: int):
-        for sim_db in (_sims := await CondSim.filter(Q(join_type="OR", cond_id=cid, cond_rel_id=cid))):
-            (op,) = {sim_db.cond_id, sim_db.cond_rel_id} - {cid}
-            op_cond = await Cond[op]
-            sim_db.similarity = get_sim(new_txt, op_cond.raw_txt)
-            await sim_db.save()
+    async def sim_check_rels(self, cid: int, new_txt: int):
+        if _old_sims := await CondSim.filter(Q(join_type="OR", cond_id=cid, cond_rel_id=cid)):
+            for sim_db in _old_sims:
+                (op,) = {sim_db.cond_id, sim_db.cond_rel_id} - {cid}
+                op_cond = await Cond[op]
+                sim_db.similarity = get_sim(new_txt, op_cond.raw_txt)
+                await sim_db.save()
+
+    def build_tree(self):
+        set(self.cond_sims.keys()) | set(self.cond_sims.values())
+        tree = defaultdict(dict)
+        # Группируем родителей по детям
+        for child, par in self.cond_sims.items():
+            tree[par] |= {child: {}}  # todo: make from self.rcond_sim
+
+        # Строим дерево снизу вверх
+        def subtree(node):
+            if not node:
+                return node
+            for key in node:
+                subnode = tree.pop(key, {})
+                d = subtree(subnode)
+                node[key] |= d  # actual tree rebuilding here!
+            return node  # todo: refact?
+
+        # Находим корни / без родителей
+        roots = set(self.cond_sims.values()) - set(self.cond_sims.keys())
+        for root in roots:
+            _ = subtree(tree[root])
+
+        self.tree = tree
 
     async def actual_cond(self):
-        self.all_conds = {
-            c.id: (c.raw_txt, {str(a.maker.exid) for a in c.ads})
-            for c in await Cond.all().prefetch_related("ads__maker")
-        }
-        self.cond_sims = {cs.cond_id: (cs.cond_rel_id, cs.similarity) for cs in await CondSim.all()}
-        for c, (o, s) in self.cond_sims.items():
-            self.sim_conds[o].add(c)
-        for ad_db in await models.Ad.filter(direction__pairex__ex=self.ex_client.ex).prefetch_related("cond", "maker"):
-            ad = Ad(id=str(ad_db.exid), userId=str(ad_db.maker.exid), remark=ad_db.cond.raw_txt)
-            await self.cond_upsert(ad, force=True)
+        for curr, old in await CondSim.all().values_list("cond_id", "cond_rel_id"):
+            self.cond_sims[curr] = old
+            self.rcond_sims[old] |= {curr}
+        self.build_tree()
+        for cid, (txt, uids) in self.all_conds.items():
+            old_cid, sim = await self.cond_get_max_sim(cid, txt, uids)
+            if sim:
+                if old_sim := await CondSim.get_or_none(cond_id=cid):
+                    if old_sim.cond_rel_id != old_cid:
+                        if sim <= old_sim.similarity:
+                            # old_rsim = await CondSim.get_or_none(cond_id=old_cid)
+                            # if not old_rsim:
+                            #     await CondSim.create(cond_id=old_cid, cond_rel_id=cid, similarity=sim)
+                            #     self.cond_sims[old_cid] = cid
+                            #     self.rcond_sims[cid] |= {old_cid}
+                            #     # self.add_to_tree(old_cid, cid, self.tree)
+                            # elif old_rsim.similarity < sim:
+                            #     await CondSim.update_or_create({"cond_rel_id": cid, "similarity": sim}, cond_id=old_cid)
+                            #     self.cond_sims[old_cid] = cid
+                            #     self.rcond_sims[cid] |= {old_cid}
+                            #     self.rcond_sims[old_rsim.cond_rel_id].remove(old_rsim.cond_id)
+                            #     logging.warning(f'{old_cid}: {old_sim.similarity}->{sim} ({old_sim.cond_rel_id}->{cid})')
+                            continue
+                        logging.warning(f"R {cid}: {old_sim.similarity}->{sim} ({old_sim.cond_rel_id}->{old_cid})")
+                        await old_sim.update_from_dict({"similarity": sim, "old_rel_id": old_cid}).save()
+                    elif sim != old_sim.similarity:
+                        logging.info(f"{cid}: {old_sim.similarity}->{sim}")
+                        await old_sim.update_from_dict({"similarity": sim}).save()
+                else:
+                    await CondSim.create(cond_id=cid, cond_rel_id=old_cid, similarity=sim)
+                    self.cond_sims[cid] = old_cid
+                    self.rcond_sims[old_cid] |= {cid}
+                    # self.add_to_tree(cid, old_cid, self.tree)
+            # for ad_db in await models.Ad.filter(direction__pairex__ex=self.ex_client.ex).prefetch_related("cond", "maker"):
+            #     ad = Ad(id=str(ad_db.exid), userId=str(ad_db.maker.exid), remark=ad_db.cond.raw_txt)
+            #     await self.cond_upsert(ad, force=True)
 
     async def get_credexs_by_norms(self, norms: list[str], cur_id: int) -> dict[models.Pmex, models.CredEx] | None:
         try:
@@ -833,7 +916,7 @@ class AgentClient(BaseAgentClient):  # Bybit client
 
 
 def get_sim(s1, s2) -> int:
-    sim = int((SequenceMatcher(None, s1, s2).ratio() - 0.9) * 10_000)
+    sim = int((SequenceMatcher(None, s1, s2).ratio() - 0.6) * 10_000)
     return sim if sim > 0 else 0
 
 
@@ -901,12 +984,24 @@ async def main():
         usdc = await models.Coinex.get(coin__ticker="USDC", ex=cl.actor.ex).prefetch_related("coin")
         rub = await models.Curex.get(cur__ticker="RUB", ex=cl.actor.ex).prefetch_related("cur")
         cl.all_conds = {
-            c.id: (c.raw_txt, {str(a.maker.exid) for a in c.ads})
-            for c in await Cond.all().prefetch_related("ads__maker")
+            c.id: (c.raw_txt, {a.maker.exid for a in c.ads}) for c in await Cond.all().prefetch_related("ads__maker")
         }
+        for curr, old in await CondSim.filter().values_list("cond_id", "cond_rel_id"):
+            cl.cond_sims[curr] = old
+            cl.rcond_sims[old] |= {curr}
+        cl.build_tree()
+        a = set()
+
+        def check_tree(tre):
+            for p, c in tre.items():
+                a.add(p)
+                check_tree(c)
+
+        for pr, ch in cl.tree.items():
+            check_tree(ch)
         # await cl.actual_cond()
         await gather(
-            cl.get_api_orders(20),  # 10, 1738357200000, 1742504399999
+            cl.get_api_orders(1, 1742504400000),  # 10, 1738357200000, 1742504399999
             cl.battle(usdt, rub, False, ["volet"], 79.8),  # гонка в стакане покупки - мы продаем
             cl.battle(usdt, rub, True, ["volet"], 79.8),  # гонка в стакане продажи - мы покупаем
             cl.battle(eth, rub, False, ["volet"], 206_000),

{xync_client-0.0.57.dev23 → xync_client-0.0.57.dev25/xync_client.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xync-client
-Version: 0.0.57.dev23
+Version: 0.0.57.dev25
 Author-email: Mike Artemiev <mixartemev@gmail.com>
 Project-URL: Homepage, https://gitlab.com/XyncNet/client
 Project-URL: Repository, https://gitlab.com/XyncNet/client