syncmodels 0.1.349__py2.py3-none-any.whl → 0.1.351__py2.py3-none-any.whl

syncmodels/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Asterio Gonzalez"""
 __email__ = "asterio.gonzalez@gmail.com"
-__version__ = "0.1.348"
+__version__ = "0.1.351"

syncmodels/crawler.py

@@ -403,7 +403,12 @@ class iAgent(iRunner):
 
             if self.add_task(task):
                 activity_log = ActivityStatus(
-                    activity_type="task", status="pending", name=task[KIND_KEY], **task
+                    **task,
+                    **{
+                        "activity_type": "task",
+                        "status": "pending",
+                        # "name": task[KIND_KEY],
+                    },
                 )
                 task[ACTIVITY_LOG_KEY] = activity_log
                 log.debug("+ Task: [%s]: [%s] %s", i, self.name, task)
@@ -418,7 +423,9 @@ class iAgent(iRunner):
             else:
                 log.debug("- Task: SKIP: [%s]: [%s] %s", i, self.name, task)
 
-        if i <= 0:
+        if i > 0:
+            log.info("[%s] %s/%s tasks added", self.name, i, len(tasks))
+        else:
             log.warning("[%s] no task provided by bootstrap()", self.name)
 
         log.debug("<< [%s] exit bootstrap()", self.name)
@@ -1140,6 +1147,8 @@ class iBot(iAgent):
                         msg = "".join(traceback.format_exception(*sys.exc_info()))
                         log.error(msg)
                         exception_raised = why
+                        await self.stop()
+                        foo = 1
 
                     log.warning("retry: %s: %s, %s", tries, call_kw, params)
                     await asyncio.sleep(self.RETRY_DELAY)
@@ -1186,7 +1195,14 @@ class iBot(iAgent):
         #     context[KIND_KEY] = context.pop("kind")
 
         for k, v in context.items():
-            if match(k) and isinstance(v, (int, str, float)):
+            if match(k) and isinstance(
+                v,
+                (
+                    int,
+                    str,
+                    float,
+                ),
+            ):
                 params[k] = v
 
         return params
@@ -1760,6 +1776,9 @@ class PutPlugin(iPlugin):
                         await activity_logger.update_activity(activity)
                         last_logged_percent = progress
                         last_logged_time = now
+                        await asyncio.sleep(
+                            0.25
+                        )  # be nice and try to unlock other fibers
 
             # Original inject logic
             await crawler.inject(data, context)
@@ -2623,6 +2642,13 @@ class iAsyncCrawler(iCrawler):
                     int(restart),
                 )
                 await asyncio.sleep(restart)
+                log.info(
+                    "[%s]: remaining cycles: [%s] : restart crawling right now!",
+                    self.bootstrap,
+                    self.cycles,
+                    # int(restart),
+                )
+                foo = 1
 
         result = await self.save()
         if result:
@@ -2726,14 +2752,13 @@ class iAsyncCrawler(iCrawler):
         bot.fiber = loop.create_task(bot.run())
 
     async def remove_bot(self, bot: iBot):
-        assert bot.name in self.bot
-
-        # request bot to finish
-        # bot.input_queue.put_nowait(None)
+        if bot.name in self.bot:
+            # request bot to finish
+            # bot.input_queue.put_nowait(None)
 
-        # remove bot for more assignations
-        self.round_robin.remove(bot)
-        self.bot.pop(bot.name)
+            # remove bot for more assignations
+            self.round_robin.remove(bot)
+            self.bot.pop(bot.name)
 
     def remain_tasks(self):
         "compute how many pending tasks still remains"

syncmodels/helpers/importers.py

@@ -1,7 +1,7 @@
 import os
 import re
 from typing import Dict, Any
-from datetime import datetime
+from datetime import datetime, date
 import pandas as pd
 
 
@@ -23,7 +23,7 @@ class JSONVerter:
 
     @classmethod
     def _to_json(cls, value: Any) -> Any:
-        if isinstance(value, datetime):
+        if isinstance(value, (datetime, date)):
             value = value.strftime("%FT%T.%f%Z")
         return value
 

syncmodels/registry.py

@@ -121,6 +121,11 @@ class iRegistry:
         for klass in __klass__:
             for pattern, info in cls.REGISTRY.items():
                 # Note: search(uri pattern) + match(uri when used will provide)
+                # try: # debug
+                #     re.search(pattern, uri) or re.match(uri, pattern)
+                # except Exception as why:
+                #     print(why)
+
                 if m := re.search(pattern, uri) or re.match(uri, pattern):
                     candidate = {}
                     for factory, options in info.items():

syncmodels/session/__init__.py

@@ -78,6 +78,7 @@ class iSession(iContext, iSchema, iRegistry):  # , iAuthenticator):
 
     QUERY_BODY_KEY = "json"
     PARAMS_KEY = "params"
+    EXTRA_KEY = None  # don't add extra info (aiohttp may fail)
 
     HEADERS = {}
 
@@ -417,6 +418,11 @@ class iSession(iContext, iSchema, iRegistry):  # , iAuthenticator):
         elif self.PARAMS_KEY:
             call_kw[self.PARAMS_KEY] = params
 
+        if self.EXTRA_KEY:
+            call_kw[self.EXTRA_KEY] = {
+                k: v for k, v in context.items() if k not in params
+            }
+
         return call_kw
 
 

{syncmodels-0.1.349.dist-info → syncmodels-0.1.351.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: syncmodels
-Version: 0.1.349
+Version: 0.1.351
 Summary: Synchronizable Models
 Home-page: https://github.com/asterio.gonzalez/syncmodels
 Author: Asterio Gonzalez
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.6
 License-File: LICENSE
 License-File: AUTHORS.rst
-Requires-Dist: agptools>=0.1.349
+Requires-Dist: agptools>=0.1.351
 Requires-Dist: aiocache
 Requires-Dist: aiohttp
 Requires-Dist: Click

{syncmodels-0.1.349.dist-info → syncmodels-0.1.351.dist-info}/RECORD

@@ -1,13 +1,13 @@
-syncmodels/__init__.py,sha256=zRskcOy0OBfyNpVc1dXZ89VD42tDgHQ5s_KX02h-_OU,146
+syncmodels/__init__.py,sha256=yY3_O0kIkf8luh4UrReqSq4x2fcMWfEiGT1p0C1LySE,146
 syncmodels/context.py,sha256=k1Gs_ip9BfyRFpyRnzqYvRDKo0sYBqJsh6z9sWln9oE,451
-syncmodels/crawler.py,sha256=lx0IU586JD0qcdozWiv0y7970CkUEg3SzFB6gOYDS6I,94540
+syncmodels/crawler.py,sha256=pE6lXz2mWgPsaYWnHWrtyjtq5J5CUjuau4D7mvHM2g4,95374
 syncmodels/crud.py,sha256=oZIcwEKR2i-lesEF_059Y4yThohd9m7gs6R6xYgLH-I,15351
 syncmodels/definitions.py,sha256=w-3TrSomp9T8OzLmJhKeZQDzrUIJLKldyh1lzlE7Yj0,5476
 syncmodels/exceptions.py,sha256=ZLAwu19cs2UN2Sv3jaLnixT_jRI7T42TfyutCkUsuIk,685
 syncmodels/geofactory.py,sha256=1FkrdEn0QA0O4_lSUAwjqXH2dmlQWi32AkntnG4AEQY,10372
 syncmodels/http.py,sha256=FFVT3QJJgur2dv1Q_7l9ZsWN8z6_gUjOT9hJff1ZAqk,3335
 syncmodels/parallel.py,sha256=Ll8HmyFF9v9fIofqqSgfhyTlklvb77mTtNdG5Y9lqdQ,7145
-syncmodels/registry.py,sha256=YaQtgbSwa0je1MpCcVHALI3_b85vrddyOlhsnrUcKZs,8224
+syncmodels/registry.py,sha256=3RhSdihKBkoxMUIflTUmPt3M4hA3mAoOGQ5V7joqL3o,8404
 syncmodels/requests.py,sha256=wWoC5hPDm1iBM_zrlyKRauzhXgdKR3pT5RqyC-5UZhQ,538
 syncmodels/runner.py,sha256=Tb5KTu_XFxCphrtfcnmIqTItG91el7P-9B5WaDTMp18,5928
 syncmodels/schema.py,sha256=HMr0LcB40se5sqHF4WYQIdzXUphLa2fhf0KFFUMzpDM,2726
@@ -29,7 +29,7 @@ syncmodels/helpers/explorer.py,sha256=-Dol3z1pALCMI9OPSvVbROaTzLjbUpS0suJ82Z6Rmb
 syncmodels/helpers/faker.py,sha256=tPtibNh28KoHb9kcwE9WaPdPrSrN6xMbCv5HhNFEVG0,1222
 syncmodels/helpers/general.py,sha256=UAcSfrvsaT15iJuxsR3WMk51UjpLLGDf14xmpBojndg,6160
 syncmodels/helpers/geojson.py,sha256=9VZjdEw19fLS_Q9topapUOPGxRg2D4GOqm2tlx11W7M,7370
-syncmodels/helpers/importers.py,sha256=KImR9pQu4ir6EI6Ipta0q3RWloFT_VTJi67kM0lZsKQ,3919
+syncmodels/helpers/importers.py,sha256=PWkORD2Qb0Ej2i8g34AODgBP7-oQlVB__9-_DlX2RmM,3933
 syncmodels/helpers/loaders.py,sha256=aus0aRcbU1vVa_zWo42aX6uV3B0fQ0aQpkTWlR9xGLA,4325
 syncmodels/helpers/models.py,sha256=c_ATzmiw5mVY1IGnwmyhjIuu5d2idHU-XeRigZSMkOQ,719
 syncmodels/helpers/orion.py,sha256=RY3hEVusUJDuSBN8YBy--RWnZ4TtP0jOUtNuHC1ySjg,30685
@@ -297,16 +297,16 @@ syncmodels/model/schema_org/webpage.py,sha256=-bqKOpEs_2lW7qrey2nHtvOZ9xbmmwcviN
 syncmodels/model/schema_org/webpageelement.py,sha256=brXfhU3l3FBXpy8qnR1Ve-EckjHW8VGoyR2IsnT7t2Y,1104
 syncmodels/model/schema_org/website.py,sha256=48Rox27BbFIg1u3wDlOtX-lLCPoFgvvrCw5Hrdf6uRU,912
 syncmodels/model/schema_org/xpathtype.py,sha256=D8gKiCrGSSuUVYw7BIWmOIUbKATfv2IpbkV1B2TmjC0,484
-syncmodels/session/__init__.py,sha256=89o6oeJsUaaiZENZDYJ2lP2PQBWX6OZPIzQacWjNh2k,15606
+syncmodels/session/__init__.py,sha256=HeyQiSg8RjCEBRF3VoZmuPyABognZ6wHNw0ekt7jLwY,15820
 syncmodels/session/http.py,sha256=tf7z0ccAEYoCOZT4Ukv3NBXz9hUO3vs2s9bm491pCj8,1480
 syncmodels/session/postgresql.py,sha256=ZMIu1Rv93pKfvFlovFBmWArzlrT2xaQWNYGZT_LW61k,175
 syncmodels/session/sql.py,sha256=Ia-Yrs_hOhAIZ4IcoVB5i8mQZVCULfnV9-TQz49knZU,7181
 syncmodels/session/sqlite.py,sha256=nCDjopLiBpX1F10qkKoARM7JrVdIpJ1WdGOduFVxaiA,2080
 syncmodels/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-syncmodels-0.1.349.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
-syncmodels-0.1.349.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
-syncmodels-0.1.349.dist-info/METADATA,sha256=IzZuWNDk71THUm5-pIXp3_l24hZ6Q28zTD_hXGp8j6E,2700
-syncmodels-0.1.349.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
-syncmodels-0.1.349.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
-syncmodels-0.1.349.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
-syncmodels-0.1.349.dist-info/RECORD,,
+syncmodels-0.1.351.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
+syncmodels-0.1.351.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
+syncmodels-0.1.351.dist-info/METADATA,sha256=WAOLdI0tEnOPkqttzU2VqOnuc6DsFsKlLyl65aeBglc,2700
+syncmodels-0.1.351.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
+syncmodels-0.1.351.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
+syncmodels-0.1.351.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
+syncmodels-0.1.351.dist-info/RECORD,,