syncmodels 0.1.331__py2.py3-none-any.whl → 0.1.334__py2.py3-none-any.whl

syncmodels/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Asterio Gonzalez"""
 __email__ = "asterio.gonzalez@gmail.com"
-__version__ = "0.1.331"
+__version__ = "0.1.334"

syncmodels/crawler.py

@@ -851,7 +851,7 @@ class iBot(iAgent):
         self.add_plugin(MetaExtractPlugin())
         # self.add_plugin(PaginatorPlugin())
 
-        self.add_plugin(PutPlugin())
+        self.add_plugin(PutPlugin())  # comment this for dry-run
 
         self.add_plugin(SetURIPlugin())
         self.add_plugin(Cleaner())
@@ -896,7 +896,7 @@ class iBot(iAgent):
 
     async def get_data(self, **task):
         """
-        Example a crawling function for recommender crawler.
+        Example a crawling function for crawler.
 
         Get data related to the given kind and path.
         May add more tasks to be done by crawler.
@@ -1201,7 +1201,6 @@ class iBot(iAgent):
         # TODO: I need to clean really old cache entries
         # TODO: in order to control any excesive memory compsumption
         try:
-
             if not self._is_already(timeout=timeout, type_=type_, **params):
                 blueprint = self.blueprint(**params)
                 universe = self.ALREADY.setdefault(type_, {})
@@ -1610,34 +1609,14 @@ class SortPlugin(iPlugin):
             # )
             reverse = list(set(sort_key).intersection(item_fields))
             if not reverse:
-                log.error("model [%s] has not datetime alike key??", model)
-                return
+                log.debug(
+                    "model [%s] attributes:[%s] doesn't match any datetime alike sort keys: [%s]",
+                    model,
+                    list(item_fields),
+                    sort_key,
+                )
+                # return
 
-            if False:
-                sample = stream[-1]
-                reverse = []
-                for key in sort_key:
-                    value = sample[key]
-                    if func := TYPES_MAP.get(value.__class__):
-                        candidates = {
-                            k: v
-                            for k, v in sample.items()
-                            if k != key and func(sample[k]) == func(value)
-                        }
-                        if candidates:
-                            _key, _ = candidates.popitem()
-                            reverse.append(_key)
-                        else:
-                            log.debug(
-                                "can't find the reverse key of '%s' in %s",
-                                sort_key,
-                                sample,
-                            )
-                            # # using same sort_key because source and target
-                            # # layouts share the same sorteable key (i.e. 'datetime')??
-                            # reverse.append(key)
-                    else:
-                        pass
             context[REVERSE_SORT_KEY] = reverse
 
         if sort_key:
@@ -1656,8 +1635,13 @@ class SortPlugin(iPlugin):
             item_fields = model.model_fields
             reverse = list(set(sort_key).intersection(item_fields))
             if not reverse:
-                log.error("model [%s] has not datetime alike key??", model)
-                return
+                log.debug(
+                    "model [%s] attributes:[%s] doesn't match any datetime alike sort keys: [%s]",
+                    model,
+                    list(item_fields),
+                    sort_key,
+                )
+                # return
 
             context[REVERSE_SORT_KEY] = reverse
 
@@ -1714,7 +1698,7 @@ class HashStreamPlugin(iPlugin):
         blueprint = blueprint.hexdigest()
         context["stream_blueprint"] = blueprint
         if not self.bot._set_already(
-            timeout=3600,
+            # timeout=3600,
             type_="response",
             blueprint=blueprint,
         ):

syncmodels/helpers/surreal.py

@@ -53,9 +53,19 @@ class SurrealServer:
     REG_VERSION = r"(?P<version>(?P<a>\d+)(\.(?P<b>\d+))?(\.(?P<c>\d+))?)"
     SURREAL_PID = ".surreal.pid"
 
-    def __init__(self, path, bind="0.0.0.0:9000", daemon=False, version=""):
+    def __init__(
+        self,
+        path,
+        bind="0.0.0.0:9000",
+        user="root",
+        password="root",
+        daemon=False,
+        version="",
+    ):
         self.path = path
         self.bind = bind
+        self.user = user
+        self.password = password
         self.daemon = daemon
         self.proc = None
         self.pid = None
@@ -99,6 +109,7 @@ class SurrealServer:
                                 print(f"exit: {version}")
                                 break
                         else:
+                            self._version = version
                             return path
 
                 except TimeoutExpired:
@@ -110,18 +121,39 @@ class SurrealServer:
             executable = self.find_executable()
         else:
             executable = "surreal"
-        return [
-            executable,
-            "start",
-            "--allow-all",
-            "--bind",
-            f"{self.bind}",
-            "--user",
-            "root",
-            "--pass",
-            "root",
-            f"file://{self.path}",
-        ]
+
+        path = os.path.abspath(self.path)
+
+        if self._version.get("a") in ("2",):
+            return [
+                executable,
+                "start",
+                "--allow-all",
+                "--bind",
+                f"{self.bind}",
+                "-u",
+                f"{self.user}",
+                "-p",
+                f"{self.password}",
+                "--log",
+                "debug",
+                "--changefeed-gc-interval",
+                "600s",
+                f"rocksdb://{path}",
+            ]
+        else:
+            return [
+                executable,
+                "start",
+                "--allow-all",
+                "--bind",
+                f"{self.bind}",
+                "-u",
+                f"{self.user}",
+                "-p",
+                f"{self.password}",
+                f"file://{path}",
+            ]
 
     def start(self):
         """starts surreal process and register a callback is anything goes wrong"""

syncmodels/storage.py

@@ -1183,7 +1183,7 @@ class WaveStorage(iWaves, iStorage):
                 if must_check:
                     t0 = time.time()
                     await prevously_inserted()
-                    if random.random() < 0.1:
+                    if False or random.random() < 0.1:
                         elapsed = time.time() - t0
                         log.info("[%s] prevously_inserted took: %s secs", uid, elapsed)
                         if elapsed > 1.0:
@@ -1207,6 +1207,7 @@ class WaveStorage(iWaves, iStorage):
 
                 data[ID_KEY] = "{thing}:{id}".format_map(_uri)
 
+                res0 = res1 = res2 = False
                 # must push the data?
                 context[PUSHED] = push
                 if push:
@@ -1217,98 +1218,93 @@ class WaveStorage(iWaves, iStorage):
                             data_sort_blueprint,
                             data,
                         )
-                        foo = 1
                     query = f"{namespace}://{database}/{thing}:{monotonic}"
                     res2 = await self.storage.put(query, data)
                 else:
                     # TODO: agp: refactor all this function when we've time!
                     res2 = True
-                    foo = 1
 
-                # long fquid version
-                # data[ID_KEY] = data[ORG_KEY]
-                # short version
-                # data[ID_KEY] = "{thing}:{id}".format_map(_uri)
+                # save Snapshot of the object
+                if push:
+                    # long fquid version
+                    # data[ID_KEY] = data[ORG_KEY]
+                    # short version
+                    # data[ID_KEY] = "{thing}:{id}".format_map(_uri)
+                    if _uri["id"] is None:
+                        __id = data.get(ORG_KEY, "")
+                        _id = parse_duri(__id)
+                        if _id["id"] is None:
+                            _uri["id"] = __id
+                        else:
+                            _uri["id"] = _id["id"]
+                        data[ORG_KEY] = build_uri(**_uri)
+
+                    data[ID_KEY] = "{thing}:{id}".format_map(_uri)
+                    query = f"{namespace}://{database}/{TUBE_SNAPSHOT}"
+                    # resuming_info = {k: kw[k] for k in kw.get(REVERSE_SORT_KEY, [])}
+                    # resuming_info = {k: kw[k] for k in kw.get(SORT_KEY) or []}
+                    resuming_info = {
+                        k: kw.get(k, data.get(k)) for k in kw.get(SORT_KEY) or []
+                    }
+                    # force to be json compatible
+                    resuming_info = JSONVerter.to_json(resuming_info)
 
-                if _uri["id"] is None:
-                    __id = data.get(ORG_KEY, "")
-                    _id = parse_duri(__id)
-                    if _id["id"] is None:
-                        _uri["id"] = __id
+                    res0 = await self.storage.put(
+                        query,
+                        data,
+                    )
+                    # 3. finally add the wave info into tube
+                    data.pop(MONOTONIC_KEY)
+
+                # update the TUBE_WAVE due the insertion of this object
+                if push:
+                    # 2. save last Wave from this particular tube
+                    # AVOID_KEYS contains all keys that aren't json serializable
+                    # wave = {
+                    #     k: v for k, v in kw.items() if k not in self.AVOID_KEYS
+                    # }
+                    # try to recover the 'intact' bootstrap that we've using
+                    for wave0 in kw.get(WAVE_LAST_KEY, []):
+                        wave = wave0.get("wave")  # TODO: use a define
+                        if wave:
+                            break
                     else:
-                        _uri["id"] = _id["id"]
-                    data[ORG_KEY] = build_uri(**_uri)
+                        # otherwise, its the 1st time and we need to create the 1st
+                        # bootstrap-wave info
+                        wave_keys = set(kw.get(WAVE_INFO_KEY, []))
+                        wave_keys.update([KIND_KEY, PREFIX_URL, PARAMS_KEY])
+                        # task = kw[TASK_KEY]
+                        wave = {k: kw[k] for k in wave_keys.intersection(kw)}
 
-                data[ID_KEY] = "{thing}:{id}".format_map(_uri)
-                query = f"{namespace}://{database}/{TUBE_SNAPSHOT}"
-                # resuming_info = {k: kw[k] for k in kw.get(REVERSE_SORT_KEY, [])}
-                # resuming_info = {k: kw[k] for k in kw.get(SORT_KEY) or []}
-                resuming_info = {
-                    k: kw.get(k, data.get(k)) for k in kw.get(SORT_KEY) or []
-                }
-                # force to be json compatible
-                resuming_info = JSONVerter.to_json(resuming_info)
-
-                res0 = await self.storage.put(
-                    query,
-                    data,
-                )
-                # 3. finally add the wave info into tube
-                data.pop(MONOTONIC_KEY)
-
-                # 2. save last Wave from this particular tube
-                # AVOID_KEYS contains all keys that aren't json serializable
-                # wave = {
-                #     k: v for k, v in kw.items() if k not in self.AVOID_KEYS
-                # }
-
-                # try to recover the 'intact' bootstrap that we've using
-                for wave0 in kw.get(WAVE_LAST_KEY, []):
-                    wave = wave0.get("wave")  # TODO: use a define
                     if wave:
-                        break
-                else:
-                    # otherwise, its the 1st time and we need to create the 1st
-                    # bootstrap-wave info
-
-                    # TODO: review for inclussion
-                    # TODO: this settings are crawler activity oriented
-                    # TODO: what about 'Task' an other use cases?
-                    wave_keys = set(kw.get(WAVE_INFO_KEY, []))
-                    wave_keys.update([KIND_KEY, PREFIX_URL, PARAMS_KEY])
-
-                    # task = kw[TASK_KEY]
-                    wave = {k: kw[k] for k in wave_keys.intersection(kw)}
-
-                if wave:
-                    # wave must be json compatible and do not use any reserved
-                    # keyword for storage (i.e. 'scope' in Surreal)
-                    query = f"{namespace}://{database}/{TUBE_WAVE}"
-
-                    # query can't containg MONOTONIC_KEY
-                    wave.pop(MONOTONIC_KEY, None)
-                    wave.pop(WAVE_RESUMING_INFO_KEY, None)
-                    exists = await self.storage.query(query, **wave)
-                    assert len(exists) <= 1
-
-                    if len(exists):
-                        # use the same record_id
-                        # otherwise a new record will be created
-                        wave = exists[0]
-                    # stamp current wave
-                    _wave = {
-                        **wave,
-                        MONOTONIC_KEY: monotonic,
-                        WAVE_RESUMING_INFO_KEY: resuming_info,
-                    }
-                    res1 = await self.storage.put(query, _wave)
-                else:
-                    # wave is empty, maybe because is not a resuming crawling task
-                    log.debug(
-                        "wave is empty, maybe because is not a resuming crawling task"
-                    )
-                    # log.info("Saving: %s", data)
-                    res1 = True
+                        # wave must be json compatible and do not use any reserved
+                        # keyword for storage (i.e. 'scope' in Surreal)
+                        query = f"{namespace}://{database}/{TUBE_WAVE}"
+
+                        # query can't containg MONOTONIC_KEY
+                        wave.pop(MONOTONIC_KEY, None)
+                        wave.pop(WAVE_RESUMING_INFO_KEY, None)
+                        exists = await self.storage.query(query, **wave)
+                        assert len(exists) <= 1
+
+                        if len(exists):
+                            # use the same record_id
+                            # otherwise a new record will be created
+                            wave = exists[0]
+                        # stamp current wave
+                        _wave = {
+                            **wave,
+                            MONOTONIC_KEY: monotonic,
+                            WAVE_RESUMING_INFO_KEY: resuming_info,
+                        }
+                        res1 = await self.storage.put(query, _wave)
+                    else:
+                        # wave is empty, maybe because is not a resuming crawling task
+                        log.debug(
+                            "wave is empty, maybe because is not a resuming crawling task"
+                        )
+                        # log.info("Saving: %s", data)
+                        res1 = True
 
                 return all([res0, res1, res2])
             else:

{syncmodels-0.1.331.dist-info → syncmodels-0.1.334.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: syncmodels
-Version: 0.1.331
+Version: 0.1.334
 Summary: Synchronizable Models
 Home-page: https://github.com/asterio.gonzalez/syncmodels
 Author: Asterio Gonzalez
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.6
 License-File: LICENSE
 License-File: AUTHORS.rst
-Requires-Dist: agptools>=0.1.331
+Requires-Dist: agptools>=0.1.334
 Requires-Dist: aiocache
 Requires-Dist: aiohttp
 Requires-Dist: Click

{syncmodels-0.1.331.dist-info → syncmodels-0.1.334.dist-info}/RECORD

@@ -1,6 +1,6 @@
-syncmodels/__init__.py,sha256=ocQlY6lVgFKLeXKNltLiiEFTWqGUZ7-BL8jUbQMZg7c,142
+syncmodels/__init__.py,sha256=79qKhIGH-rtdnic0Qst_kuzypvEYgtYjXJQicUUrO7I,142
 syncmodels/context.py,sha256=k1Gs_ip9BfyRFpyRnzqYvRDKo0sYBqJsh6z9sWln9oE,451
-syncmodels/crawler.py,sha256=_pgelyrIKuVl8vdINJ6NSh5qkSnZf4rAACph4SZ_2H4,95281
+syncmodels/crawler.py,sha256=DAD2tL93iG-LjNQRCg2h_snusjGhQyhdZyMPOCEyvNU,94499
 syncmodels/crud.py,sha256=oZIcwEKR2i-lesEF_059Y4yThohd9m7gs6R6xYgLH-I,15351
 syncmodels/definitions.py,sha256=w-3TrSomp9T8OzLmJhKeZQDzrUIJLKldyh1lzlE7Yj0,5476
 syncmodels/exceptions.py,sha256=ZLAwu19cs2UN2Sv3jaLnixT_jRI7T42TfyutCkUsuIk,685
@@ -11,7 +11,7 @@ syncmodels/registry.py,sha256=YaQtgbSwa0je1MpCcVHALI3_b85vrddyOlhsnrUcKZs,8224
 syncmodels/requests.py,sha256=wWoC5hPDm1iBM_zrlyKRauzhXgdKR3pT5RqyC-5UZhQ,538
 syncmodels/runner.py,sha256=IHDKuQ3yJ1DN9wktMiIrerPepYX61tc3AzbFfuUqEFw,5454
 syncmodels/schema.py,sha256=uinUt8Asq_x7xa6MKWVXNyoWO6gKocjGPppjimaXzEU,2492
-syncmodels/storage.py,sha256=zjKnwgiuG4Vt4nm4n74sngBo6kLkki3sCK0U9tXISHY,74739
+syncmodels/storage.py,sha256=HIoh_KSvalgTzBbb5py_0dUBHHipmHWqxmVu1hcEV4s,74991
 syncmodels/syncmodels.py,sha256=jcUxVbv1hrx5hI81VCO1onIM6WyORTqJVPwIqlPocOc,10596
 syncmodels/timequeue.py,sha256=YRd3ULRaIhoszaBsYhfr0epMqAbL6-NwVEtScjUYttM,595
 syncmodels/wave.py,sha256=Gra22BLiA9z2nF-6diXpjAc4GZv9nebmyvHxdAfXec4,7764
@@ -33,7 +33,7 @@ syncmodels/helpers/importers.py,sha256=KImR9pQu4ir6EI6Ipta0q3RWloFT_VTJi67kM0lZs
 syncmodels/helpers/loaders.py,sha256=aus0aRcbU1vVa_zWo42aX6uV3B0fQ0aQpkTWlR9xGLA,4325
 syncmodels/helpers/models.py,sha256=c_ATzmiw5mVY1IGnwmyhjIuu5d2idHU-XeRigZSMkOQ,719
 syncmodels/helpers/orion.py,sha256=6lRp1w3yaq_rxOI7nJIjuHdsgBjQu92y0bW0IX_gq44,30719
-syncmodels/helpers/surreal.py,sha256=zoWtGm5oAxwvgJNq_NTpKOHN3h9FNObhFDLuiBOl1YY,10050
+syncmodels/helpers/surreal.py,sha256=lHXvm5oNvpDBXFnIksTNV4c3PWp35myw3TiQpP2dqN0,10855
 syncmodels/helpers/units.py,sha256=g50m5DQrAyP_qpDRa4LCEA5Rz2UZUmlIixfWG_ddw9I,3571
 syncmodels/logic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 syncmodels/logic/activity_logger.py,sha256=8wjvgRwaNbibYWGgl-trovSS70yNkoCTlb-AIx3aZEE,14053
@@ -303,10 +303,10 @@ syncmodels/session/postgresql.py,sha256=ZMIu1Rv93pKfvFlovFBmWArzlrT2xaQWNYGZT_LW
 syncmodels/session/sql.py,sha256=17C8EHn_1twHezhMlD5esMvx4m0iIrnD7JK-TuBswgU,6573
 syncmodels/session/sqlite.py,sha256=nCDjopLiBpX1F10qkKoARM7JrVdIpJ1WdGOduFVxaiA,2080
 syncmodels/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-syncmodels-0.1.331.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
-syncmodels-0.1.331.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
-syncmodels-0.1.331.dist-info/METADATA,sha256=gcR0Vk04bekJbu7pwkLBWpeUEvvtmqQuUerKgm0JOwg,2700
-syncmodels-0.1.331.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
-syncmodels-0.1.331.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
-syncmodels-0.1.331.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
-syncmodels-0.1.331.dist-info/RECORD,,
+syncmodels-0.1.334.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
+syncmodels-0.1.334.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
+syncmodels-0.1.334.dist-info/METADATA,sha256=tMGMIT9CJEpVOlBmUqV7MCE_Z-myAFJo918gpJdV4KQ,2700
+syncmodels-0.1.334.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
+syncmodels-0.1.334.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
+syncmodels-0.1.334.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
+syncmodels-0.1.334.dist-info/RECORD,,