syncmodels 0.1.330__py2.py3-none-any.whl → 0.1.333__py2.py3-none-any.whl

syncmodels/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Asterio Gonzalez"""
 __email__ = "asterio.gonzalez@gmail.com"
-__version__ = "0.1.330"
+__version__ = "0.1.333"

syncmodels/crawler.py

@@ -851,7 +851,7 @@ class iBot(iAgent):
         self.add_plugin(MetaExtractPlugin())
         # self.add_plugin(PaginatorPlugin())
 
-        self.add_plugin(PutPlugin())
+        self.add_plugin(PutPlugin())  # comment this for dry-run
 
         self.add_plugin(SetURIPlugin())
         self.add_plugin(Cleaner())
@@ -1201,7 +1201,6 @@ class iBot(iAgent):
         # TODO: I need to clean really old cache entries
         # TODO: in order to control any excesive memory compsumption
         try:
-
             if not self._is_already(timeout=timeout, type_=type_, **params):
                 blueprint = self.blueprint(**params)
                 universe = self.ALREADY.setdefault(type_, {})
@@ -1610,34 +1609,14 @@ class SortPlugin(iPlugin):
             # )
             reverse = list(set(sort_key).intersection(item_fields))
             if not reverse:
-                log.error("model [%s] has not datetime alike key??", model)
-                return
+                log.debug(
+                    "model [%s] attributes:[%s] doesn't match any datetime alike sort keys: [%s]",
+                    model,
+                    list(item_fields),
+                    sort_key,
+                )
+                # return
 
-            if False:
-                sample = stream[-1]
-                reverse = []
-                for key in sort_key:
-                    value = sample[key]
-                    if func := TYPES_MAP.get(value.__class__):
-                        candidates = {
-                            k: v
-                            for k, v in sample.items()
-                            if k != key and func(sample[k]) == func(value)
-                        }
-                        if candidates:
-                            _key, _ = candidates.popitem()
-                            reverse.append(_key)
-                        else:
-                            log.debug(
-                                "can't find the reverse key of '%s' in %s",
-                                sort_key,
-                                sample,
-                            )
-                            # # using same sort_key because source and target
-                            # # layouts share the same sorteable key (i.e. 'datetime')??
-                            # reverse.append(key)
-                    else:
-                        pass
             context[REVERSE_SORT_KEY] = reverse
 
         if sort_key:
@@ -1656,8 +1635,13 @@ class SortPlugin(iPlugin):
             item_fields = model.model_fields
             reverse = list(set(sort_key).intersection(item_fields))
             if not reverse:
-                log.error("model [%s] has not datetime alike key??", model)
-                return
+                log.debug(
+                    "model [%s] attributes:[%s] doesn't match any datetime alike sort keys: [%s]",
+                    model,
+                    list(item_fields),
+                    sort_key,
+                )
+                # return
 
             context[REVERSE_SORT_KEY] = reverse
 
@@ -1714,7 +1698,7 @@ class HashStreamPlugin(iPlugin):
         blueprint = blueprint.hexdigest()
         context["stream_blueprint"] = blueprint
         if not self.bot._set_already(
-            timeout=3600,
+            # timeout=3600,
             type_="response",
             blueprint=blueprint,
         ):

syncmodels/helpers/surreal.py

@@ -120,7 +120,8 @@ class SurrealServer:
             "root",
             "--pass",
             "root",
-            f"file://{self.path}",
+            # f"file://{self.url}",
+            f"rocksdb://{self.url}",
         ]
 
     def start(self):

syncmodels/session/sql.py

@@ -111,7 +111,7 @@ class iSQLSession(iSession):
 
         since_key = params.get(MONOTONIC_SINCE_KEY)
         table = context[KIND_KEY]
-        limit = params.get(LIMIT_KEY_VALUE, context.get(LIMIT_KEY_VALUE)) or 1024
+        limit = params.get(LIMIT_KEY_VALUE, context.get(LIMIT_KEY_VALUE)) or 1024 * 8
 
         query = f"SELECT * FROM {table}"
         if MONOTONIC_SINCE_VALUE in params:

syncmodels/storage.py

@@ -1183,8 +1183,13 @@ class WaveStorage(iWaves, iStorage):
                 if must_check:
                     t0 = time.time()
                     await prevously_inserted()
-                    elapsed = time.time() - t0
-                    log.info("[%s] prevously_inserted took: %s secs", uid, elapsed)
+                    if False or random.random() < 0.1:
+                        elapsed = time.time() - t0
+                        log.info("[%s] prevously_inserted took: %s secs", uid, elapsed)
+                        if elapsed > 1.0:
+                            # TODO: debug what's going here
+                            foo = 1
+
                 else:
                     # hack for not altering the data
                     # push = False
@@ -1202,6 +1207,7 @@ class WaveStorage(iWaves, iStorage):
 
                 data[ID_KEY] = "{thing}:{id}".format_map(_uri)
 
+                res0 = res1 = res2 = False
                 # must push the data?
                 context[PUSHED] = push
                 if push:
@@ -1212,98 +1218,93 @@ class WaveStorage(iWaves, iStorage):
                             data_sort_blueprint,
                             data,
                         )
-                        foo = 1
                     query = f"{namespace}://{database}/{thing}:{monotonic}"
                     res2 = await self.storage.put(query, data)
                 else:
                     # TODO: agp: refactor all this function when we've time!
                     res2 = True
-                    foo = 1
 
-                # long fquid version
-                # data[ID_KEY] = data[ORG_KEY]
-                # short version
-                # data[ID_KEY] = "{thing}:{id}".format_map(_uri)
+                # save Snapshot of the object
+                if push:
+                    # long fquid version
+                    # data[ID_KEY] = data[ORG_KEY]
+                    # short version
+                    # data[ID_KEY] = "{thing}:{id}".format_map(_uri)
+                    if _uri["id"] is None:
+                        __id = data.get(ORG_KEY, "")
+                        _id = parse_duri(__id)
+                        if _id["id"] is None:
+                            _uri["id"] = __id
+                        else:
+                            _uri["id"] = _id["id"]
+                        data[ORG_KEY] = build_uri(**_uri)
+
+                    data[ID_KEY] = "{thing}:{id}".format_map(_uri)
+                    query = f"{namespace}://{database}/{TUBE_SNAPSHOT}"
+                    # resuming_info = {k: kw[k] for k in kw.get(REVERSE_SORT_KEY, [])}
+                    # resuming_info = {k: kw[k] for k in kw.get(SORT_KEY) or []}
+                    resuming_info = {
+                        k: kw.get(k, data.get(k)) for k in kw.get(SORT_KEY) or []
+                    }
+                    # force to be json compatible
+                    resuming_info = JSONVerter.to_json(resuming_info)
 
-                if _uri["id"] is None:
-                    __id = data.get(ORG_KEY, "")
-                    _id = parse_duri(__id)
-                    if _id["id"] is None:
-                        _uri["id"] = __id
+                    res0 = await self.storage.put(
+                        query,
+                        data,
+                    )
+                    # 3. finally add the wave info into tube
+                    data.pop(MONOTONIC_KEY)
+
+                # update the TUBE_WAVE due the insertion of this object
+                if push:
+                    # 2. save last Wave from this particular tube
+                    # AVOID_KEYS contains all keys that aren't json serializable
+                    # wave = {
+                    #     k: v for k, v in kw.items() if k not in self.AVOID_KEYS
+                    # }
+                    # try to recover the 'intact' bootstrap that we've using
+                    for wave0 in kw.get(WAVE_LAST_KEY, []):
+                        wave = wave0.get("wave")  # TODO: use a define
+                        if wave:
+                            break
                     else:
-                        _uri["id"] = _id["id"]
-                    data[ORG_KEY] = build_uri(**_uri)
+                        # otherwise, its the 1st time and we need to create the 1st
+                        # bootstrap-wave info
+                        wave_keys = set(kw.get(WAVE_INFO_KEY, []))
+                        wave_keys.update([KIND_KEY, PREFIX_URL, PARAMS_KEY])
+                        # task = kw[TASK_KEY]
+                        wave = {k: kw[k] for k in wave_keys.intersection(kw)}
 
-                data[ID_KEY] = "{thing}:{id}".format_map(_uri)
-                query = f"{namespace}://{database}/{TUBE_SNAPSHOT}"
-                # resuming_info = {k: kw[k] for k in kw.get(REVERSE_SORT_KEY, [])}
-                # resuming_info = {k: kw[k] for k in kw.get(SORT_KEY) or []}
-                resuming_info = {
-                    k: kw.get(k, data.get(k)) for k in kw.get(SORT_KEY) or []
-                }
-                # force to be json compatible
-                resuming_info = JSONVerter.to_json(resuming_info)
-
-                res0 = await self.storage.put(
-                    query,
-                    data,
-                )
-                # 3. finally add the wave info into tube
-                data.pop(MONOTONIC_KEY)
-
-                # 2. save last Wave from this particular tube
-                # AVOID_KEYS contains all keys that aren't json serializable
-                # wave = {
-                #     k: v for k, v in kw.items() if k not in self.AVOID_KEYS
-                # }
-
-                # try to recover the 'intact' bootstrap that we've using
-                for wave0 in kw.get(WAVE_LAST_KEY, []):
-                    wave = wave0.get("wave")  # TODO: use a define
                     if wave:
-                        break
-                else:
-                    # otherwise, its the 1st time and we need to create the 1st
-                    # bootstrap-wave info
-
-                    # TODO: review for inclussion
-                    # TODO: this settings are crawler activity oriented
-                    # TODO: what about 'Task' an other use cases?
-                    wave_keys = set(kw.get(WAVE_INFO_KEY, []))
-                    wave_keys.update([KIND_KEY, PREFIX_URL, PARAMS_KEY])
-
-                    # task = kw[TASK_KEY]
-                    wave = {k: kw[k] for k in wave_keys.intersection(kw)}
-
-                if wave:
-                    # wave must be json compatible and do not use any reserved
-                    # keyword for storage (i.e. 'scope' in Surreal)
-                    query = f"{namespace}://{database}/{TUBE_WAVE}"
-
-                    # query can't containg MONOTONIC_KEY
-                    wave.pop(MONOTONIC_KEY, None)
-                    wave.pop(WAVE_RESUMING_INFO_KEY, None)
-                    exists = await self.storage.query(query, **wave)
-                    assert len(exists) <= 1
-
-                    if len(exists):
-                        # use the same record_id
-                        # otherwise a new record will be created
-                        wave = exists[0]
-                    # stamp current wave
-                    _wave = {
-                        **wave,
-                        MONOTONIC_KEY: monotonic,
-                        WAVE_RESUMING_INFO_KEY: resuming_info,
-                    }
-                    res1 = await self.storage.put(query, _wave)
-                else:
-                    # wave is empty, maybe because is not a resuming crawling task
-                    log.debug(
-                        "wave is empty, maybe because is not a resuming crawling task"
-                    )
-                    # log.info("Saving: %s", data)
-                    res1 = True
+                        # wave must be json compatible and do not use any reserved
+                        # keyword for storage (i.e. 'scope' in Surreal)
+                        query = f"{namespace}://{database}/{TUBE_WAVE}"
+
+                        # query can't containg MONOTONIC_KEY
+                        wave.pop(MONOTONIC_KEY, None)
+                        wave.pop(WAVE_RESUMING_INFO_KEY, None)
+                        exists = await self.storage.query(query, **wave)
+                        assert len(exists) <= 1
+
+                        if len(exists):
+                            # use the same record_id
+                            # otherwise a new record will be created
+                            wave = exists[0]
+                        # stamp current wave
+                        _wave = {
+                            **wave,
+                            MONOTONIC_KEY: monotonic,
+                            WAVE_RESUMING_INFO_KEY: resuming_info,
+                        }
+                        res1 = await self.storage.put(query, _wave)
+                    else:
+                        # wave is empty, maybe because is not a resuming crawling task
+                        log.debug(
+                            "wave is empty, maybe because is not a resuming crawling task"
+                        )
+                        # log.info("Saving: %s", data)
+                        res1 = True
 
                 return all([res0, res1, res2])
             else:

{syncmodels-0.1.330.dist-info → syncmodels-0.1.333.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: syncmodels
-Version: 0.1.330
+Version: 0.1.333
 Summary: Synchronizable Models
 Home-page: https://github.com/asterio.gonzalez/syncmodels
 Author: Asterio Gonzalez
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.6
 License-File: LICENSE
 License-File: AUTHORS.rst
-Requires-Dist: agptools>=0.1.330
+Requires-Dist: agptools>=0.1.333
 Requires-Dist: aiocache
 Requires-Dist: aiohttp
 Requires-Dist: Click

{syncmodels-0.1.330.dist-info → syncmodels-0.1.333.dist-info}/RECORD

@@ -1,6 +1,6 @@
-syncmodels/__init__.py,sha256=g59bA8lWKxQA9npRZE5WiXLHT-QTCR2ggetQwG4iExQ,142
+syncmodels/__init__.py,sha256=heMvuebb5E-VxcjkidHaEr6oN5d2fTQ9wqrWcqxCzBE,142
 syncmodels/context.py,sha256=k1Gs_ip9BfyRFpyRnzqYvRDKo0sYBqJsh6z9sWln9oE,451
-syncmodels/crawler.py,sha256=_pgelyrIKuVl8vdINJ6NSh5qkSnZf4rAACph4SZ_2H4,95281
+syncmodels/crawler.py,sha256=sjyuLLBT5RGb-3LVTtjW9nd2PTJyIV5Tldoz8Jq3dVs,94511
 syncmodels/crud.py,sha256=oZIcwEKR2i-lesEF_059Y4yThohd9m7gs6R6xYgLH-I,15351
 syncmodels/definitions.py,sha256=w-3TrSomp9T8OzLmJhKeZQDzrUIJLKldyh1lzlE7Yj0,5476
 syncmodels/exceptions.py,sha256=ZLAwu19cs2UN2Sv3jaLnixT_jRI7T42TfyutCkUsuIk,685
@@ -11,7 +11,7 @@ syncmodels/registry.py,sha256=YaQtgbSwa0je1MpCcVHALI3_b85vrddyOlhsnrUcKZs,8224
 syncmodels/requests.py,sha256=wWoC5hPDm1iBM_zrlyKRauzhXgdKR3pT5RqyC-5UZhQ,538
 syncmodels/runner.py,sha256=IHDKuQ3yJ1DN9wktMiIrerPepYX61tc3AzbFfuUqEFw,5454
 syncmodels/schema.py,sha256=uinUt8Asq_x7xa6MKWVXNyoWO6gKocjGPppjimaXzEU,2492
-syncmodels/storage.py,sha256=tnF74q94wCidUo4dW77RWLVSKzJRv5ZGKFsNfoxWOkg,74546
+syncmodels/storage.py,sha256=HIoh_KSvalgTzBbb5py_0dUBHHipmHWqxmVu1hcEV4s,74991
 syncmodels/syncmodels.py,sha256=jcUxVbv1hrx5hI81VCO1onIM6WyORTqJVPwIqlPocOc,10596
 syncmodels/timequeue.py,sha256=YRd3ULRaIhoszaBsYhfr0epMqAbL6-NwVEtScjUYttM,595
 syncmodels/wave.py,sha256=Gra22BLiA9z2nF-6diXpjAc4GZv9nebmyvHxdAfXec4,7764
@@ -33,7 +33,7 @@ syncmodels/helpers/importers.py,sha256=KImR9pQu4ir6EI6Ipta0q3RWloFT_VTJi67kM0lZs
 syncmodels/helpers/loaders.py,sha256=aus0aRcbU1vVa_zWo42aX6uV3B0fQ0aQpkTWlR9xGLA,4325
 syncmodels/helpers/models.py,sha256=c_ATzmiw5mVY1IGnwmyhjIuu5d2idHU-XeRigZSMkOQ,719
 syncmodels/helpers/orion.py,sha256=6lRp1w3yaq_rxOI7nJIjuHdsgBjQu92y0bW0IX_gq44,30719
-syncmodels/helpers/surreal.py,sha256=zoWtGm5oAxwvgJNq_NTpKOHN3h9FNObhFDLuiBOl1YY,10050
+syncmodels/helpers/surreal.py,sha256=4bBBj__kQta_5ShWzUKlsuq6xSmICP5UwpeA6QTD-rA,10088
 syncmodels/helpers/units.py,sha256=g50m5DQrAyP_qpDRa4LCEA5Rz2UZUmlIixfWG_ddw9I,3571
 syncmodels/logic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 syncmodels/logic/activity_logger.py,sha256=8wjvgRwaNbibYWGgl-trovSS70yNkoCTlb-AIx3aZEE,14053
@@ -300,13 +300,13 @@ syncmodels/model/schema_org/xpathtype.py,sha256=D8gKiCrGSSuUVYw7BIWmOIUbKATfv2Ip
 syncmodels/session/__init__.py,sha256=NxFkOiL_oGaYt2qv9yAvWrNcXn_xT9yLzCLd7PGRaWI,15564
 syncmodels/session/http.py,sha256=tf7z0ccAEYoCOZT4Ukv3NBXz9hUO3vs2s9bm491pCj8,1480
 syncmodels/session/postgresql.py,sha256=ZMIu1Rv93pKfvFlovFBmWArzlrT2xaQWNYGZT_LW61k,175
-syncmodels/session/sql.py,sha256=bD7zXRrEKKJmqY2UoibWENuWb5zHrrU72F3_dYbS6LY,6569
+syncmodels/session/sql.py,sha256=17C8EHn_1twHezhMlD5esMvx4m0iIrnD7JK-TuBswgU,6573
 syncmodels/session/sqlite.py,sha256=nCDjopLiBpX1F10qkKoARM7JrVdIpJ1WdGOduFVxaiA,2080
 syncmodels/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-syncmodels-0.1.330.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
-syncmodels-0.1.330.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
-syncmodels-0.1.330.dist-info/METADATA,sha256=tNxb2OSD2M4XsGc7gAC8FQRkCOcnZdoHFtpRESCA9tk,2700
-syncmodels-0.1.330.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
-syncmodels-0.1.330.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
-syncmodels-0.1.330.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
-syncmodels-0.1.330.dist-info/RECORD,,
+syncmodels-0.1.333.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
+syncmodels-0.1.333.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
+syncmodels-0.1.333.dist-info/METADATA,sha256=KChKUTfP3d_Je0m3czd5SC3iTebjP1loPJn9VHXmoKE,2700
+syncmodels-0.1.333.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
+syncmodels-0.1.333.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
+syncmodels-0.1.333.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
+syncmodels-0.1.333.dist-info/RECORD,,