syncmodels 0.1.322__py2.py3-none-any.whl → 0.1.324__py2.py3-none-any.whl

syncmodels/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Asterio Gonzalez"""
 __email__ = "asterio.gonzalez@gmail.com"
-__version__ = "0.1.322"
+__version__ = "0.1.324"

syncmodels/crawler.py

@@ -763,7 +763,7 @@ class iBot(iAgent):
         # TODO: agp: generalize and provide a better way to save activity records
 
         notify = 0
-        t0 = time.time()
+        # t0 = time.time()
         N = len(self.plugins)
         for i, plugin in enumerate(self.plugins):
             t1 = time.time()
@@ -779,7 +779,7 @@ class iBot(iAgent):
             except Exception as why:
                 log.error(why)
                 log.error("".join(traceback.format_exception(*sys.exc_info())))
-                foo = 1
+                # foo = 1
             elapsed = time.time() - t1
             notify -= elapsed
             if notify <= 0:
@@ -1927,6 +1927,7 @@ class MetaExtractPlugin(iPlugin):
     """
 
     SCORE = 1000  # before NormalizePlugin
+    USED_GEO = set()
 
     def __init__(self, geojson=True, **kw):
         super().__init__(**kw)
@@ -1950,6 +1951,9 @@ class MetaExtractPlugin(iPlugin):
                 # of the value of the item data
                 # this will provent mistakes of assuming that each geojson
                 # relate to the same fquid changs within the stream live
+
+                # Note: we use this geoid to check if different geo objects are different
+                # Note: can be commended  in the future
                 geoids = dict()
 
                 # -------------------------------------------------
@@ -1974,35 +1978,40 @@ class MetaExtractPlugin(iPlugin):
                 # maybe in the future for some EndPoints
                 # that we trust, this restriction can be relaxed
                 if fquid:  #  fquid not in geoids:
-                    # TODO: use a new dict (item) and kind
-                    # TODO: or share same dict (data) and provide multiples { id, kind }
-                    # TODO: letting pydantic takes only the data needed for each kind/mapper
-
-                    # Note: we `stich` all items belongs to the same wave
-                    # Note: with the same monotonic value
-                    item = {
-                        **data,
-                        "id": fquid,
-                        KIND_KEY: kind,
-                        # MONOTONIC_KEY: monotonic, # not needed (data has)
-                    }
+                    # discard geo info that has been already saved in Storage
+                    blueprint = hashlib.md5(fquid.encode("utf-8")).digest()
+                    if blueprint not in self.USED_GEO:
+                        # TODO: use a new dict (item) and kind
+                        # TODO: or share same dict (data) and provide multiples { id, kind }
+                        # TODO: letting pydantic takes only the data needed for each kind/mapper
+
+                        # Note: we `stich` all items belongs to the same wave
+                        # Note: with the same monotonic value
+                        item = {
+                            **data,
+                            "id": fquid,
+                            KIND_KEY: kind,
+                            # MONOTONIC_KEY: monotonic, # not needed (data has)
+                        }
+
+                        # add the related geojson item
+
+                        holder = geoids.setdefault(fquid, [])
+                        if not holder:
+                            # only add
+                            # stream.insert(0, item)
+                            stream.append(item)
+
+                        # don't trust the EndPoint and check geojson
+                        # related to the same fquid
+                        _new = item[GEOJSON_KEY]
+                        for _existing in holder:
+                            assert (
+                                _new == _existing
+                            ), f"GeoJSON related to {fquid} doesn't match, you need to review fqui and geojson association"
+                        holder.append(_new)
 
-                    # add the related geojson item
-
-                    holder = geoids.setdefault(fquid, [])
-                    if not holder:
-                        # only add
-                        # stream.insert(0, item)
-                        stream.append(item)
-
-                    # don't trust the EndPoint and check geojson
-                    # related to the same fquid
-                    _new = item[GEOJSON_KEY]
-                    for _existing in holder:
-                        assert (
-                            _new == _existing
-                        ), f"GeoJSON related to {fquid} doesn't match, you need to review fqui and geojson association"
-                    holder.append(_new)
+                        self.USED_GEO.add(blueprint)
                 else:
                     log.error("can't build geo_uri from [%s]", data)
                     log.error(

syncmodels/crud.py

@@ -261,7 +261,7 @@ class iStorage(iCRUD):
                 f"/{_uri['_path']}"  # force remove "id" and give back compability
             )
             table = build_uri(**_uri)
-            self._index_2_create[table] = keys
+            self._index_2_create.setdefault(table, set()).update(keys)
 
     async def build_indexes(self):
         "build the registered indexes"

syncmodels/session/sql.py

@@ -111,7 +111,7 @@ class iSQLSession(iSession):
 
         since_key = params.get(MONOTONIC_SINCE_KEY)
         table = context[KIND_KEY]
-        limit = params.get(LIMIT_KEY_VALUE, context.get(LIMIT_KEY_VALUE)) or 10240
+        limit = params.get(LIMIT_KEY_VALUE, context.get(LIMIT_KEY_VALUE)) or 42
 
         query = f"SELECT * FROM {table}"
         if MONOTONIC_SINCE_VALUE in params:

syncmodels/storage.py

@@ -980,6 +980,9 @@ class WaveStorage(iWaves, iStorage):
                         ORG_KEY: uid,
                         **data_sort_blueprint,  # implies sv = True
                     }
+                    # requests index creation
+                    self.register_index(uid, [ORG_KEY])
+
                     identical = await self.storage.query(
                         query,
                         **identical_bp,
@@ -1182,7 +1185,7 @@ class WaveStorage(iWaves, iStorage):
                     log.info("[%s] prevously_inserted took: %s secs", uid, elapsed)
                 else:
                     # hack for not altering the data
-                    # push = False
+                    push = False
                     pass
 
                 # check if ORG_KEY could be formated wrong
@@ -1650,11 +1653,11 @@ class SurrealistStorage(Storage):
         sufix = "_".join(columns)
         keynames = ", ".join(columns)
         # TODO: agp: comment these lines next week :)
-        name = f"index_{table}_{sufix}"
-        sql = f"REMOVE INDEX IF EXISTS {name} ON TABLE {table}"
-        log.info("deleting index: [%s] on [%s]", name, keynames)
-        res = connection.query(sql)
-        log.info("result: [%s]", res)
+        # name = f"index_{table}_{sufix}"
+        # sql = f"REMOVE INDEX IF EXISTS {name} ON TABLE {table}"
+        # log.info("deleting index: [%s] on [%s]", name, keynames)
+        # res = connection.query(sql)
+        # log.info("result: [%s]", res)
 
         name = f"idx_{table}_{sufix}"
         sql = f"DEFINE INDEX IF NOT EXISTS {name} ON TABLE {table} COLUMNS {keynames}"

{syncmodels-0.1.322.dist-info → syncmodels-0.1.324.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: syncmodels
-Version: 0.1.322
+Version: 0.1.324
 Summary: Synchronizable Models
 Home-page: https://github.com/asterio.gonzalez/syncmodels
 Author: Asterio Gonzalez
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.6
 License-File: LICENSE
 License-File: AUTHORS.rst
-Requires-Dist: agptools>=0.1.322
+Requires-Dist: agptools>=0.1.324
 Requires-Dist: aiocache
 Requires-Dist: aiohttp
 Requires-Dist: Click

{syncmodels-0.1.322.dist-info → syncmodels-0.1.324.dist-info}/RECORD

@@ -1,7 +1,7 @@
-syncmodels/__init__.py,sha256=fBTsP1kMfqOtZ5igUFKisp7ErzBjDkZxECgekm_4nbg,142
+syncmodels/__init__.py,sha256=afw4_wa5FaFcdZ_w4hP1EVGiGoXhSRbq4dIM-6aVcP0,142
 syncmodels/context.py,sha256=k1Gs_ip9BfyRFpyRnzqYvRDKo0sYBqJsh6z9sWln9oE,451
-syncmodels/crawler.py,sha256=FthNzipF8cafNvogFC7AFn_N4qxOHkJTeo3Lbcw0P_g,94746
-syncmodels/crud.py,sha256=ozumS7XgmXSFcFN2SZBH0jB0j_1vK2xE-FeFcTG7ikw,15327
+syncmodels/crawler.py,sha256=_pgelyrIKuVl8vdINJ6NSh5qkSnZf4rAACph4SZ_2H4,95281
+syncmodels/crud.py,sha256=oZIcwEKR2i-lesEF_059Y4yThohd9m7gs6R6xYgLH-I,15351
 syncmodels/definitions.py,sha256=vQ6-Zsftzy5y02z6Dd3_p5cd37Zqk0lcVrv-06gnDZk,5475
 syncmodels/exceptions.py,sha256=ZLAwu19cs2UN2Sv3jaLnixT_jRI7T42TfyutCkUsuIk,685
 syncmodels/geofactory.py,sha256=1FkrdEn0QA0O4_lSUAwjqXH2dmlQWi32AkntnG4AEQY,10372
@@ -11,7 +11,7 @@ syncmodels/registry.py,sha256=YaQtgbSwa0je1MpCcVHALI3_b85vrddyOlhsnrUcKZs,8224
 syncmodels/requests.py,sha256=wWoC5hPDm1iBM_zrlyKRauzhXgdKR3pT5RqyC-5UZhQ,538
 syncmodels/runner.py,sha256=IHDKuQ3yJ1DN9wktMiIrerPepYX61tc3AzbFfuUqEFw,5454
 syncmodels/schema.py,sha256=uinUt8Asq_x7xa6MKWVXNyoWO6gKocjGPppjimaXzEU,2492
-syncmodels/storage.py,sha256=FuKkTlakwp_kjVoB99Cye5QNt6eDlYLgZMkDZrpGz_Y,73794
+syncmodels/storage.py,sha256=Q5P0Kuu0vPLJbgGMyfst_6t9k9lJkGtAy0blIbGDtLM,73905
 syncmodels/syncmodels.py,sha256=jcUxVbv1hrx5hI81VCO1onIM6WyORTqJVPwIqlPocOc,10596
 syncmodels/timequeue.py,sha256=YRd3ULRaIhoszaBsYhfr0epMqAbL6-NwVEtScjUYttM,595
 syncmodels/wave.py,sha256=Gra22BLiA9z2nF-6diXpjAc4GZv9nebmyvHxdAfXec4,7764
@@ -299,13 +299,13 @@ syncmodels/model/schema_org/xpathtype.py,sha256=D8gKiCrGSSuUVYw7BIWmOIUbKATfv2Ip
 syncmodels/session/__init__.py,sha256=NxFkOiL_oGaYt2qv9yAvWrNcXn_xT9yLzCLd7PGRaWI,15564
 syncmodels/session/http.py,sha256=tf7z0ccAEYoCOZT4Ukv3NBXz9hUO3vs2s9bm491pCj8,1480
 syncmodels/session/postgresql.py,sha256=ZMIu1Rv93pKfvFlovFBmWArzlrT2xaQWNYGZT_LW61k,175
-syncmodels/session/sql.py,sha256=7FbiKpnalFWkG9AEMDPloJ_IByyViwOR4EZRi1f7ulw,6570
+syncmodels/session/sql.py,sha256=4501Prer2Fz0e4goP98tNXMg7oD36B3X91l4tf2waFU,6567
 syncmodels/session/sqlite.py,sha256=nCDjopLiBpX1F10qkKoARM7JrVdIpJ1WdGOduFVxaiA,2080
 syncmodels/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-syncmodels-0.1.322.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
-syncmodels-0.1.322.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
-syncmodels-0.1.322.dist-info/METADATA,sha256=gBvtU5H0_xVQrpiJS5ofb2eZajGlwh1YKeTGArMyoDw,2700
-syncmodels-0.1.322.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
-syncmodels-0.1.322.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
-syncmodels-0.1.322.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
-syncmodels-0.1.322.dist-info/RECORD,,
+syncmodels-0.1.324.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
+syncmodels-0.1.324.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
+syncmodels-0.1.324.dist-info/METADATA,sha256=zvOOR5KpvS238h1j2Hs-NAWoFzmHBno2Gp7xUM2n9eI,2700
+syncmodels-0.1.324.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
+syncmodels-0.1.324.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
+syncmodels-0.1.324.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
+syncmodels-0.1.324.dist-info/RECORD,,