syncmodels 0.1.314__py2.py3-none-any.whl → 0.1.316__py2.py3-none-any.whl

syncmodels/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Asterio Gonzalez"""
 __email__ = "asterio.gonzalez@gmail.com"
-__version__ = "0.1.314"
+__version__ = "0.1.316"

syncmodels/crawler.py

@@ -283,9 +283,12 @@ class iAgent(iRunner):
             _tasks = [_tasks]
 
         tasks = []
+        default_tasks = self.default_bootstrap()
+
         for _ in _tasks:
             if _ is None:
-                tasks.extend(self.default_bootstrap())
+                tasks.extend(default_tasks)
+                default_tasks.clear()  # just once
             else:
                 tasks.append(_)
 
@@ -1502,6 +1505,7 @@ class NormalizePlugin(iPlugin):
     SCORE = 950  # before SetURIPlugin
 
     async def handle(self, stream: List[Dict], context: Dict):
+        stream[:] = stream[: iPlugin.MAX_RECORDS]  # debug code
 
         for idx, data in enumerate(stream):
 
@@ -1701,6 +1705,8 @@ class HashStreamPlugin(iPlugin):
     SCORE = 50  # before SortPlugin
 
     async def handle(self, stream: List[Dict], context: Dict):
+        stream[:] = stream[: iPlugin.MAX_RECORDS]  # debug code
+
         blueprint = hashlib.sha1(b"")
         for data in stream:
             new = hashlib.sha1(pickle.dumps(data))
@@ -2648,13 +2654,16 @@ class iAsyncCrawler(iCrawler):
         else:
             item, model = data, None
 
+        result = False
         if item:
             # assert isinstance(item, BaseModel)
             # result = await self.syncmodel.put(item)
             context.update(data)
             context[MODEL_KEY] = model
             context["item"] = item
-            result = all([await sync.put(**context) for sync in self.syncmodel])
+            result = all(
+                [await sync.put(context=context, **context) for sync in self.syncmodel]
+            )
             # save original item if a raw storage has been specified
             if self.raw_storage:
                 fqid = item.id
@@ -2667,6 +2676,7 @@ class iAsyncCrawler(iCrawler):
                 await self.save(nice=True)
         else:
             foo = 1
+        return result
 
     async def update_meta(self, tube, meta: Dict) -> bool:
         meta = json_compatible(meta)

syncmodels/crud.py

@@ -175,7 +175,7 @@ class iCRUD:
         "Get an object from URI"
         raise NotImplementedError()
 
-    async def put(self, uri: URI, data: JSON = None, **kw) -> bool:
+    async def put(self, uri: URI, data: JSON = None, context={}, **kw) -> bool:
         "Put an object from URI"
         raise NotImplementedError()
 
@@ -351,7 +351,7 @@ class iStorage(iCRUD):
             log.error(why)
             log.error("".join(traceback.format_exception(*sys.exc_info())))
 
-    async def put(self, uri: URI, data: JSON = None, **kw) -> bool:
+    async def put(self, uri: URI, data: JSON = None, context={}, **kw) -> bool:
         if data is None:
             data = kw
         # else:

syncmodels/definitions.py

@@ -130,6 +130,7 @@ MONOTONIC_SINCE_OPERATOR = "since_operator__"
 DATETIME_KEY = "datetime"
 DATETIME_LAST_KEY = "last"
 ID_KEY = "id"
+PUSHED = "pushed__"
 FQUID_KEY = "fquid"
 ORG_KEY = "id__"
 ORG_URL = "url__"

syncmodels/helpers/crawler.py

@@ -137,7 +137,7 @@ class GeojsonManager:
         "returns the uri of a related object"
         for candidate in [ORG_KEY, ID_KEY]:
             if _sid := data.get(candidate):
-                _sid = parse_duri(_sid)
+                _sid = parse_duri(str(_sid))
                 if _sid["fscheme"] != "test":
 
                     _sid["path"] = "/{thing}/geo".format_map(_sid)

syncmodels/logic/analyzer.py

@@ -250,6 +250,10 @@ class XPathAnalyzer:
         return found > 0
 
     def _analyze_meta(self, line, ctx) -> bool:
+        # TODO: tampodary skip meta info
+        return False
+
+    def _analyze_meta_old(self, line, ctx) -> bool:
         keyword = ctx.get("keyword") or {"name", "itemprop"}
         item = ctx["item"]
         info = ctx["info"]
@@ -292,6 +296,10 @@ class XPathAnalyzer:
         return self._analyze_text(line, ctx)
 
     def _analyze_ldjson(self, line, ctx) -> bool:
+        # TODO: this funcion has been temporary disabled
+        return False
+
+    def _analyze_ldjson_old(self, line, ctx) -> bool:
         # TODO: helper for loading json
         line = line.strip()
         line = html2text(line)

syncmodels/logic/browser.py

@@ -1824,9 +1824,10 @@ class BrowserLogicSession(iBrowserSession):
 
     async def _locate(self, page, selector, ctx):
         elements = []
-        timeout = ctx["info"].get("timeout", 5)
-        num_selectors = len(ctx["info"]["selector"])
-        timeout = 1 + timeout // num_selectors
+        # timeout = ctx["info"].get("timeout", 5)
+        # num_selectors = len(ctx["info"]["selector"])
+        # timeout = 1 + timeout // num_selectors
+        timeout = 1.2
         t1 = time.time() + timeout
         while time.time() < t1:
             try:
@@ -1841,7 +1842,6 @@ class BrowserLogicSession(iBrowserSession):
                 await asyncio.sleep(1)
             except Exception as why:
                 log.error("selector: %s -> %s", selector, why)
-
                 foo = 1
 
     async def _mark_element(self, element, style):

syncmodels/storage.py

@@ -86,6 +86,7 @@ from .definitions import (
     LIMIT_KEY,
     WAVE_LAST_KEY,
     COMPARISON_PATTERNS,
+    PUSHED,
 )
 from .crud import (
     DEFAULT_DATABASE,
@@ -785,7 +786,7 @@ class WaveStorage(iWaves, iStorage):
     async def update(self, query: URI | QUERY, data) -> List[JSON]:
         return await self.storage.update(query, data)
 
-    async def put(self, uri: URI, data: JSON = None, **kw) -> bool:
+    async def put(self, uri: URI, data: JSON = None, context={}, **kw) -> bool:
         """
         Try to insert a new wave object into storage:
 
@@ -797,7 +798,7 @@ class WaveStorage(iWaves, iStorage):
 
         2. if the object has been inserted in `tube` then:
            - create / update Snapshot
-           - create / update Wave info with original the original query data
+           - cteate / update Wave info with original the original query data
 
 
         e:  exists
@@ -905,6 +906,7 @@ class WaveStorage(iWaves, iStorage):
                         since_value = pytz.utc.localize(since_value)
                     since_value = since_value.astimezone(UTC_TZ)
                     since_value = since_value.strftime("%Y-%m-%dT%H:%M:%SZ")
+
                     break
                 else:
                     monotonic_key = MONOTONIC_KEY  # ??
@@ -916,16 +918,16 @@ class WaveStorage(iWaves, iStorage):
 
                 data_sort_blueprint = build_dict(data, sort_keys)
                 # data_sort_blueprint = build_comparisson_dict(data, reverse_sort_keys)
-                blueprint = {
+                data_sort_bp = {
                     MONOTONIC_SINCE_KEY: monotonic_key,
                     MONOTONIC_SINCE_VALUE: since_value,
                     MONOTONIC_SINCE_OPERATOR: ">=",
                     ORDER_KEY: monotonic_key,
                     DIRECTION_KEY: DIRECTION_DESC,
-                    LIMIT_KEY: kw.get(
-                        LIMIT_KEY, 50  # TODO: agp: set in definition?
-                    ),  # TODO: this is temporal, ideally None
-                    ORG_KEY: uid,
+                    # LIMIT_KEY: kw.get(
+                    #     LIMIT_KEY, 50  # TODO: agp: set in definition?
+                    # ),  # TODO: this is temporal, ideally None
+                    # ORG_KEY: uid,
                     # **data_sort_blueprint,  # implies sv = True
                 }
                 # TODO: LIMIT 1 ?
@@ -958,29 +960,32 @@ class WaveStorage(iWaves, iStorage):
                 t0 = time.time()
                 # search the same data
                 # TODO: update blueprint
-                blueprint = {
-                    # MONOTONIC_SINCE_KEY: monotonic_key,
-                    # MONOTONIC_SINCE_VALUE: since_value,
-                    # MONOTONIC_SINCE_OPERATOR: ">=",
-                    # ORDER_KEY: monotonic_key,
-                    # DIRECTION_KEY: DIRECTION_DESC,
+                identical_bp = {
                     LIMIT_KEY: kw.get(
-                        LIMIT_KEY, 50  # TODO: agp: set in definition?
+                        LIMIT_KEY, 10  # TODO: agp: set in definition?
                     ),  # TODO: this is temporal, ideally None
                     ORG_KEY: uid,
                     **data_sort_blueprint,  # implies sv = True
                 }
                 identical = await self.storage.query(
                     query,
-                    **blueprint,
+                    **identical_bp,
                     # **data_sort_bp,
                 )
+
                 # TODO: try to create only a single query
                 # TODO: review different structures case
+                similar_bp = {
+                    LIMIT_KEY: kw.get(
+                        LIMIT_KEY, 50  # TODO: agp: set in definition?
+                    ),  # TODO: this is temporal, ideally None
+                    ORG_KEY: uid,
+                    **data_sort_blueprint,  # implies sv = True
+                }
                 similar = await self.storage.query(
                     query,
-                    **blueprint,
-                    # **data_sort_bp,
+                    **similar_bp,
+                    **data_sort_bp,
                 )
                 t1 = time.time()
                 _elapsed = t1 - t0
@@ -988,7 +993,7 @@ class WaveStorage(iWaves, iStorage):
                 N = len(existing)
                 log.debug(
                     "[%s] found [%s] similar records in %s secs",
-                    blueprint,
+                    identical_bp,
                     N,
                     _elapsed,
                 )
@@ -1165,6 +1170,7 @@ class WaveStorage(iWaves, iStorage):
                 data[ID_KEY] = "{thing}:{id}".format_map(_uri)
 
                 # must push the data?
+                context[PUSHED] = push
                 if push:
                     if isinstance(self.storage, SurrealistStorage):
                         log_records.debug(

syncmodels/syncmodels.py

@@ -255,7 +255,7 @@ class SyncModel(iCRUD):
         # )
         # return []
 
-    async def put(self, item: BaseModel, **kw) -> bool:
+    async def put(self, item: BaseModel, context={}, **kw) -> bool:
         """Try to create / update an item of `type_` class from raw data
 
         - get the pydantic item
@@ -291,7 +291,7 @@ class SyncModel(iCRUD):
             kw["uri"] = fqid
             kw["data"] = data
             for storage in self.storage:
-                result = await storage.put(**kw)
+                result = await storage.put(context=context, **kw)
                 results.append(result)
 
         return all(results)

{syncmodels-0.1.314.dist-info → syncmodels-0.1.316.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: syncmodels
-Version: 0.1.314
+Version: 0.1.316
 Summary: Synchronizable Models
 Home-page: https://github.com/asterio.gonzalez/syncmodels
 Author: Asterio Gonzalez
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.6
 License-File: LICENSE
 License-File: AUTHORS.rst
-Requires-Dist: agptools>=0.1.314
+Requires-Dist: agptools>=0.1.316
 Requires-Dist: aiocache
 Requires-Dist: aiohttp
 Requires-Dist: Click

{syncmodels-0.1.314.dist-info → syncmodels-0.1.316.dist-info}/RECORD

@@ -1,8 +1,8 @@
-syncmodels/__init__.py,sha256=G_3tlPvmtf1CQaauaKV6PoulB_iHJ43bVbH_1q_p8aI,142
+syncmodels/__init__.py,sha256=QPo81OufxMI_nbk_t3PobuPDPY-t3LkredszkWC7OC4,142
 syncmodels/context.py,sha256=k1Gs_ip9BfyRFpyRnzqYvRDKo0sYBqJsh6z9sWln9oE,451
-syncmodels/crawler.py,sha256=U6F0QlzmxkReCB6X2Xbxmm2rRn99GEX005dG7AFl8xE,94430
-syncmodels/crud.py,sha256=viHBwzczcjNyFiLxL7VGYSbWJW5VjU8AvKaPufBMP7M,15303
-syncmodels/definitions.py,sha256=dTMxcwcIeYQthZCZXNQmaO1SolWY-WhafASyl91xTo0,5455
+syncmodels/crawler.py,sha256=L7-3E1Gv27NqusM5Diq-4_DjW4OPCR1X2suWz7d-Hss,94741
+syncmodels/crud.py,sha256=ozumS7XgmXSFcFN2SZBH0jB0j_1vK2xE-FeFcTG7ikw,15327
+syncmodels/definitions.py,sha256=vQ6-Zsftzy5y02z6Dd3_p5cd37Zqk0lcVrv-06gnDZk,5475
 syncmodels/exceptions.py,sha256=ZLAwu19cs2UN2Sv3jaLnixT_jRI7T42TfyutCkUsuIk,685
 syncmodels/geofactory.py,sha256=1FkrdEn0QA0O4_lSUAwjqXH2dmlQWi32AkntnG4AEQY,10372
 syncmodels/http.py,sha256=FFVT3QJJgur2dv1Q_7l9ZsWN8z6_gUjOT9hJff1ZAqk,3335
@@ -11,8 +11,8 @@ syncmodels/registry.py,sha256=YaQtgbSwa0je1MpCcVHALI3_b85vrddyOlhsnrUcKZs,8224
 syncmodels/requests.py,sha256=wWoC5hPDm1iBM_zrlyKRauzhXgdKR3pT5RqyC-5UZhQ,538
 syncmodels/runner.py,sha256=IHDKuQ3yJ1DN9wktMiIrerPepYX61tc3AzbFfuUqEFw,5454
 syncmodels/schema.py,sha256=uinUt8Asq_x7xa6MKWVXNyoWO6gKocjGPppjimaXzEU,2492
-syncmodels/storage.py,sha256=-kdXuPjkHBPauoXTWcmwK4jmH9hr2X1JFERqluznL1Q,71513
-syncmodels/syncmodels.py,sha256=EzSC4C75V4wJDmsLLbp8YUVwqA6A16KCNW8nB-MqPcs,10567
+syncmodels/storage.py,sha256=invDyQMfDHavQH5Fnbi7afSeIgH6NoVtD_LlNPOqSTI,71646
+syncmodels/syncmodels.py,sha256=jcUxVbv1hrx5hI81VCO1onIM6WyORTqJVPwIqlPocOc,10596
 syncmodels/timequeue.py,sha256=YRd3ULRaIhoszaBsYhfr0epMqAbL6-NwVEtScjUYttM,595
 syncmodels/wave.py,sha256=Gra22BLiA9z2nF-6diXpjAc4GZv9nebmyvHxdAfXec4,7764
 syncmodels/auth/__init__.py,sha256=xs9Y_bTR99TKt7NyEtMI3lLPxQy9FuP6CYakgAAthCo,1139
@@ -24,7 +24,7 @@ syncmodels/cli/surreal.py,sha256=eL7pDicLo0_68JhpCZacde6DOVcfHE_UEzczvgDuvAc,106
 syncmodels/cli/wingdbstub.py,sha256=q4z-RqHN1ttzNtiLYTzqQG2ZYZ6W3NOnEd2E5NGhfao,17165
 syncmodels/cli/workspace.py,sha256=wajZnxf567nYoQysTEgxrDAp8ZBU8zSuoP4KyZtqvdc,2461
 syncmodels/helpers/__init__.py,sha256=qZet64gMJNAAqzUdEqCV5WDk5D2Dbw1Kxlt9Jo6x3m4,23
-syncmodels/helpers/crawler.py,sha256=FrYYUzGcbLK1gszscc1rzwl9Y86P4Q1H_-k2Z804CcM,4800
+syncmodels/helpers/crawler.py,sha256=2WiDxnjxZKbGa8x71bgeSc2wmabKa_wD-ofQMbgVXLo,4805
 syncmodels/helpers/explorer.py,sha256=-Dol3z1pALCMI9OPSvVbROaTzLjbUpS0suJ82Z6Rmb4,7447
 syncmodels/helpers/faker.py,sha256=tPtibNh28KoHb9kcwE9WaPdPrSrN6xMbCv5HhNFEVG0,1222
 syncmodels/helpers/general.py,sha256=UAcSfrvsaT15iJuxsR3WMk51UjpLLGDf14xmpBojndg,6160
@@ -37,8 +37,8 @@ syncmodels/helpers/surreal.py,sha256=zoWtGm5oAxwvgJNq_NTpKOHN3h9FNObhFDLuiBOl1YY
 syncmodels/helpers/units.py,sha256=g50m5DQrAyP_qpDRa4LCEA5Rz2UZUmlIixfWG_ddw9I,3571
 syncmodels/logic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 syncmodels/logic/activity_logger.py,sha256=8wjvgRwaNbibYWGgl-trovSS70yNkoCTlb-AIx3aZEE,14053
-syncmodels/logic/analyzer.py,sha256=7JDIvZIiDVgdABaaRk7TEbzOFM6HAH3RaSsIDmL4QVE,12291
-syncmodels/logic/browser.py,sha256=KqvOLPAl2p5EBol7FK-XYvOXeDnu0h9Dz7xfagKTheM,84768
+syncmodels/logic/analyzer.py,sha256=OiRZBJoqjc_qb3w1jBjGftWgd20Cig0s7_GLTCbp3fw,12539
+syncmodels/logic/browser.py,sha256=vMxGaYIR1ov0tOMbWNpd4MMFQBunRuvxE8F-Gfo9cTo,84795
 syncmodels/logic/swarm.py,sha256=eRBVlNAOzzWKFGCb7LGLx2aj7yQlTY1OwLoeSEllvXY,17207
 syncmodels/mapper/__init__.py,sha256=jS82LFr9zzyqXBz82tSw04vDowhTpKxhg_W2XvhUlt0,129
 syncmodels/mapper/fiware.py,sha256=auszPmhCS46z_68MXjksrQAFUfctjbVrVdBvOpOkMj8,523
@@ -302,10 +302,10 @@ syncmodels/session/postgresql.py,sha256=ZMIu1Rv93pKfvFlovFBmWArzlrT2xaQWNYGZT_LW
 syncmodels/session/sql.py,sha256=bD7zXRrEKKJmqY2UoibWENuWb5zHrrU72F3_dYbS6LY,6569
 syncmodels/session/sqlite.py,sha256=nCDjopLiBpX1F10qkKoARM7JrVdIpJ1WdGOduFVxaiA,2080
 syncmodels/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-syncmodels-0.1.314.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
-syncmodels-0.1.314.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
-syncmodels-0.1.314.dist-info/METADATA,sha256=59uo0tcMb8sXzIoBOlGNPUdyivZA7wIhkYj1hXZ1sMU,2700
-syncmodels-0.1.314.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
-syncmodels-0.1.314.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
-syncmodels-0.1.314.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
-syncmodels-0.1.314.dist-info/RECORD,,
+syncmodels-0.1.316.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
+syncmodels-0.1.316.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
+syncmodels-0.1.316.dist-info/METADATA,sha256=p969_vaLR5hO36CD2i3zn5LI6rx4RHRk4spuUh18vNQ,2700
+syncmodels-0.1.316.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
+syncmodels-0.1.316.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
+syncmodels-0.1.316.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
+syncmodels-0.1.316.dist-info/RECORD,,