syncmodels 0.1.313__py2.py3-none-any.whl → 0.1.314__py2.py3-none-any.whl

syncmodels/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Asterio Gonzalez"""
 __email__ = "asterio.gonzalez@gmail.com"
-__version__ = "0.1.313"
+__version__ = "0.1.314"

syncmodels/crawler.py

@@ -152,7 +152,7 @@ from syncmodels.http import (
 # from syncmodels.registry import iRegistry
 from syncmodels.auth import iAuthenticator
 from syncmodels.storage import WaveStorage, Storage
-from syncmodels.exceptions import NonRecoverable, NonRecoverableAuth, BadData
+from syncmodels.exceptions import BadLogic, NonRecoverable, NonRecoverableAuth, BadData
 from syncmodels.syncmodels import SyncModel, COPY
 from .crud import parse_duri, DEFAULT_DATABASE, DEFAULT_NAMESPACE
 
@@ -770,6 +770,10 @@ class iBot(iAgent):
                     # abort responsability chain
                     log.error("Inexpected responsability chain abortion")
                     break
+            except (BadLogic,) as why:
+                # separate to have more fine grain control
+                await self.handle_non_recoverable(why, context)
+
             except Exception as why:
                 log.error(why)
                 log.error("".join(traceback.format_exception(*sys.exc_info())))
@@ -1071,7 +1075,7 @@ class iBot(iAgent):
                                     result = await extract_result(response)
                                     # result = await response.text()
                                     log.error(
-                                        "[%s] server sent: %s",
+                                        "Forbidden [%s] server sent: %s",
                                         response.status,
                                         result,
                                     )

syncmodels/definitions.py

@@ -100,6 +100,8 @@ GEOMETRY_COMP_KEY = "geometry__"
 GEOJSON_KEY = "geojson"
 GEOLINK_KEY = "geolink"
 
+COMPARISON_PATTERNS = "compare__"
+
 GEOSPECS_KEYS = [GEOMETRY_KEY, GEOMETRY_SHAPE_KEY, GEOJSON_KEY]
 
 UBICATION_KEY = "ubication"

syncmodels/exceptions.py

@@ -23,5 +23,9 @@ class NonRecoverableAuth(NonRecoverable):
     """
 
 
+class BadLogic(NonRecoverable):
+    """Bad Logic File"""
+
+
 class BadData(SyncModelException):
     """Data is not properly formatted."""

syncmodels/logic/browser.py

@@ -45,7 +45,7 @@ from syncmodels.crud import parse_duri
 from ..definitions import TASK_KEY, KIND_KEY, ORG_URL, DURI, ID_KEY, VOL_DATA
 from ..session import iSession
 from ..crawler import iBot
-from ..crawler import MetaExtractPlugin, PutPlugin, SetURIPlugin
+from ..crawler import MetaExtractPlugin, PutPlugin, SetURIPlugin, SortPlugin
 
 from .swarm import SwarmBot, SwarmCrawler, SWARM_REGISTER, SWARM_TASKS
 from .analyzer import XPathAnalyzer
@@ -717,13 +717,15 @@ class BrowserLogicBot(SwarmBot):
         )
 
     def _add_plugins(self):
-        super()._add_plugins()
+        # super()._add_plugins()
+        # self.plugins.clear()
 
-        self.plugins.clear()
         # self.add_plugin(HashStreamPlugin())
         self.add_plugin(MetaExtractPlugin(geojson=False))
         self.add_plugin(PutPlugin())
         self.add_plugin(SetURIPlugin())
+        # self.add_plugin(SortPlugin())
+
         # self.add_plugin(RenameKeys())
         # self.add_plugin(UnwrapResponse())
         # self.add_plugin(RegExtractor())

syncmodels/storage.py

@@ -26,7 +26,13 @@ from surrealist import Surreal as Surrealist
 
 from agptools.logs import logger
 from agptools.helpers import parse_uri, build_uri, DATE, tf
-from agptools.containers import merge as merge_dict, build_dict, json_compatible
+from agptools.containers import (
+    merge as merge_dict,
+    build_dict,
+    json_compatible,
+    Walk,
+    CWalk,
+)
 
 from syncmodels.definitions import (
     REVERSE_SORT_KEY,
@@ -40,6 +46,13 @@ from syncmodels.http import (
     # USER_AGENT,
     APPLICATION_JSON,
 )
+from syncmodels.exceptions import (
+    SyncModelException,
+    BadLogic,
+    NonRecoverable,
+    NonRecoverableAuth,
+    BadData,
+)
 
 from .helpers.importers import JSONVerter
 
@@ -72,6 +85,7 @@ from .definitions import (
     SORT_KEY,
     LIMIT_KEY,
     WAVE_LAST_KEY,
+    COMPARISON_PATTERNS,
 )
 from .crud import (
     DEFAULT_DATABASE,
@@ -120,6 +134,12 @@ UTC_TZ = pytz.timezone("UTC")
 
 
 # REGEXP_FQUI = re.compile(r"((?P<ns>[^/]*?)/)?(?P<table>[^:]+):(?P<uid>.*)$")
+
+
+def comparable_struct(data, patterns):
+    wdata = Walk(data)
+
+
 def split_fqui(fqid):
     "try to split FQUID into table and uid"
     try:
@@ -130,7 +150,7 @@ def split_fqui(fqid):
 
 
 def normalize_payload(data, keys):
-    for key in set(keys).intersection(data):
+    for key in set(keys or []).intersection(data):
         value = DATE(data[key])
         if isinstance(value, datetime):
             if not value.tzinfo:
@@ -777,7 +797,7 @@ class WaveStorage(iWaves, iStorage):
 
         2. if the object has been inserted in `tube` then:
            - create / update Snapshot
-           - cteate / update Wave info with original the original query data
+           - create / update Wave info with original the original query data
 
 
         e:  exists
@@ -818,8 +838,8 @@ class WaveStorage(iWaves, iStorage):
                 if model := kw.get(MODEL_KEY):
                     self.register_metadata(uri, {"model": model})
 
-                sort_keys = kw.get(SORT_KEY, [])
-                reverse_sort_keys = kw.get(REVERSE_SORT_KEY, [])
+                sort_keys = kw.get(SORT_KEY) or []
+                reverse_sort_keys = kw.get(REVERSE_SORT_KEY) or []
                 sort_kw_presence = all([kw.get(_) for _ in sort_keys])
                 sort_data_presence = all([data.get(_) for _ in reverse_sort_keys])
 
@@ -844,7 +864,7 @@ class WaveStorage(iWaves, iStorage):
                     # stream = [kw]
                     if not (sort_keys := SortKeyFinder.find_sort_key(stream=stream)):
                         kind = kw.get(KIND_KEY)
-                        sort_keys = kw[SORT_KEY] = SortKeyFinder.get(kind)
+                        sort_keys = kw[SORT_KEY] = SortKeyFinder.get(kind) or []
 
                     kw[SORT_KEY] = sort_keys
                     log.debug("[%s] found: %s as sort_keys", uri, sort_keys)
@@ -871,7 +891,7 @@ class WaveStorage(iWaves, iStorage):
                 normalize_payload(data, sort_keys)
 
                 monotonic = data.setdefault(MONOTONIC_KEY, monotonic_wave())
-                for monotonic_key in set(sort_keys or []).intersection(data):
+                for monotonic_key in set(sort_keys).intersection(data):
                     monotonic_value = DATE(data[monotonic_key])
 
                     # seconds
@@ -885,7 +905,6 @@ class WaveStorage(iWaves, iStorage):
                         since_value = pytz.utc.localize(since_value)
                     since_value = since_value.astimezone(UTC_TZ)
                     since_value = since_value.strftime("%Y-%m-%dT%H:%M:%SZ")
-
                     break
                 else:
                     monotonic_key = MONOTONIC_KEY  # ??
@@ -921,8 +940,6 @@ class WaveStorage(iWaves, iStorage):
                         sort_keys,
                         MASK,
                     )
-                # existing = await self.storage.query(query, **blueprint)
-                # N = len(existing)
 
                 # TODO: agp: cache and get behaviour from database?
                 if not (behavior := self.behavior_uri.get(query)):
@@ -938,8 +955,7 @@ class WaveStorage(iWaves, iStorage):
 
                     self.behavior_uri[query] = behavior
 
-                # t0 = time.time()
-
+                t0 = time.time()
                 # search the same data
                 # TODO: update blueprint
                 blueprint = {
@@ -966,49 +982,67 @@ class WaveStorage(iWaves, iStorage):
                     **blueprint,
                     # **data_sort_bp,
                 )
-                # t1 = time.time()
-                # _elapsed = t1 - t0
+                t1 = time.time()
+                _elapsed = t1 - t0
                 existing = identical + similar
-                N2 = len(existing)
-                if data_sort_blueprint and N2 > 1:
+                N = len(existing)
+                log.debug(
+                    "[%s] found [%s] similar records in %s secs",
+                    blueprint,
+                    N,
+                    _elapsed,
+                )
+                if data_sort_blueprint and N > 1:
                     if behavior & ALLOW_DUPLICATED_ITEMS:
                         log.debug(
                             "tube [%s] has multiples records: [%s] records, but ALLOW_SAME_DATE_DIFFERENT_VALUES is defined",
                             uid,
-                            N2,
+                            N,
                         )
                         existing.clear()
                     else:
                         log.debug(
                             "tube has multiples records: [%s] = %s records, must just 1 and sort_key is defined by: [%s]",
                             uid,
-                            N2,
+                            N,
                             data_sort_blueprint,
                         )
 
                 push = True
+                patterns = kw.get(COMPARISON_PATTERNS)
+                if patterns:
+                    wdata = CWalk(data, include=patterns, exclude=MASK)
+                    if not wdata:
+                        log.warning("patterns don't get any data")
+                        raise BadLogic(data)
+                else:
+                    patterns = [r".*"]
+                    wdata = CWalk(data, include=patterns, exclude=MASK)
                 for exists in existing:
+                    wexists = CWalk(exists, include=patterns, exclude=MASK)
                     existing_sort_blueprint = build_dict(exists, reverse_sort_keys)
                     # existing_sort_blueprint = build_comparisson_dict(exists, reverse_sort_keys)
 
                     same_sort_key = existing_sort_blueprint == data_sort_blueprint
 
                     # check if we must "duplicate" data inside tube
-                    keys0 = set(exists).difference(MASK)
-                    keys1 = set(data).difference(MASK)
+                    # keys0 = set(exists).difference(MASK)
+                    # keys1 = set(data).difference(MASK)
+                    keys0 = set(wexists)
+                    keys1 = set(wdata)
                     same_structure = keys0 == keys1
 
                     same_values = False
                     if same_sort_key and same_structure:
                         for key in keys0:
-                            if exists[key] != data[key]:
+                            if wexists[key] != wdata[key]:
                                 log.debug(
                                     "[%s].[%s].[%s]: %s != %s",
                                     uid,
                                     data_sort_blueprint,
                                     key,
-                                    exists[key],
-                                    data[key],
+                                    wexists[key],
+                                    wdata[key],
                                 )
                                 break
                         else:
@@ -1235,6 +1269,8 @@ class WaveStorage(iWaves, iStorage):
                 return all([res0, res1, res2])
             else:
                 return True
+        except SyncModelException as why:
+            raise why
         except Exception as why:  # pragma: nocover
             log.error(why)
             log.error("".join(traceback.format_exception(*sys.exc_info())))

{syncmodels-0.1.313.dist-info → syncmodels-0.1.314.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: syncmodels
-Version: 0.1.313
+Version: 0.1.314
 Summary: Synchronizable Models
 Home-page: https://github.com/asterio.gonzalez/syncmodels
 Author: Asterio Gonzalez
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.6
 License-File: LICENSE
 License-File: AUTHORS.rst
-Requires-Dist: agptools>=0.1.313
+Requires-Dist: agptools>=0.1.314
 Requires-Dist: aiocache
 Requires-Dist: aiohttp
 Requires-Dist: Click

{syncmodels-0.1.313.dist-info → syncmodels-0.1.314.dist-info}/RECORD

@@ -1,9 +1,9 @@
-syncmodels/__init__.py,sha256=F-gwFoSn9hneX4i5kpR9dYY6CUDVidYhx9nW2WiEJcU,142
+syncmodels/__init__.py,sha256=G_3tlPvmtf1CQaauaKV6PoulB_iHJ43bVbH_1q_p8aI,142
 syncmodels/context.py,sha256=k1Gs_ip9BfyRFpyRnzqYvRDKo0sYBqJsh6z9sWln9oE,451
-syncmodels/crawler.py,sha256=vfMn39Fev1yKY8ELvTK3rM-RUPiiy8YoxAG2q7b9MkM,94247
+syncmodels/crawler.py,sha256=U6F0QlzmxkReCB6X2Xbxmm2rRn99GEX005dG7AFl8xE,94430
 syncmodels/crud.py,sha256=viHBwzczcjNyFiLxL7VGYSbWJW5VjU8AvKaPufBMP7M,15303
-syncmodels/definitions.py,sha256=2P-Sfgj18viSHZ-wAK4WfQEzDKdyygs8Z-XLzA0jg_k,5420
-syncmodels/exceptions.py,sha256=8EOYW8h_2noeoKAYqG4aEQTgB1FEkamxKt3t4pDJ3pM,626
+syncmodels/definitions.py,sha256=dTMxcwcIeYQthZCZXNQmaO1SolWY-WhafASyl91xTo0,5455
+syncmodels/exceptions.py,sha256=ZLAwu19cs2UN2Sv3jaLnixT_jRI7T42TfyutCkUsuIk,685
 syncmodels/geofactory.py,sha256=1FkrdEn0QA0O4_lSUAwjqXH2dmlQWi32AkntnG4AEQY,10372
 syncmodels/http.py,sha256=FFVT3QJJgur2dv1Q_7l9ZsWN8z6_gUjOT9hJff1ZAqk,3335
 syncmodels/parallel.py,sha256=Ll8HmyFF9v9fIofqqSgfhyTlklvb77mTtNdG5Y9lqdQ,7145
@@ -11,7 +11,7 @@ syncmodels/registry.py,sha256=YaQtgbSwa0je1MpCcVHALI3_b85vrddyOlhsnrUcKZs,8224
 syncmodels/requests.py,sha256=wWoC5hPDm1iBM_zrlyKRauzhXgdKR3pT5RqyC-5UZhQ,538
 syncmodels/runner.py,sha256=IHDKuQ3yJ1DN9wktMiIrerPepYX61tc3AzbFfuUqEFw,5454
 syncmodels/schema.py,sha256=uinUt8Asq_x7xa6MKWVXNyoWO6gKocjGPppjimaXzEU,2492
-syncmodels/storage.py,sha256=N69l-CSZOZomwVJMALKbPI1jPsngxDNlxbxX9SV4vfg,70505
+syncmodels/storage.py,sha256=-kdXuPjkHBPauoXTWcmwK4jmH9hr2X1JFERqluznL1Q,71513
 syncmodels/syncmodels.py,sha256=EzSC4C75V4wJDmsLLbp8YUVwqA6A16KCNW8nB-MqPcs,10567
 syncmodels/timequeue.py,sha256=YRd3ULRaIhoszaBsYhfr0epMqAbL6-NwVEtScjUYttM,595
 syncmodels/wave.py,sha256=Gra22BLiA9z2nF-6diXpjAc4GZv9nebmyvHxdAfXec4,7764
@@ -38,7 +38,7 @@ syncmodels/helpers/units.py,sha256=g50m5DQrAyP_qpDRa4LCEA5Rz2UZUmlIixfWG_ddw9I,3
 syncmodels/logic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 syncmodels/logic/activity_logger.py,sha256=8wjvgRwaNbibYWGgl-trovSS70yNkoCTlb-AIx3aZEE,14053
 syncmodels/logic/analyzer.py,sha256=7JDIvZIiDVgdABaaRk7TEbzOFM6HAH3RaSsIDmL4QVE,12291
-syncmodels/logic/browser.py,sha256=W_s_rIWceGni2GMdhlrdZTTxTG-43n2CQDhPgyJfdE0,84711
+syncmodels/logic/browser.py,sha256=KqvOLPAl2p5EBol7FK-XYvOXeDnu0h9Dz7xfagKTheM,84768
 syncmodels/logic/swarm.py,sha256=eRBVlNAOzzWKFGCb7LGLx2aj7yQlTY1OwLoeSEllvXY,17207
 syncmodels/mapper/__init__.py,sha256=jS82LFr9zzyqXBz82tSw04vDowhTpKxhg_W2XvhUlt0,129
 syncmodels/mapper/fiware.py,sha256=auszPmhCS46z_68MXjksrQAFUfctjbVrVdBvOpOkMj8,523
@@ -302,10 +302,10 @@ syncmodels/session/postgresql.py,sha256=ZMIu1Rv93pKfvFlovFBmWArzlrT2xaQWNYGZT_LW
 syncmodels/session/sql.py,sha256=bD7zXRrEKKJmqY2UoibWENuWb5zHrrU72F3_dYbS6LY,6569
 syncmodels/session/sqlite.py,sha256=nCDjopLiBpX1F10qkKoARM7JrVdIpJ1WdGOduFVxaiA,2080
 syncmodels/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-syncmodels-0.1.313.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
-syncmodels-0.1.313.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
-syncmodels-0.1.313.dist-info/METADATA,sha256=NRPuII9unMl4-z_TQqvb9W-Tp94LIT6CXU3ctK_v3us,2700
-syncmodels-0.1.313.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
-syncmodels-0.1.313.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
-syncmodels-0.1.313.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
-syncmodels-0.1.313.dist-info/RECORD,,
+syncmodels-0.1.314.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
+syncmodels-0.1.314.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
+syncmodels-0.1.314.dist-info/METADATA,sha256=59uo0tcMb8sXzIoBOlGNPUdyivZA7wIhkYj1hXZ1sMU,2700
+syncmodels-0.1.314.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
+syncmodels-0.1.314.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
+syncmodels-0.1.314.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
+syncmodels-0.1.314.dist-info/RECORD,,