syncmodels 0.1.341__py2.py3-none-any.whl → 0.1.343__py2.py3-none-any.whl

syncmodels/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Asterio Gonzalez"""
 __email__ = "asterio.gonzalez@gmail.com"
-__version__ = "0.1.341"
+__version__ = "0.1.343"

syncmodels/crawler.py

@@ -1578,6 +1578,11 @@ class SortPlugin(iPlugin):
             or self.bot.context.get(SORT_KEY)
             or getattr(self.bot, "SORT_KEY", None)
         )
+        if isinstance(sort_key, str):
+            sort_key = (sort_key,)
+        elif isinstance(sort_key, list):
+            sort_key = tuple(sort_key)
+
         if sort_key:
             context[SORT_KEY] = sort_key
 
@@ -1588,36 +1593,37 @@ class SortPlugin(iPlugin):
             # figure out the sort_key
             sort_key = SortKeyFinder.find_sort_key(stream)
 
-            kind = context.get(KIND_KEY)  # must exixts
-            if not kind:
-                log.error("cant find KIND_KEY in context??")
-                return
-            mapper = self.bot.parent.MAPPERS.get(kind)
-            if not mapper:
-                log.error("cant find mapper [%s] in crawler??", kind)
-                return
+        kind = context.get(KIND_KEY)  # must exixts
+        if not kind:
+            log.error("cant find KIND_KEY in context??")
+            return
 
-            model = mapper.PYDANTIC
-            item_fields = model.model_fields
-            # datetime
-            # reverse = list(
-            #     set(
-            #         [
-            #             "datetime",
-            #         ]
-            #     ).intersection(item_fields)
-            # )
-            reverse = list(set(sort_key).intersection(item_fields))
-            if not reverse:
-                log.debug(
-                    "model [%s] attributes:[%s] doesn't match any datetime alike sort keys: [%s]",
-                    model,
-                    list(item_fields),
-                    sort_key,
-                )
-                # return
+        mapper = self.bot.parent.MAPPERS.get(kind)
+        if not mapper:
+            log.error("cant find mapper [%s] in crawler??", kind)
+            return
 
-            context[REVERSE_SORT_KEY] = reverse
+        model = mapper.PYDANTIC
+        item_fields = model.model_fields
+        # datetime
+        # reverse = list(
+        #     set(
+        #         [
+        #             "datetime",
+        #         ]
+        #     ).intersection(item_fields)
+        # )
+        reverse = list(set(sort_key).intersection(item_fields))
+        if not reverse:
+            log.debug(
+                "model [%s] attributes:[%s] doesn't match any datetime alike sort keys: [%s]",
+                model,
+                list(item_fields),
+                sort_key,
+            )
+            # return
+
+        context[REVERSE_SORT_KEY] = reverse
 
         if sort_key:
             context[SORT_KEY] = sort_key
@@ -1947,7 +1953,7 @@ class MetaExtractPlugin(iPlugin):
                 # MAP = self.bot.parent.MAPPERS
                 geojson = data.get(GEOJSON_KEY)  # TODO: fix, coords are tuples!
                 if geojson is None:
-                    log.warning("no geojson info: [%s]", data)
+                    log.debug("no geojson info: [%s]", data)
                     continue
                 kind = geojson["type"]
                 # TODO: use (slower) more general jinja render?

syncmodels/helpers/crawler.py

@@ -35,6 +35,8 @@ class SortKeyFinder:
     # )
     DATE_KEYS = [
         # ".*date", # keys that does not belongs to Pydantic models
+        # "id",
+        # "record_id",
         "time",
         "year",
         "ts",

syncmodels/session/__init__.py

@@ -32,6 +32,7 @@ from syncmodels.definitions import (
     # MONOTONIC_SINCE,
     MONOTONIC_SINCE_KEY,
     MONOTONIC_SINCE_VALUE,
+    SORT_KEY,
     URI,
     DURI,
     WAVE_LAST_KEY,
@@ -410,6 +411,11 @@ class iSession(iContext, iSchema, iRegistry):  # , iAuthenticator):
                 break
             break
 
+        if not params.get(MONOTONIC_SINCE_KEY):
+            if since_key := params.get(SORT_KEY):
+                # if the 1st time that crawler want to retrieve data
+                params[MONOTONIC_SINCE_KEY] = since_key
+
         call_kw = {
             "url": url,
             "headers": self.headers,

syncmodels/session/sql.py

@@ -3,6 +3,9 @@ import re
 from itertools import chain
 
 from agptools.helpers import parse_uri, build_uri, tf
+from agptools.containers import overlap
+from syncmodels.http import CONTENT_TYPE, APPLICATION_PYTHON
+
 from syncmodels.definitions import (
     KIND_KEY,
     TABLE_KEY,
@@ -13,6 +16,7 @@ from syncmodels.definitions import (
     JSON,
     LIMIT_KEY_VALUE,
 )
+from syncmodels.storage import is_sort_key_id
 
 from ..crud import DEFAULT_NAMESPACE, parse_duri
 from ..schema import StructShema
@@ -115,7 +119,11 @@ class iSQLSession(iSession):
 
         query = f"SELECT * FROM {table}"
         if MONOTONIC_SINCE_VALUE in params:
-            query += f" WHERE {since_key} >= :{MONOTONIC_SINCE_VALUE}"
+            if is_sort_key_id(since_key):
+                query += f" WHERE {since_key} > :{MONOTONIC_SINCE_VALUE}"
+            else:
+                query += f" WHERE {since_key} >= :{MONOTONIC_SINCE_VALUE}"
+
         if since_key:
             query += f" ORDER BY {since_key}"
         # limit = 128 # TODO: agp: REMOVE
@@ -164,7 +172,7 @@ class iSQLSession(iSession):
 
     async def get(self, url, headers=None, params=None, **kw):
         headers = headers or {}
-        params = params or {}
+        params = params or kw["json"]["params"]  # TODO: check with centesimal
 
         _uri = parse_uri(url)
         _uri["query_"].update(params)
@@ -190,6 +198,12 @@ class iSQLSession(iSession):
         #     {schema.names[i]: v for i, v in enumerate(row)} for row in data
         # ]
 
+        overlap(
+            headers,
+            {
+                CONTENT_TYPE: APPLICATION_PYTHON,
+            },
+        )
         response = iResponse(
             status=200, headers=headers, links=None, real_url=url, body=body
         )

syncmodels/storage.py

@@ -137,6 +137,12 @@ UTC_TZ = pytz.timezone("UTC")
 # REGEXP_FQUI = re.compile(r"((?P<ns>[^/]*?)/)?(?P<table>[^:]+):(?P<uid>.*)$")
 
 
+def is_sort_key_id(sort_keys):
+    if isinstance(sort_keys, str):
+        return re.search(REGEXP_RECORD_ID, sort_keys)
+    return any([re.search(REGEXP_RECORD_ID, _) for _ in sort_keys])
+
+
 def comparable_struct(data, patterns):
     wdata = Walk(data)
 
@@ -150,8 +156,15 @@ def split_fqui(fqid):
         return fqid, None
 
 
+REGEXP_RECORD_ID = r"(\W*|_)id$"
+
+
 def normalize_payload(data, keys):
     for key in set(keys or []).intersection(data):
+        # skip record_id and similar keys
+        if re.search(REGEXP_RECORD_ID, key):
+            continue
+
         value = DATE(data[key])
         if isinstance(value, datetime):
             if not value.tzinfo:
@@ -908,27 +921,45 @@ class WaveStorage(iWaves, iStorage):
                     nonlocal uid
                     nonlocal monotonic
                     nonlocal data_sort_blueprint
-                    for monotonic_key in set(sort_keys).intersection(data):
-                        monotonic_value = DATE(data[monotonic_key])
-
-                        # seconds
-                        grace_period = kw.get(GRACE_PERIOD_KEY, DEFAULT_GRACE_PERIOD)
-                        grace_period = timedelta(seconds=grace_period)
-                        since_value = monotonic_value - grace_period
-                        # pass to UTC time
-                        if not since_value.tzinfo:
-                            # x = x.replace(tzinfo=timezone.utc)
-                            # x = x.replace(tzinfo=LOCAL_TZ)
-                            since_value = pytz.utc.localize(since_value)
-                        since_value = since_value.astimezone(UTC_TZ)
-                        since_value = since_value.strftime("%Y-%m-%dT%H:%M:%SZ")
 
-                        break
+                    # using record_id alike sort_key?
+                    if is_sort_key_id(sort_keys):
+                        for monotonic_key in set(sort_keys).intersection(data):
+                            monotonic_value = data[monotonic_key]
+                            break
+                        else:
+                            log.error("can't find %s key in %s", monotonic_key, data)
+                        # in this case, no grace_period is needed
+                        since_value = monotonic_value
+                        monotonic_operator = ">"
                     else:
-                        monotonic_key = MONOTONIC_KEY  # ??
-                        grace_period = kw.get(GRACE_PERIOD_KEY, DEFAULT_GRACE_PERIOD)
-                        grace_period *= 10**9  # nanoseconds
-                        since_value = monotonic - grace_period
+                        # using datetime alike sort_key
+                        monotonic_operator = ">="
+                        for monotonic_key in set(sort_keys).intersection(data):
+                            monotonic_value = DATE(data[monotonic_key])
+
+                            # seconds
+                            grace_period = kw.get(
+                                GRACE_PERIOD_KEY, DEFAULT_GRACE_PERIOD
+                            )
+                            grace_period = timedelta(seconds=grace_period)
+                            since_value = monotonic_value - grace_period
+                            # pass to UTC time
+                            if not since_value.tzinfo:
+                                # x = x.replace(tzinfo=timezone.utc)
+                                # x = x.replace(tzinfo=LOCAL_TZ)
+                                since_value = pytz.utc.localize(since_value)
+                            since_value = since_value.astimezone(UTC_TZ)
+                            since_value = since_value.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+                            break
+                        else:
+                            monotonic_key = MONOTONIC_KEY  # ??
+                            grace_period = kw.get(
+                                GRACE_PERIOD_KEY, DEFAULT_GRACE_PERIOD
+                            )
+                            grace_period *= 10**9  # nanoseconds
+                            since_value = monotonic - grace_period
 
                     query = f"{namespace}://{database}/{thing}"
 
@@ -937,7 +968,7 @@ class WaveStorage(iWaves, iStorage):
                     data_sort_bp = {
                         MONOTONIC_SINCE_KEY: monotonic_key,
                         MONOTONIC_SINCE_VALUE: since_value,
-                        MONOTONIC_SINCE_OPERATOR: ">=",
+                        MONOTONIC_SINCE_OPERATOR: monotonic_operator,
                         ORDER_KEY: monotonic_key,
                         DIRECTION_KEY: DIRECTION_DESC,
                         # LIMIT_KEY: kw.get(
@@ -1186,7 +1217,7 @@ class WaveStorage(iWaves, iStorage):
                 if must_check:
                     t0 = time.time()
                     await prevously_inserted()
-                    if False or random.random() < 0.1:
+                    if False or random.random() < 0.05:
                         elapsed = time.time() - t0
                         log.info("[%s] prevously_inserted took: %s secs", uid, elapsed)
                         if elapsed > 1.0:

{syncmodels-0.1.341.dist-info → syncmodels-0.1.343.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: syncmodels
-Version: 0.1.341
+Version: 0.1.343
 Summary: Synchronizable Models
 Home-page: https://github.com/asterio.gonzalez/syncmodels
 Author: Asterio Gonzalez
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.6
 License-File: LICENSE
 License-File: AUTHORS.rst
-Requires-Dist: agptools>=0.1.341
+Requires-Dist: agptools>=0.1.343
 Requires-Dist: aiocache
 Requires-Dist: aiohttp
 Requires-Dist: Click

{syncmodels-0.1.341.dist-info → syncmodels-0.1.343.dist-info}/RECORD

@@ -1,6 +1,6 @@
-syncmodels/__init__.py,sha256=OjT8zP-x3BGV0BADjHaNommB-qVVJL-X_QLxxfJ_dIM,142
+syncmodels/__init__.py,sha256=Mpde6Eca9MTKd_YLLbuJt4XCAjhQkFJWIMHIgyNtej0,142
 syncmodels/context.py,sha256=k1Gs_ip9BfyRFpyRnzqYvRDKo0sYBqJsh6z9sWln9oE,451
-syncmodels/crawler.py,sha256=DAD2tL93iG-LjNQRCg2h_snusjGhQyhdZyMPOCEyvNU,94499
+syncmodels/crawler.py,sha256=lx0IU586JD0qcdozWiv0y7970CkUEg3SzFB6gOYDS6I,94540
 syncmodels/crud.py,sha256=oZIcwEKR2i-lesEF_059Y4yThohd9m7gs6R6xYgLH-I,15351
 syncmodels/definitions.py,sha256=w-3TrSomp9T8OzLmJhKeZQDzrUIJLKldyh1lzlE7Yj0,5476
 syncmodels/exceptions.py,sha256=ZLAwu19cs2UN2Sv3jaLnixT_jRI7T42TfyutCkUsuIk,685
@@ -11,7 +11,7 @@ syncmodels/registry.py,sha256=YaQtgbSwa0je1MpCcVHALI3_b85vrddyOlhsnrUcKZs,8224
 syncmodels/requests.py,sha256=wWoC5hPDm1iBM_zrlyKRauzhXgdKR3pT5RqyC-5UZhQ,538
 syncmodels/runner.py,sha256=IHDKuQ3yJ1DN9wktMiIrerPepYX61tc3AzbFfuUqEFw,5454
 syncmodels/schema.py,sha256=uinUt8Asq_x7xa6MKWVXNyoWO6gKocjGPppjimaXzEU,2492
-syncmodels/storage.py,sha256=kMtErJYjn0x3J4Z4L8-ZjNhfC3o6GYafgauZ_UUV6-g,74907
+syncmodels/storage.py,sha256=GLEDryiBkgN9SgtvcqBRxP_22T00gGkIbiZqM6AqKDI,76158
 syncmodels/syncmodels.py,sha256=jcUxVbv1hrx5hI81VCO1onIM6WyORTqJVPwIqlPocOc,10596
 syncmodels/timequeue.py,sha256=YRd3ULRaIhoszaBsYhfr0epMqAbL6-NwVEtScjUYttM,595
 syncmodels/wave.py,sha256=Gra22BLiA9z2nF-6diXpjAc4GZv9nebmyvHxdAfXec4,7764
@@ -24,7 +24,7 @@ syncmodels/cli/surreal.py,sha256=eL7pDicLo0_68JhpCZacde6DOVcfHE_UEzczvgDuvAc,106
 syncmodels/cli/wingdbstub.py,sha256=q4z-RqHN1ttzNtiLYTzqQG2ZYZ6W3NOnEd2E5NGhfao,17165
 syncmodels/cli/workspace.py,sha256=wajZnxf567nYoQysTEgxrDAp8ZBU8zSuoP4KyZtqvdc,2461
 syncmodels/helpers/__init__.py,sha256=qZet64gMJNAAqzUdEqCV5WDk5D2Dbw1Kxlt9Jo6x3m4,23
-syncmodels/helpers/crawler.py,sha256=2WiDxnjxZKbGa8x71bgeSc2wmabKa_wD-ofQMbgVXLo,4805
+syncmodels/helpers/crawler.py,sha256=ptqj4g96nQfQIvdp18HRDovnWfB0RJXtTBzHQsNPrM8,4844
 syncmodels/helpers/explorer.py,sha256=-Dol3z1pALCMI9OPSvVbROaTzLjbUpS0suJ82Z6Rmb4,7447
 syncmodels/helpers/faker.py,sha256=tPtibNh28KoHb9kcwE9WaPdPrSrN6xMbCv5HhNFEVG0,1222
 syncmodels/helpers/general.py,sha256=UAcSfrvsaT15iJuxsR3WMk51UjpLLGDf14xmpBojndg,6160
@@ -297,16 +297,16 @@ syncmodels/model/schema_org/webpage.py,sha256=-bqKOpEs_2lW7qrey2nHtvOZ9xbmmwcviN
 syncmodels/model/schema_org/webpageelement.py,sha256=brXfhU3l3FBXpy8qnR1Ve-EckjHW8VGoyR2IsnT7t2Y,1104
 syncmodels/model/schema_org/website.py,sha256=48Rox27BbFIg1u3wDlOtX-lLCPoFgvvrCw5Hrdf6uRU,912
 syncmodels/model/schema_org/xpathtype.py,sha256=D8gKiCrGSSuUVYw7BIWmOIUbKATfv2IpbkV1B2TmjC0,484
-syncmodels/session/__init__.py,sha256=uy9nMxUUT6VeExOLLzr5Zebe7LvP67WY34WzOs67WB4,15765
+syncmodels/session/__init__.py,sha256=xjgmogPs-Nu5merFY-foJVT9X-qDbLk_fYbe9uGx6sY,16003
 syncmodels/session/http.py,sha256=tf7z0ccAEYoCOZT4Ukv3NBXz9hUO3vs2s9bm491pCj8,1480
 syncmodels/session/postgresql.py,sha256=ZMIu1Rv93pKfvFlovFBmWArzlrT2xaQWNYGZT_LW61k,175
-syncmodels/session/sql.py,sha256=17C8EHn_1twHezhMlD5esMvx4m0iIrnD7JK-TuBswgU,6573
+syncmodels/session/sql.py,sha256=auZ_3RZTSWViED4UrExYaes42Y2bTZKixEsfOpa_zaU,7036
 syncmodels/session/sqlite.py,sha256=nCDjopLiBpX1F10qkKoARM7JrVdIpJ1WdGOduFVxaiA,2080
 syncmodels/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-syncmodels-0.1.341.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
-syncmodels-0.1.341.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
-syncmodels-0.1.341.dist-info/METADATA,sha256=buhnYKc5Ir69IET-mVE4ACfMaxY6NlP279rQaBSodtE,2700
-syncmodels-0.1.341.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
-syncmodels-0.1.341.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
-syncmodels-0.1.341.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
-syncmodels-0.1.341.dist-info/RECORD,,
+syncmodels-0.1.343.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
+syncmodels-0.1.343.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
+syncmodels-0.1.343.dist-info/METADATA,sha256=sXMUvCabKs5rQ2jAfNc1Vgp6XBoQbcpPCXhRq4rQ9a8,2700
+syncmodels-0.1.343.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
+syncmodels-0.1.343.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
+syncmodels-0.1.343.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
+syncmodels-0.1.343.dist-info/RECORD,,