syncmodels 0.1.312__py2.py3-none-any.whl → 0.1.313__py2.py3-none-any.whl

syncmodels/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Asterio Gonzalez"""
 __email__ = "asterio.gonzalez@gmail.com"
-__version__ = "0.1.312"
+__version__ = "0.1.313"

syncmodels/crawler.py

@@ -217,7 +217,7 @@ class iAgent(iRunner):
         include=None,
         exclude=None,
         credentials=None,
-        prefix=None,
+        prefix="",
         *args,
         **kw,
     ):
@@ -235,9 +235,8 @@ class iAgent(iRunner):
         }
 
         # prefix template
-        if prefix:
-            self.prefix_template = self._compile_template(prefix)
-            self.prefix = prefix
+        self.prefix_template = self._compile_template(prefix)
+        self.prefix = prefix
 
         # config file
         if not config_path:
@@ -268,6 +267,11 @@ class iAgent(iRunner):
 
         return template
 
+    async def start(self):
+        "start runner"
+        await super().start()
+        await self._bootstrap()
+
     async def _bootstrap(self):
         "Add the initial tasks to be executed by agent"
         log.debug(">> [%s] entering bootstrap()", self.name)
@@ -403,7 +407,7 @@ class iAgent(iRunner):
                 log.debug("+ Task: [%s]: [%s] %s", i, self.name, task)
 
                 # TODO: agp: generalize and provide a better way to save activity records
-                if activity_logger := getattr(self, "activity_logger"):
+                if activity_logger := getattr(self, "activity_logger", None):
                     if activity := task.get(ACTIVITY_LOG_KEY):
                         activity.status = "pending"
                         await activity_logger.update_activity(activity)
@@ -1635,6 +1639,21 @@ class SortPlugin(iPlugin):
             kind = context.get(KIND_KEY)
             SortKeyFinder.register(kind, sort_key)
 
+            mapper = self.bot.parent.MAPPERS.get(kind)
+            if not mapper:
+                log.error("cant find mapper [%s] in crawler??", kind)
+                return
+
+            # TODO: unify this code (exists sort_key or not exists)
+            model = mapper.PYDANTIC
+            item_fields = model.model_fields
+            reverse = list(set(sort_key).intersection(item_fields))
+            if not reverse:
+                log.error("model [%s] has not datetime alike key??", model)
+                return
+
+            context[REVERSE_SORT_KEY] = reverse
+
             def order(data, accessor):
                 result = data
                 for key in accessor:
@@ -1642,7 +1661,6 @@ class SortPlugin(iPlugin):
                         result = result[key]
                     except Exception as why:
                         pass
-
                 return result
 
             stream.sort(key=partial(order, accessor=sort_key))
@@ -2521,11 +2539,6 @@ class iAsyncCrawler(iCrawler):
             # TODO: REVIEW: always is a single value (len==1), why not return waves[0]
         return waves
 
-    async def start(self):
-        "start runner"
-        await super().start()
-        await self._bootstrap()
-
     async def idle(self):
         "default implementation when loop has nothing to do"
         # check running bots

syncmodels/helpers/orion.py

@@ -117,7 +117,7 @@ class OrionInjector:
                     "put",
                     self.target_url
                     # + "/v2/entities/{id}/attrs?options=append,keyValues",
-                    + "/v2/entities/{id}/attrs",
+                    + "/v2/entities/{id}/attrs?type={type}",
                     self.FULL_EXCLUDE,
                 ],
             ),
@@ -137,6 +137,7 @@ class OrionInjector:
                     self.target_url
                     # + "/v2/entities/{id}/attrs?options=append,keyValues",
                     + "/v2/entities/{id}",
+                    # + "/v2/entities/{id}?type={type}", # TODO: avoid posibly 409 if id is used in other fs_pathservice
                     # self.EXCLUDE_ALL,
                     self.FULL_EXCLUDE,
                 ],

syncmodels/model/__init__.py

@@ -7,4 +7,5 @@ from .model import (
     IntEnum,
     Field,
     field_validator,
+    model_validator,
 )

syncmodels/model/model.py

@@ -23,7 +23,7 @@ from typing_extensions import Annotated
 from pydantic import BaseModel as _BaseModel
 from pydantic import Field
 from pydantic import PlainSerializer, BeforeValidator
-from pydantic.functional_validators import field_validator
+from pydantic.functional_validators import field_validator, model_validator
 
 # from pydantic.dataclasses import dataclass
 

syncmodels/runner.py

@@ -62,7 +62,9 @@ class iRunner:
         self.t0 = 0
         self.t1 = 0
         self.nice = 600
+        self.clock = 0
 
+        self.hooks = []
         self.stop_when_empty = stop_when_empty
 
         self._wip = []
@@ -160,6 +162,9 @@ class iRunner:
         if not func:
             func = getattr(self, task[FUNC_KEY])  # let fail with AttributeError
         await func(**task)
+        self.clock += 1
+        for hook in self.hooks:
+            await hook(**task)
 
     async def idle(self):
         "default implementation when loop has nothing to do"

syncmodels/storage.py

@@ -9,7 +9,7 @@ import time
 import sys
 import traceback
 from typing import List
-from datetime import timedelta
+from datetime import timedelta, datetime
 import pytz
 from multiprocessing import Process
 import random
@@ -129,6 +129,20 @@ def split_fqui(fqid):
         return fqid, None
 
 
+def normalize_payload(data, keys):
+    for key in set(keys).intersection(data):
+        value = DATE(data[key])
+        if isinstance(value, datetime):
+            if not value.tzinfo:
+                value = pytz.utc.localize(value)
+            value = value.astimezone(UTC_TZ)
+            value = value.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+        data[key] = value
+
+    return data
+
+
 # ---------------------------------------------------------
 # Data Store / Ignore Policies
 # ---------------------------------------------------------
@@ -853,6 +867,9 @@ class WaveStorage(iWaves, iStorage):
                 # now
                 # ['2005-06-01T00:00:00.000+02:00']
                 # [datetime.datetime(2005, 6, 1, 0, 0, tzinfo=tzoffset(None, 7200))]
+
+                normalize_payload(data, sort_keys)
+
                 monotonic = data.setdefault(MONOTONIC_KEY, monotonic_wave())
                 for monotonic_key in set(sort_keys or []).intersection(data):
                     monotonic_value = DATE(data[monotonic_key])
@@ -878,8 +895,8 @@ class WaveStorage(iWaves, iStorage):
 
                 query = f"{namespace}://{database}/{thing}"
 
-                # data_sort_blueprint = build_dict(data, sort_keys)
-                data_sort_blueprint = build_dict(data, reverse_sort_keys)
+                data_sort_blueprint = build_dict(data, sort_keys)
+                # data_sort_blueprint = build_comparisson_dict(data, reverse_sort_keys)
                 blueprint = {
                     MONOTONIC_SINCE_KEY: monotonic_key,
                     MONOTONIC_SINCE_VALUE: since_value,
@@ -890,7 +907,7 @@ class WaveStorage(iWaves, iStorage):
                         LIMIT_KEY, 50  # TODO: agp: set in definition?
                     ),  # TODO: this is temporal, ideally None
                     ORG_KEY: uid,
-                    **data_sort_blueprint,  # implies sv = True
+                    # **data_sort_blueprint,  # implies sv = True
                 }
                 # TODO: LIMIT 1 ?
 
@@ -921,15 +938,38 @@ class WaveStorage(iWaves, iStorage):
 
                     self.behavior_uri[query] = behavior
 
-                t0 = time.time()
-                existing2 = await self.storage.query(
+                # t0 = time.time()
+
+                # search the same data
+                # TODO: update blueprint
+                blueprint = {
+                    # MONOTONIC_SINCE_KEY: monotonic_key,
+                    # MONOTONIC_SINCE_VALUE: since_value,
+                    # MONOTONIC_SINCE_OPERATOR: ">=",
+                    # ORDER_KEY: monotonic_key,
+                    # DIRECTION_KEY: DIRECTION_DESC,
+                    LIMIT_KEY: kw.get(
+                        LIMIT_KEY, 50  # TODO: agp: set in definition?
+                    ),  # TODO: this is temporal, ideally None
+                    ORG_KEY: uid,
+                    **data_sort_blueprint,  # implies sv = True
+                }
+                identical = await self.storage.query(
                     query,
                     **blueprint,
                     # **data_sort_bp,
                 )
-                t1 = time.time()
-                _elapsed = t1 - t0
-                N2 = len(existing2)
+                # TODO: try to create only a single query
+                # TODO: review different structures case
+                similar = await self.storage.query(
+                    query,
+                    **blueprint,
+                    # **data_sort_bp,
+                )
+                # t1 = time.time()
+                # _elapsed = t1 - t0
+                existing = identical + similar
+                N2 = len(existing)
                 if data_sort_blueprint and N2 > 1:
                     if behavior & ALLOW_DUPLICATED_ITEMS:
                         log.debug(
@@ -937,7 +977,7 @@ class WaveStorage(iWaves, iStorage):
                             uid,
                             N2,
                         )
-                        existing2.clear()
+                        existing.clear()
                     else:
                         log.debug(
                             "tube has multiples records: [%s] = %s records, must just 1 and sort_key is defined by: [%s]",
@@ -947,20 +987,9 @@ class WaveStorage(iWaves, iStorage):
                         )
 
                 push = True
-                for exists in existing2:
+                for exists in existing:
                     existing_sort_blueprint = build_dict(exists, reverse_sort_keys)
-
-                    # check if stream provide an old-echo item
-                    # sort_keys must exists in both data
-                    old_echo = [
-                        old_echo.append(data[_] < exists[_]) if _ in exists else False
-                    ]
-                    if any(old_echo):
-                        log.warning(
-                            "provider give old data again, preserving storage value and ignore this old_echo"
-                        )
-                        push = False
-                        break
+                    # existing_sort_blueprint = build_comparisson_dict(exists, reverse_sort_keys)
 
                     same_sort_key = existing_sort_blueprint == data_sort_blueprint
 

{syncmodels-0.1.312.dist-info → syncmodels-0.1.313.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: syncmodels
-Version: 0.1.312
+Version: 0.1.313
 Summary: Synchronizable Models
 Home-page: https://github.com/asterio.gonzalez/syncmodels
 Author: Asterio Gonzalez
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.6
 License-File: LICENSE
 License-File: AUTHORS.rst
-Requires-Dist: agptools>=0.1.312
+Requires-Dist: agptools>=0.1.313
 Requires-Dist: aiocache
 Requires-Dist: aiohttp
 Requires-Dist: Click

{syncmodels-0.1.312.dist-info → syncmodels-0.1.313.dist-info}/RECORD

@@ -1,6 +1,6 @@
-syncmodels/__init__.py,sha256=ZH2kRAl4tVru0sACzSbL7-N33RPSBU6YP06lTOu_Bzk,142
+syncmodels/__init__.py,sha256=F-gwFoSn9hneX4i5kpR9dYY6CUDVidYhx9nW2WiEJcU,142
 syncmodels/context.py,sha256=k1Gs_ip9BfyRFpyRnzqYvRDKo0sYBqJsh6z9sWln9oE,451
-syncmodels/crawler.py,sha256=cyGX1ZiVWaEWdQAUjDhjH9Qb8VAeV-yB4rFA1weAC9I,93701
+syncmodels/crawler.py,sha256=vfMn39Fev1yKY8ELvTK3rM-RUPiiy8YoxAG2q7b9MkM,94247
 syncmodels/crud.py,sha256=viHBwzczcjNyFiLxL7VGYSbWJW5VjU8AvKaPufBMP7M,15303
 syncmodels/definitions.py,sha256=2P-Sfgj18viSHZ-wAK4WfQEzDKdyygs8Z-XLzA0jg_k,5420
 syncmodels/exceptions.py,sha256=8EOYW8h_2noeoKAYqG4aEQTgB1FEkamxKt3t4pDJ3pM,626
@@ -9,9 +9,9 @@ syncmodels/http.py,sha256=FFVT3QJJgur2dv1Q_7l9ZsWN8z6_gUjOT9hJff1ZAqk,3335
 syncmodels/parallel.py,sha256=Ll8HmyFF9v9fIofqqSgfhyTlklvb77mTtNdG5Y9lqdQ,7145
 syncmodels/registry.py,sha256=YaQtgbSwa0je1MpCcVHALI3_b85vrddyOlhsnrUcKZs,8224
 syncmodels/requests.py,sha256=wWoC5hPDm1iBM_zrlyKRauzhXgdKR3pT5RqyC-5UZhQ,538
-syncmodels/runner.py,sha256=T6A_aZPtj4vqoqEf1bhd_efFEGbE6aSVjP8nJlAHFWQ,5320
+syncmodels/runner.py,sha256=IHDKuQ3yJ1DN9wktMiIrerPepYX61tc3AzbFfuUqEFw,5454
 syncmodels/schema.py,sha256=uinUt8Asq_x7xa6MKWVXNyoWO6gKocjGPppjimaXzEU,2492
-syncmodels/storage.py,sha256=vyIW_C6FoSh4VdCs-GcvNOKdpIfqyaIc1Cq5PV081Lo,69485
+syncmodels/storage.py,sha256=N69l-CSZOZomwVJMALKbPI1jPsngxDNlxbxX9SV4vfg,70505
 syncmodels/syncmodels.py,sha256=EzSC4C75V4wJDmsLLbp8YUVwqA6A16KCNW8nB-MqPcs,10567
 syncmodels/timequeue.py,sha256=YRd3ULRaIhoszaBsYhfr0epMqAbL6-NwVEtScjUYttM,595
 syncmodels/wave.py,sha256=Gra22BLiA9z2nF-6diXpjAc4GZv9nebmyvHxdAfXec4,7764
@@ -32,7 +32,7 @@ syncmodels/helpers/geojson.py,sha256=9VZjdEw19fLS_Q9topapUOPGxRg2D4GOqm2tlx11W7M
 syncmodels/helpers/importers.py,sha256=KImR9pQu4ir6EI6Ipta0q3RWloFT_VTJi67kM0lZsKQ,3919
 syncmodels/helpers/loaders.py,sha256=aus0aRcbU1vVa_zWo42aX6uV3B0fQ0aQpkTWlR9xGLA,4325
 syncmodels/helpers/models.py,sha256=c_ATzmiw5mVY1IGnwmyhjIuu5d2idHU-XeRigZSMkOQ,719
-syncmodels/helpers/orion.py,sha256=Fx-sVvI35FyKFcSd0vfY2Tjp7hlIwBJpshs2vb3fcsc,30584
+syncmodels/helpers/orion.py,sha256=Lblk4OGRNidYl1-2lZKUPXyvuR1hXip7FHloXLtU-XA,30717
 syncmodels/helpers/surreal.py,sha256=zoWtGm5oAxwvgJNq_NTpKOHN3h9FNObhFDLuiBOl1YY,10050
 syncmodels/helpers/units.py,sha256=g50m5DQrAyP_qpDRa4LCEA5Rz2UZUmlIixfWG_ddw9I,3571
 syncmodels/logic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -43,11 +43,11 @@ syncmodels/logic/swarm.py,sha256=eRBVlNAOzzWKFGCb7LGLx2aj7yQlTY1OwLoeSEllvXY,172
 syncmodels/mapper/__init__.py,sha256=jS82LFr9zzyqXBz82tSw04vDowhTpKxhg_W2XvhUlt0,129
 syncmodels/mapper/fiware.py,sha256=auszPmhCS46z_68MXjksrQAFUfctjbVrVdBvOpOkMj8,523
 syncmodels/mapper/mapper.py,sha256=SphMhr59bbTWWxnvitonURk3lSPDerGqUTs5-P-Tjlg,17397
-syncmodels/model/__init__.py,sha256=tHoKorgbN77ulv2SZ_fGqZqkq7yoWUvqM5SKKxrJWRE,127
+syncmodels/model/__init__.py,sha256=mD6fpRvBOl7i98pqqwpB2NKlviyD-G8iRXqufjU-9z8,148
 syncmodels/model/activity.py,sha256=tKzc9zc_5Sx549yTdaTYkfnLnbEsEmzpu53BHstqvFQ,4592
 syncmodels/model/geofilters.py,sha256=YxZZPk60MvzrBslOrpJec_Er1aTWGg1wgYrXPovrpP8,9284
 syncmodels/model/geojson.py,sha256=BsW3phXHBGeaIOyOZg7U8bRYKKerAEgNjCl8EnBS2Dg,8329
-syncmodels/model/model.py,sha256=0DtnsqlK1oEzHe5CdPMRnNaYJT3Hi2S-adqdydBSyYM,11541
+syncmodels/model/model.py,sha256=c8fAvGZ8_oXYQbdNixltRFhWjY3zYE1Y7MTLXIjbpac,11558
 syncmodels/model/swarm.py,sha256=0QMc1iyiuW5goLLQGFTIky3dJcc5BfuBPHtVbn7iRK4,1266
 syncmodels/model/schema_org/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 syncmodels/model/schema_org/aboutpage.py,sha256=_sqPMc7LxGWNSHwkvTB6OZZAfEovgHltisp6AAz2t9Y,452
@@ -302,10 +302,10 @@ syncmodels/session/postgresql.py,sha256=ZMIu1Rv93pKfvFlovFBmWArzlrT2xaQWNYGZT_LW
 syncmodels/session/sql.py,sha256=bD7zXRrEKKJmqY2UoibWENuWb5zHrrU72F3_dYbS6LY,6569
 syncmodels/session/sqlite.py,sha256=nCDjopLiBpX1F10qkKoARM7JrVdIpJ1WdGOduFVxaiA,2080
 syncmodels/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-syncmodels-0.1.312.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
-syncmodels-0.1.312.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
-syncmodels-0.1.312.dist-info/METADATA,sha256=CvmyXbM7GlQD3qm93hNGd_yq6yWd_MXAC6BatNPu_iA,2700
-syncmodels-0.1.312.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
-syncmodels-0.1.312.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
-syncmodels-0.1.312.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
-syncmodels-0.1.312.dist-info/RECORD,,
+syncmodels-0.1.313.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
+syncmodels-0.1.313.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
+syncmodels-0.1.313.dist-info/METADATA,sha256=NRPuII9unMl4-z_TQqvb9W-Tp94LIT6CXU3ctK_v3us,2700
+syncmodels-0.1.313.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
+syncmodels-0.1.313.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
+syncmodels-0.1.313.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
+syncmodels-0.1.313.dist-info/RECORD,,