syncmodels 0.1.317__py2.py3-none-any.whl → 0.1.319__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
syncmodels/__init__.py CHANGED
@@ -2,4 +2,4 @@
2
2
 
3
3
  __author__ = """Asterio Gonzalez"""
4
4
  __email__ = "asterio.gonzalez@gmail.com"
5
- __version__ = "0.1.317"
5
+ __version__ = "0.1.319"
@@ -751,7 +751,7 @@ class OrionInjector:
751
751
  # swarm_wave: 1726715149029566739
752
752
  # -19 nanosecods !
753
753
  # maybe due an aritmetic precission with Orion
754
- orion_wave, swarm_wave = DATE(orion_wave), DATE(swarm_wave)
754
+ # orion_wave, swarm_wave = DATE(orion_wave), DATE(swarm_wave)
755
755
  # (orion_wave // 1000) == (swarm_wave // 1000)
756
756
 
757
757
  if orion_wave >= swarm_wave:
@@ -39,7 +39,7 @@ class XPathAnalyzer:
39
39
 
40
40
  REGEXP_PRICES = {
41
41
  "euro": [
42
- r"(?P<price>\d+([\.\,]\d+)?)\s*(?P<currency>€|\$)?",
42
+ r"(?P<price>\d+([\.\,\s]\d+)?)\s*(?P<currency>€|\$)?",
43
43
  {
44
44
  "price": FLOAT,
45
45
  },
@@ -295,11 +295,11 @@ class XPathAnalyzer:
295
295
  def _analyze_brand(self, line, ctx) -> bool:
296
296
  return self._analyze_text(line, ctx)
297
297
 
298
- def _analyze_ldjson(self, line, ctx) -> bool:
298
+ def _analyze_ldjson_hide(self, line, ctx) -> bool:
299
299
  # TODO: this funcion has been temporary disabled
300
300
  return False
301
301
 
302
- def _analyze_ldjson_old(self, line, ctx) -> bool:
302
+ def _analyze_ldjson(self, line, ctx) -> bool:
303
303
  # TODO: helper for loading json
304
304
  line = line.strip()
305
305
  line = html2text(line)
@@ -745,18 +745,25 @@ class BrowserLogicBot(SwarmBot):
745
745
  return await super()._get_session(url, **context)
746
746
 
747
747
  async def _swarm_get_tasks(self):
748
+ """
749
+ universe
750
+ {'www.notino.es': [{'bot': '0',
751
+ 'datetime': '2025-07-12 10:22:37.128127+00:00',
752
+ 'fquid': 'pricemon://crawler/queue:ff97c78fc536e13407b60fab27054f2a',
753
+ 'id': 'queue:ff97c78fc536e13407b60fab27054f2a',
754
+ 'kind__': 'prices',
755
+ 'last': '2025-07-12 10:07:37.128127',
756
+ 'payload': None,
757
+ 'url': 'https://www.notino.es/ardell/remover-producto-para-quitar-las-pestanas-postizas/p-16258869/'}]}
758
+ """
748
759
  universe = {}
749
760
  if False:
750
761
  _task = {
751
762
  "bot": "wip",
752
- "created": "2025-01-13 06:10:48.341231",
753
763
  "datetime": "2025-01-18 11:46:18.241069+00:00",
754
- "description": None,
755
- "ecommerce": "www.atida.com",
756
764
  "id": "queue:00a401e322d3a3b43e8687cf5d4c10bd",
757
- "name": "Sendo champú soothing 250ml - lenitivo",
758
- "restart": None,
759
- "url": "https://www.arenal.com/ardell-pestanas-postizas-seamless-extensions-naked/p/000000000000120871",
765
+ "kind__": "prices",
766
+ "url": "https://wells.pt/total-plex-bond-reconstruction-shampoo-7932859.html",
760
767
  # "url": "https://www.maquillalia.com/ardell-kit-de-pestanas-postizas-seamless-naked-p-84985.html",
761
768
  # "url": "https://www.douglas.es/es/p/5010993205",
762
769
  # "url": "https://www.amazon.es/gp/offer-listing/B00LM8CW6Y",
@@ -766,7 +773,7 @@ class BrowserLogicBot(SwarmBot):
766
773
  # "url": "https://www.caretobeauty.com/es/kativa-coconut-conditioner-355ml",
767
774
  # "url": "https://www.holaprincesa.es/products/mascarilla-con-manteca-de-karite-aceite-de-coco-y-marula-300-ml-kativa",
768
775
  }
769
- _task.setdefault("type", "ldjson")
776
+ # _task.setdefault("type", "ldjson")
770
777
  host = parse_uri(_task.get("url")).get("host")
771
778
  universe.setdefault(host, []).append(_task)
772
779
 
syncmodels/storage.py CHANGED
@@ -878,284 +878,310 @@ class WaveStorage(iWaves, iStorage):
878
878
  foo = 1
879
879
  # return
880
880
 
881
- # 1. check if *same* object has been inserted in `tube` previously:
882
- # - (within grace period)
883
- # - if an identical object has been found, just ignore it.
884
- # - if an object with the same `sort_key` exists in the tube warning about the differences.
885
- # - if such *holder* object is not found in `tube` the insert it.
886
-
887
- # >>>>
888
- # now
889
- # ['2005-06-01T00:00:00.000+02:00']
890
- # [datetime.datetime(2005, 6, 1, 0, 0, tzinfo=tzoffset(None, 7200))]
891
-
881
+ push = True
892
882
  normalize_payload(data, sort_keys)
893
883
 
894
884
  monotonic = data.setdefault(MONOTONIC_KEY, monotonic_wave())
895
- for monotonic_key in set(sort_keys).intersection(data):
896
- monotonic_value = DATE(data[monotonic_key])
897
-
898
- # seconds
899
- grace_period = kw.get(GRACE_PERIOD_KEY, DEFAULT_GRACE_PERIOD)
900
- grace_period = timedelta(seconds=grace_period)
901
- since_value = monotonic_value - grace_period
902
- # pass to UTC time
903
- if not since_value.tzinfo:
904
- # x = x.replace(tzinfo=timezone.utc)
905
- # x = x.replace(tzinfo=LOCAL_TZ)
906
- since_value = pytz.utc.localize(since_value)
907
- since_value = since_value.astimezone(UTC_TZ)
908
- since_value = since_value.strftime("%Y-%m-%dT%H:%M:%SZ")
909
-
910
- break
911
- else:
912
- monotonic_key = MONOTONIC_KEY # ??
913
- grace_period = kw.get(GRACE_PERIOD_KEY, DEFAULT_GRACE_PERIOD)
914
- grace_period *= 10**9 # nanoseconds
915
- since_value = monotonic - grace_period
916
-
917
- query = f"{namespace}://{database}/{thing}"
918
-
919
- data_sort_blueprint = build_dict(data, sort_keys)
920
- # data_sort_blueprint = build_comparisson_dict(data, reverse_sort_keys)
921
- data_sort_bp = {
922
- MONOTONIC_SINCE_KEY: monotonic_key,
923
- MONOTONIC_SINCE_VALUE: since_value,
924
- MONOTONIC_SINCE_OPERATOR: ">=",
925
- ORDER_KEY: monotonic_key,
926
- DIRECTION_KEY: DIRECTION_DESC,
927
- # LIMIT_KEY: kw.get(
928
- # LIMIT_KEY, 50 # TODO: agp: set in definition?
929
- # ), # TODO: this is temporal, ideally None
930
- # ORG_KEY: uid,
931
- # **data_sort_blueprint, # implies sv = True
932
- }
933
- # TODO: LIMIT 1 ?
934
-
935
- # MASK = set([ID_KEY, MONOTONIC_KEY, *sort_keys])
936
- MASK = set([ID_KEY, MONOTONIC_KEY])
937
885
 
938
- if not reverse_sort_keys and sort_keys:
939
- MASK.update(sort_keys)
940
- log.debug(
941
- "including sort_keys: [%s] in excluded MASK: [%s] to find similar registers",
942
- sort_keys,
943
- MASK,
944
- )
886
+ async def prevously_inserted():
887
+ """check if *same* object has been inserted in `tube` previously:
888
+ - (within grace period)
889
+ - if an identical object has been found, just ignore it.
890
+ - if an object with the same `sort_key` exists in the tube warning about the differences.
891
+ - if such *holder* object is not found in `tube` the insert it.
892
+
893
+ # Note:
894
+ # now
895
+ # ['2005-06-01T00:00:00.000+02:00']
896
+ # [datetime.datetime(2005, 6, 1, 0, 0, tzinfo=tzoffset(None, 7200))]
897
+ """
898
+ nonlocal push
899
+ nonlocal sort_keys
900
+ nonlocal reverse_sort_keys
901
+ nonlocal namespace
902
+ nonlocal database
903
+ nonlocal thing
904
+ nonlocal uid
905
+ nonlocal monotonic
906
+ for monotonic_key in set(sort_keys).intersection(data):
907
+ monotonic_value = DATE(data[monotonic_key])
908
+
909
+ # seconds
910
+ grace_period = kw.get(GRACE_PERIOD_KEY, DEFAULT_GRACE_PERIOD)
911
+ grace_period = timedelta(seconds=grace_period)
912
+ since_value = monotonic_value - grace_period
913
+ # pass to UTC time
914
+ if not since_value.tzinfo:
915
+ # x = x.replace(tzinfo=timezone.utc)
916
+ # x = x.replace(tzinfo=LOCAL_TZ)
917
+ since_value = pytz.utc.localize(since_value)
918
+ since_value = since_value.astimezone(UTC_TZ)
919
+ since_value = since_value.strftime("%Y-%m-%dT%H:%M:%SZ")
945
920
 
946
- # TODO: agp: cache and get behaviour from database?
947
- if not (behavior := self.behavior_uri.get(query)):
948
- for (
949
- pattern,
950
- permissions,
951
- ) in self.behavior_templates.items():
952
- if re.match(pattern, query):
953
- behavior = permissions
954
- break
921
+ break
955
922
  else:
956
- behavior = ALL_RESTRICTIONS
957
-
958
- self.behavior_uri[query] = behavior
959
-
960
- t0 = time.time()
961
- # search the same data
962
- # TODO: update blueprint
963
- identical_bp = {
964
- LIMIT_KEY: kw.get(
965
- LIMIT_KEY, 10 # TODO: agp: set in definition?
966
- ), # TODO: this is temporal, ideally None
967
- ORG_KEY: uid,
968
- **data_sort_blueprint, # implies sv = True
969
- }
970
- identical = await self.storage.query(
971
- query,
972
- **identical_bp,
973
- # **data_sort_bp,
974
- )
923
+ monotonic_key = MONOTONIC_KEY # ??
924
+ grace_period = kw.get(GRACE_PERIOD_KEY, DEFAULT_GRACE_PERIOD)
925
+ grace_period *= 10**9 # nanoseconds
926
+ since_value = monotonic - grace_period
927
+
928
+ query = f"{namespace}://{database}/{thing}"
929
+
930
+ data_sort_blueprint = build_dict(data, sort_keys)
931
+ # data_sort_blueprint = build_comparisson_dict(data, reverse_sort_keys)
932
+ data_sort_bp = {
933
+ MONOTONIC_SINCE_KEY: monotonic_key,
934
+ MONOTONIC_SINCE_VALUE: since_value,
935
+ MONOTONIC_SINCE_OPERATOR: ">=",
936
+ ORDER_KEY: monotonic_key,
937
+ DIRECTION_KEY: DIRECTION_DESC,
938
+ # LIMIT_KEY: kw.get(
939
+ # LIMIT_KEY, 50 # TODO: agp: set in definition?
940
+ # ), # TODO: this is temporal, ideally None
941
+ # ORG_KEY: uid,
942
+ # **data_sort_blueprint, # implies sv = True
943
+ }
944
+ # TODO: LIMIT 1 ?
975
945
 
976
- # TODO: try to create only a single query
977
- # TODO: review different structures case
978
- similar_bp = {
979
- LIMIT_KEY: kw.get(
980
- LIMIT_KEY, 50 # TODO: agp: set in definition?
981
- ), # TODO: this is temporal, ideally None
982
- ORG_KEY: uid,
983
- **data_sort_blueprint, # implies sv = True
984
- }
985
- similar = await self.storage.query(
986
- query,
987
- **similar_bp,
988
- **data_sort_bp,
989
- )
990
- t1 = time.time()
991
- _elapsed = t1 - t0
992
- existing = identical + similar
993
- N = len(existing)
994
- log.debug(
995
- "[%s] found [%s] similar records in %s secs",
996
- identical_bp,
997
- N,
998
- _elapsed,
999
- )
1000
- if data_sort_blueprint and N > 1:
1001
- if behavior & ALLOW_DUPLICATED_ITEMS:
1002
- log.debug(
1003
- "tube [%s] has multiples records: [%s] records, but ALLOW_SAME_DATE_DIFFERENT_VALUES is defined",
1004
- uid,
1005
- N,
1006
- )
1007
- existing.clear()
1008
- else:
946
+ # MASK = set([ID_KEY, MONOTONIC_KEY, *sort_keys])
947
+ MASK = set([ID_KEY, MONOTONIC_KEY])
948
+
949
+ if not reverse_sort_keys and sort_keys:
950
+ MASK.update(sort_keys)
1009
951
  log.debug(
1010
- "tube has multiples records: [%s] = %s records, must just 1 and sort_key is defined by: [%s]",
1011
- uid,
1012
- N,
1013
- data_sort_blueprint,
952
+ "including sort_keys: [%s] in excluded MASK: [%s] to find similar registers",
953
+ sort_keys,
954
+ MASK,
1014
955
  )
1015
956
 
1016
- push = True
1017
- patterns = kw.get(COMPARISON_PATTERNS)
1018
- if patterns:
1019
- wdata = CWalk(data, include=patterns, exclude=MASK)
1020
- if not wdata:
1021
- log.warning("patterns don't get any data")
1022
- raise BadLogic(data)
1023
- else:
1024
- patterns = [r".*"]
1025
- wdata = CWalk(data, include=patterns, exclude=MASK)
1026
- for exists in existing:
1027
- wexists = CWalk(exists, include=patterns, exclude=MASK)
1028
- existing_sort_blueprint = build_dict(exists, reverse_sort_keys)
1029
- # existing_sort_blueprint = build_comparisson_dict(exists, reverse_sort_keys)
1030
-
1031
- same_sort_key = existing_sort_blueprint == data_sort_blueprint
1032
-
1033
- # check if we must "duplicate" data inside tube
1034
- # keys0 = set(exists).difference(MASK)
1035
- # keys1 = set(data).difference(MASK)
1036
- keys0 = set(wexists)
1037
- keys1 = set(wdata)
1038
- same_structure = keys0 == keys1
1039
-
1040
- same_values = False
1041
- if same_sort_key and same_structure:
1042
- for key in keys0:
1043
- if wexists[key] != wdata[key]:
1044
- log.debug(
1045
- "[%s].[%s].[%s]: %s != %s",
1046
- uid,
1047
- data_sort_blueprint,
1048
- key,
1049
- wexists[key],
1050
- wdata[key],
1051
- )
957
+ # TODO: agp: cache and get behaviour from database?
958
+ if not (behavior := self.behavior_uri.get(query)):
959
+ for (
960
+ pattern,
961
+ permissions,
962
+ ) in self.behavior_templates.items():
963
+ if re.match(pattern, query):
964
+ behavior = permissions
1052
965
  break
1053
966
  else:
1054
- same_values = True
967
+ behavior = ALL_RESTRICTIONS
968
+
969
+ self.behavior_uri[query] = behavior
970
+
971
+ t0 = time.time()
972
+ # search the same data
973
+ # TODO: update blueprint
974
+ identical_bp = {
975
+ LIMIT_KEY: kw.get(
976
+ LIMIT_KEY, 10 # TODO: agp: set in definition?
977
+ ), # TODO: this is temporal, ideally None
978
+ ORG_KEY: uid,
979
+ **data_sort_blueprint, # implies sv = True
980
+ }
981
+ identical = await self.storage.query(
982
+ query,
983
+ **identical_bp,
984
+ # **data_sort_bp,
985
+ )
1055
986
 
1056
- if not same_values:
987
+ # TODO: try to create only a single query
988
+ # TODO: review different structures case
989
+ similar_bp = {
990
+ LIMIT_KEY: kw.get(
991
+ LIMIT_KEY, 50 # TODO: agp: set in definition?
992
+ ), # TODO: this is temporal, ideally None
993
+ ORG_KEY: uid,
994
+ **data_sort_blueprint, # implies sv = True
995
+ }
996
+ similar = await self.storage.query(
997
+ query,
998
+ **similar_bp,
999
+ **data_sort_bp,
1000
+ )
1001
+ t1 = time.time()
1002
+ _elapsed = t1 - t0
1003
+ existing = identical + similar
1004
+ N = len(existing)
1005
+ log.debug(
1006
+ "[%s] found [%s] similar records in %s secs",
1007
+ identical_bp,
1008
+ N,
1009
+ _elapsed,
1010
+ )
1011
+ if data_sort_blueprint and N > 1:
1012
+ if behavior & ALLOW_DUPLICATED_ITEMS:
1013
+ log.debug(
1014
+ "tube [%s] has multiples records: [%s] records, but ALLOW_SAME_DATE_DIFFERENT_VALUES is defined",
1015
+ uid,
1016
+ N,
1017
+ )
1018
+ existing.clear()
1019
+ else:
1057
1020
  log.debug(
1058
- "[%s].sort_keys: %s",
1021
+ "tube has multiples records: [%s] = %s records, must just 1 and sort_key is defined by: [%s]",
1059
1022
  uid,
1023
+ N,
1060
1024
  data_sort_blueprint,
1061
1025
  )
1062
- log.debug("existing: %s", DATE(exists.get(MONOTONIC_KEY)))
1063
- log.debug("new data: %s", DATE(data.get(MONOTONIC_KEY)))
1064
- foo = 1
1026
+
1027
+ push = True
1028
+ patterns = kw.get(COMPARISON_PATTERNS)
1029
+ if patterns:
1030
+ wdata = CWalk(data, include=patterns, exclude=MASK)
1031
+ if not wdata:
1032
+ log.warning("patterns don't get any data")
1033
+ raise BadLogic(data)
1065
1034
  else:
1035
+ patterns = [r".*"]
1036
+ wdata = CWalk(data, include=patterns, exclude=MASK)
1037
+ for exists in existing:
1038
+ wexists = CWalk(exists, include=patterns, exclude=MASK)
1039
+ existing_sort_blueprint = build_dict(exists, reverse_sort_keys)
1040
+ # existing_sort_blueprint = build_comparisson_dict(exists, reverse_sort_keys)
1041
+
1042
+ same_sort_key = existing_sort_blueprint == data_sort_blueprint
1043
+
1044
+ # check if we must "duplicate" data inside tube
1045
+ # keys0 = set(exists).difference(MASK)
1046
+ # keys1 = set(data).difference(MASK)
1047
+ keys0 = set(wexists)
1048
+ keys1 = set(wdata)
1049
+ same_structure = keys0 == keys1
1050
+
1066
1051
  same_values = False
1052
+ if same_sort_key and same_structure:
1053
+ for key in keys0:
1054
+ if wexists[key] != wdata[key]:
1055
+ log.debug(
1056
+ "[%s].[%s].[%s]: %s != %s",
1057
+ uid,
1058
+ data_sort_blueprint,
1059
+ key,
1060
+ wexists[key],
1061
+ wdata[key],
1062
+ )
1063
+ break
1064
+ else:
1065
+ same_values = True
1067
1066
 
1068
- # explain why object will be skipped
1069
- if same_sort_key:
1070
- # same sort_key
1071
- if same_structure:
1072
- # EP preserver known structure
1073
- if same_values:
1074
- # new object and existing one are identical
1075
- # including `sort_keys`
1076
- # object is not inserted, continue with the next one
1067
+ if not same_values:
1077
1068
  log.debug(
1078
- "[%s][%s]: SKIP, new and existing are identical.",
1069
+ "[%s].sort_keys: %s",
1079
1070
  uid,
1080
1071
  data_sort_blueprint,
1081
1072
  )
1082
- push = False
1083
- break
1084
- elif data_sort_blueprint:
1085
- if behavior & ALLOW_SAME_DATE_DIFFERENT_VALUES:
1073
+ log.debug(
1074
+ "existing: %s", DATE(exists.get(MONOTONIC_KEY))
1075
+ )
1076
+ log.debug("new data: %s", DATE(data.get(MONOTONIC_KEY)))
1077
+ foo = 1
1078
+ else:
1079
+ same_values = False
1080
+
1081
+ # explain why object will be skipped
1082
+ if same_sort_key:
1083
+ # same sort_key
1084
+ if same_structure:
1085
+ # EP preserver known structure
1086
+ if same_values:
1087
+ # new object and existing one are identical
1088
+ # including `sort_keys`
1089
+ # object is not inserted, continue with the next one
1086
1090
  log.debug(
1087
- "[%s][%s], EP send a modified version of an already sent object, but behavior has ALLOW_SAME_DATE_DIFFERENT_VALUES, so restriction is RELAXED",
1091
+ "[%s][%s]: SKIP, new and existing are identical.",
1088
1092
  uid,
1089
1093
  data_sort_blueprint,
1090
1094
  )
1095
+ push = False
1096
+ break
1097
+ elif data_sort_blueprint:
1098
+ if behavior & ALLOW_SAME_DATE_DIFFERENT_VALUES:
1099
+ log.debug(
1100
+ "[%s][%s], EP send a modified version of an already sent object, but behavior has ALLOW_SAME_DATE_DIFFERENT_VALUES, so restriction is RELAXED",
1101
+ uid,
1102
+ data_sort_blueprint,
1103
+ )
1104
+ else:
1105
+ # but EP modified an already sent object
1106
+ log.error(
1107
+ "[%s][%s], EP send a modified version of an already sent object",
1108
+ uid,
1109
+ data_sort_blueprint,
1110
+ )
1111
+ push = False
1112
+ break
1091
1113
  else:
1092
1114
  # but EP modified an already sent object
1093
- log.error(
1094
- "[%s][%s], EP send a modified version of an already sent object",
1115
+ log.debug(
1116
+ "OK [%s] EP send a modified version of an already sent object, but data has't sort_keys, so must be inserted each time data is different",
1095
1117
  uid,
1096
- data_sort_blueprint,
1097
1118
  )
1098
- push = False
1099
- break
1119
+ foo = 1
1120
+
1100
1121
  else:
1101
- # but EP modified an already sent object
1122
+ # and EP yield more/less data for same object
1123
+ log.debug(
1124
+ "[%s].[%s], EP send a different structure that previous ones",
1125
+ uid,
1126
+ data_sort_blueprint,
1127
+ )
1102
1128
  log.debug(
1103
- "OK [%s] EP send a modified version of an already sent object, but data has't sort_keys, so must be inserted each time data is different",
1129
+ "[%s].[%s]: existing: [%s]",
1104
1130
  uid,
1131
+ data_sort_blueprint,
1132
+ exists,
1133
+ )
1134
+ log.debug(
1135
+ "[%s].[%s]: new : [%s]",
1136
+ uid,
1137
+ data_sort_blueprint,
1138
+ data,
1105
1139
  )
1106
1140
  foo = 1
1107
1141
 
1108
1142
  else:
1109
- # and EP yield more/less data for same object
1110
- log.debug(
1111
- "[%s].[%s], EP send a different structure that previous ones",
1112
- uid,
1113
- data_sort_blueprint,
1114
- )
1115
- log.debug(
1116
- "[%s].[%s]: existing: [%s]",
1117
- uid,
1118
- data_sort_blueprint,
1119
- exists,
1120
- )
1121
- log.debug(
1122
- "[%s].[%s]: new : [%s]",
1123
- uid,
1124
- data_sort_blueprint,
1125
- data,
1126
- )
1127
- foo = 1
1128
-
1129
- else:
1130
- # sort_key values differs
1131
- if same_structure:
1132
- # struct doesn't change
1133
- if same_values:
1134
- # data is unchanged but the `sort_keys`
1143
+ # sort_key values differs
1144
+ if same_structure:
1145
+ # struct doesn't change
1146
+ if same_values:
1147
+ # data is unchanged but the `sort_keys`
1148
+ log.error(
1149
+ "[%s]: data is unchanged but the `sort_keys`: %s <--> %s",
1150
+ uid,
1151
+ data_sort_blueprint,
1152
+ existing_sort_blueprint,
1153
+ )
1154
+ push = False
1155
+ break
1156
+ else:
1157
+ # a new update of the same object
1158
+ pass
1159
+ # push = True
1160
+ else:
1161
+ # a new object with a change in its structure
1162
+ # push = True
1135
1163
  log.error(
1136
- "[%s]: data is unchanged but the `sort_keys`: %s <--> %s",
1164
+ "[%s].[%s]: has change its structure: [%s]",
1137
1165
  uid,
1138
- data_sort_blueprint,
1139
- existing_sort_blueprint,
1166
+ keys0.symmetric_difference(keys1),
1140
1167
  )
1141
- push = False
1142
- break
1143
- else:
1144
- # a new update of the same object
1145
- pass
1146
- # push = True
1147
- else:
1148
- # a new object with a change in its structure
1149
- # push = True
1150
- log.error(
1151
- "[%s].[%s]: has change its structure: [%s]",
1152
- uid,
1153
- keys0.symmetric_difference(keys1),
1154
- )
1155
- foo = 1
1168
+ foo = 1
1169
+ else:
1170
+ pass
1171
+ # push = True
1172
+
1173
+ # TODO: hack to speed up the process for some data
1174
+ # TODO: remove when not needed
1175
+ must_check = kw.get(KIND_KEY) not in ("raw_energy",)
1176
+ if must_check:
1177
+ t0 = time.time()
1178
+ await prevously_inserted()
1179
+ elapsed = time.time() - t0
1180
+ log.info("[%s] prevously_inserted took: %s secs", uid, elapsed)
1156
1181
  else:
1182
+ # hack for not altering the data
1183
+ # push = False
1157
1184
  pass
1158
- # push = True
1159
1185
 
1160
1186
  # check if ORG_KEY could be formated wrong
1161
1187
  if _uri["id"] is None:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: syncmodels
3
- Version: 0.1.317
3
+ Version: 0.1.319
4
4
  Summary: Synchronizable Models
5
5
  Home-page: https://github.com/asterio.gonzalez/syncmodels
6
6
  Author: Asterio Gonzalez
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.11
18
18
  Requires-Python: >=3.6
19
19
  License-File: LICENSE
20
20
  License-File: AUTHORS.rst
21
- Requires-Dist: agptools>=0.1.317
21
+ Requires-Dist: agptools>=0.1.319
22
22
  Requires-Dist: aiocache
23
23
  Requires-Dist: aiohttp
24
24
  Requires-Dist: Click
@@ -1,4 +1,4 @@
1
- syncmodels/__init__.py,sha256=Z7wOaIpC5ZfxOzCzYQg9-0sPjUEYzhd6DPDh_s7sHd0,142
1
+ syncmodels/__init__.py,sha256=VBs43e71g7bJb2zmdoJDqxCfvbKnCG6kATlaZ78DvpA,142
2
2
  syncmodels/context.py,sha256=k1Gs_ip9BfyRFpyRnzqYvRDKo0sYBqJsh6z9sWln9oE,451
3
3
  syncmodels/crawler.py,sha256=FthNzipF8cafNvogFC7AFn_N4qxOHkJTeo3Lbcw0P_g,94746
4
4
  syncmodels/crud.py,sha256=ozumS7XgmXSFcFN2SZBH0jB0j_1vK2xE-FeFcTG7ikw,15327
@@ -11,7 +11,7 @@ syncmodels/registry.py,sha256=YaQtgbSwa0je1MpCcVHALI3_b85vrddyOlhsnrUcKZs,8224
11
11
  syncmodels/requests.py,sha256=wWoC5hPDm1iBM_zrlyKRauzhXgdKR3pT5RqyC-5UZhQ,538
12
12
  syncmodels/runner.py,sha256=IHDKuQ3yJ1DN9wktMiIrerPepYX61tc3AzbFfuUqEFw,5454
13
13
  syncmodels/schema.py,sha256=uinUt8Asq_x7xa6MKWVXNyoWO6gKocjGPppjimaXzEU,2492
14
- syncmodels/storage.py,sha256=invDyQMfDHavQH5Fnbi7afSeIgH6NoVtD_LlNPOqSTI,71646
14
+ syncmodels/storage.py,sha256=HjyJ62Fxz7Ot-UlS3atQ-7sP6FD6BoddLsnKboUjkgE,73696
15
15
  syncmodels/syncmodels.py,sha256=jcUxVbv1hrx5hI81VCO1onIM6WyORTqJVPwIqlPocOc,10596
16
16
  syncmodels/timequeue.py,sha256=YRd3ULRaIhoszaBsYhfr0epMqAbL6-NwVEtScjUYttM,595
17
17
  syncmodels/wave.py,sha256=Gra22BLiA9z2nF-6diXpjAc4GZv9nebmyvHxdAfXec4,7764
@@ -32,13 +32,13 @@ syncmodels/helpers/geojson.py,sha256=9VZjdEw19fLS_Q9topapUOPGxRg2D4GOqm2tlx11W7M
32
32
  syncmodels/helpers/importers.py,sha256=KImR9pQu4ir6EI6Ipta0q3RWloFT_VTJi67kM0lZsKQ,3919
33
33
  syncmodels/helpers/loaders.py,sha256=aus0aRcbU1vVa_zWo42aX6uV3B0fQ0aQpkTWlR9xGLA,4325
34
34
  syncmodels/helpers/models.py,sha256=c_ATzmiw5mVY1IGnwmyhjIuu5d2idHU-XeRigZSMkOQ,719
35
- syncmodels/helpers/orion.py,sha256=Lblk4OGRNidYl1-2lZKUPXyvuR1hXip7FHloXLtU-XA,30717
35
+ syncmodels/helpers/orion.py,sha256=6lRp1w3yaq_rxOI7nJIjuHdsgBjQu92y0bW0IX_gq44,30719
36
36
  syncmodels/helpers/surreal.py,sha256=zoWtGm5oAxwvgJNq_NTpKOHN3h9FNObhFDLuiBOl1YY,10050
37
37
  syncmodels/helpers/units.py,sha256=g50m5DQrAyP_qpDRa4LCEA5Rz2UZUmlIixfWG_ddw9I,3571
38
38
  syncmodels/logic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
39
39
  syncmodels/logic/activity_logger.py,sha256=8wjvgRwaNbibYWGgl-trovSS70yNkoCTlb-AIx3aZEE,14053
40
- syncmodels/logic/analyzer.py,sha256=OiRZBJoqjc_qb3w1jBjGftWgd20Cig0s7_GLTCbp3fw,12539
41
- syncmodels/logic/browser.py,sha256=vMxGaYIR1ov0tOMbWNpd4MMFQBunRuvxE8F-Gfo9cTo,84795
40
+ syncmodels/logic/analyzer.py,sha256=AJncaSGlgKKG2X-Y9-6_phBZkIz4CGWYLI2JSKqLhhI,12542
41
+ syncmodels/logic/browser.py,sha256=e0LdpHMmBaJ7cMdCFdIBLEfpewvO_mc5wqiCyNYPNoc,85197
42
42
  syncmodels/logic/swarm.py,sha256=eRBVlNAOzzWKFGCb7LGLx2aj7yQlTY1OwLoeSEllvXY,17207
43
43
  syncmodels/mapper/__init__.py,sha256=jS82LFr9zzyqXBz82tSw04vDowhTpKxhg_W2XvhUlt0,129
44
44
  syncmodels/mapper/fiware.py,sha256=auszPmhCS46z_68MXjksrQAFUfctjbVrVdBvOpOkMj8,523
@@ -302,10 +302,10 @@ syncmodels/session/postgresql.py,sha256=ZMIu1Rv93pKfvFlovFBmWArzlrT2xaQWNYGZT_LW
302
302
  syncmodels/session/sql.py,sha256=bD7zXRrEKKJmqY2UoibWENuWb5zHrrU72F3_dYbS6LY,6569
303
303
  syncmodels/session/sqlite.py,sha256=nCDjopLiBpX1F10qkKoARM7JrVdIpJ1WdGOduFVxaiA,2080
304
304
  syncmodels/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
305
- syncmodels-0.1.317.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
306
- syncmodels-0.1.317.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
307
- syncmodels-0.1.317.dist-info/METADATA,sha256=ysR8CVg8FxFsfK0SohKSX68OCKdwnD1by-vAw9D3tiw,2700
308
- syncmodels-0.1.317.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
309
- syncmodels-0.1.317.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
310
- syncmodels-0.1.317.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
311
- syncmodels-0.1.317.dist-info/RECORD,,
305
+ syncmodels-0.1.319.dist-info/AUTHORS.rst,sha256=3ZPoqg8Aav8DSYKd0fwcwn4_5HwSiMLart0E5Un00-U,168
306
+ syncmodels-0.1.319.dist-info/LICENSE,sha256=uzMOYtIiUsnsD0xHJR7aJWJ4v_bvan0kTnvufy5eNoA,1075
307
+ syncmodels-0.1.319.dist-info/METADATA,sha256=P-Fou_OhAgBSEp7YQNUlsdcilVoAcNAeHFZXax5VGHc,2700
308
+ syncmodels-0.1.319.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
309
+ syncmodels-0.1.319.dist-info/entry_points.txt,sha256=dMnigjZsHMxTwXiiZyBZdBbMYE0-hY3L5cG15EcDAzw,51
310
+ syncmodels-0.1.319.dist-info/top_level.txt,sha256=2DfQ9NuAhKMjY3BvQGVBA7GfqTm7EoHNbaehSUiqiHQ,11
311
+ syncmodels-0.1.319.dist-info/RECORD,,