velocity-python 0.10.2__py3-none-any.whl → 0.12.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. velocity/__init__.py +1 -1
  2. velocity/aws/authz.py +20 -2
  3. velocity/aws/dirty_pipeline.py +18 -4
  4. velocity/aws/handlers/base_handler.py +43 -0
  5. velocity/aws/handlers/context.py +15 -0
  6. velocity/aws/handlers/lambda_handler.py +37 -0
  7. velocity/aws/handlers/mixins/__init__.py +5 -0
  8. velocity/aws/handlers/mixins/data_service.py +42 -7
  9. velocity/aws/handlers/mixins/import_actions.py +681 -0
  10. velocity/aws/handlers/mixins/web_handler.py +141 -6
  11. velocity/aws/handlers/sqs_handler.py +68 -1
  12. velocity/aws/runtime/queue_actions.py +4 -0
  13. velocity/aws/runtime/tick.py +63 -0
  14. velocity/aws/scheduler.py +22 -1
  15. velocity/aws/schema.py +13 -1
  16. velocity/aws/tenancy.py +592 -0
  17. velocity/cache.py +19 -0
  18. velocity/db/_pgddl.py +15 -0
  19. velocity/db/audit.py +24 -1
  20. velocity/db/core/column.py +48 -6
  21. velocity/db/core/database.py +93 -26
  22. velocity/db/core/decorators.py +41 -19
  23. velocity/db/core/engine.py +492 -105
  24. velocity/db/core/lazy_engine.py +13 -4
  25. velocity/db/core/result.py +69 -3
  26. velocity/db/core/row.py +133 -24
  27. velocity/db/core/sequence.py +72 -44
  28. velocity/db/core/table.py +541 -150
  29. velocity/db/core/tenant.py +698 -0
  30. velocity/db/core/transaction.py +976 -56
  31. velocity/db/core/view.py +26 -1
  32. velocity/db/effective.py +2 -1
  33. velocity/db/exceptions.py +19 -0
  34. velocity/db/integrity.py +6 -0
  35. velocity/db/lane/cli.py +58 -0
  36. velocity/db/lane/ensure.py +29 -10
  37. velocity/db/lane/ledger.py +23 -6
  38. velocity/db/lane/tenants.py +250 -0
  39. velocity/db/maintenance.py +125 -15
  40. velocity/db/servers/mysql/operators.py +2 -3
  41. velocity/db/servers/mysql/sql.py +1334 -251
  42. velocity/db/servers/postgres/__init__.py +60 -0
  43. velocity/db/servers/postgres/reserved.py +122 -1
  44. velocity/db/servers/postgres/sql.py +210 -78
  45. velocity/db/servers/sqlite/__init__.py +114 -1
  46. velocity/db/servers/sqlite/operators.py +2 -2
  47. velocity/db/servers/sqlite/sql.py +886 -201
  48. velocity/db/servers/sqlserver/sql.py +11 -0
  49. velocity/db/servers/tablehelper.py +109 -7
  50. velocity/db/sync.py +5 -0
  51. velocity/db/tenancy.py +920 -0
  52. velocity/db/tests/postgres/test_tenancy.py +1150 -0
  53. velocity/db/utils.py +33 -8
  54. velocity/misc/staged_import/__init__.py +191 -0
  55. velocity/misc/staged_import/batches.py +1335 -0
  56. velocity/misc/staged_import/files.py +547 -0
  57. velocity/misc/staged_import/spec.py +618 -0
  58. velocity/testing/fake_db.py +25 -25
  59. {velocity_python-0.10.2.dist-info → velocity_python-0.12.0.dist-info}/METADATA +1 -1
  60. {velocity_python-0.10.2.dist-info → velocity_python-0.12.0.dist-info}/RECORD +64 -54
  61. {velocity_python-0.10.2.dist-info → velocity_python-0.12.0.dist-info}/WHEEL +0 -0
  62. {velocity_python-0.10.2.dist-info → velocity_python-0.12.0.dist-info}/entry_points.txt +0 -0
  63. {velocity_python-0.10.2.dist-info → velocity_python-0.12.0.dist-info}/licenses/LICENSE +0 -0
  64. {velocity_python-0.10.2.dist-info → velocity_python-0.12.0.dist-info}/top_level.txt +0 -0
velocity/__init__.py CHANGED
@@ -1,4 +1,4 @@
1
- __version__ = version = "0.10.2"
1
+ __version__ = version = "0.12.0"
2
2
 
3
3
  import importlib as _importlib
4
4
 
velocity/aws/authz.py CHANGED
@@ -947,8 +947,26 @@ def protect_tables(*names):
947
947
  _PROTECTED.update(name for name in names if isinstance(name, str) and name)
948
948
 
949
949
 
950
- def is_protected(table_name):
951
- return table_name in _PROTECTED
950
+ # Tables that hold code or decide what runs. In an installation without
951
+ # tenants an operator edits them through the generic Data actions, as
952
+ # always; with tenants a client write there would let one tenant's admin
953
+ # plant work that runs with the installation's rights.
954
+ _TENANT_PROTECTED = frozenset({"program_objects", "sys_task_triggers"})
955
+
956
+
957
+ def is_protected(table_name, handler=None):
958
+ """Whether generic Data actions refuse client writes to the table.
959
+ ``handler`` (optional): the handler asking; one with a
960
+ ``tenant_resolver`` protects the program tables whatever is ambient."""
961
+ if table_name in _PROTECTED:
962
+ return True
963
+ if table_name in _TENANT_PROTECTED:
964
+ if getattr(handler, "tenant_resolver", None) is not None:
965
+ return True
966
+ from velocity.db.core import tenant as _tenant_scope
967
+
968
+ return _tenant_scope.current_scope() is not None
969
+ return False
952
970
 
953
971
 
954
972
  # Tables whose rows are confidential (the security audit: everyone's sign-ins,
@@ -179,7 +179,19 @@ def get_excluded_tables(tx, context) -> set[str]:
179
179
  }
180
180
 
181
181
 
182
- def list_tables_with_sys_dirty(tx, *, schema: str = "public") -> list[str]:
182
+ def _dirty_schema(tx, schema) -> str:
183
+ """The schema the dirty helpers look in: the one named, else the
184
+ transaction's tenant schema, else ``public``."""
185
+ if schema:
186
+ return schema
187
+ from velocity.db.core.tenant import TenantScope
188
+
189
+ scope = getattr(tx, "_tenant", None)
190
+ return scope.schema if isinstance(scope, TenantScope) else "public"
191
+
192
+
193
+ def list_tables_with_sys_dirty(tx, *, schema: str = None) -> list[str]:
194
+ schema = _dirty_schema(tx, schema)
183
195
  sql = """
184
196
  SELECT table_name
185
197
  FROM information_schema.columns
@@ -211,12 +223,13 @@ def _sql_helper():
211
223
  return SQL
212
224
 
213
225
 
214
- def dirty_index_state(tx, *, schema: str = "public") -> dict:
226
+ def dirty_index_state(tx, *, schema: str = None) -> dict:
215
227
  """``{table: "ok" | "missing" | "invalid"}`` for every table carrying
216
228
  ``sys_dirty``: whether the sweep's partial index
217
229
  (``(sys_modified) WHERE sys_dirty = TRUE``) is there and usable. An
218
230
  ``invalid`` one is what an interrupted ``CREATE INDEX CONCURRENTLY``
219
231
  leaves behind."""
232
+ schema = _dirty_schema(tx, schema)
220
233
  sql_helper = _sql_helper()
221
234
  tables = list_tables_with_sys_dirty(tx, schema=schema)
222
235
  names = {table: sql_helper.sys_dirty_index_name(schema, table) for table in tables}
@@ -235,7 +248,7 @@ def dirty_index_state(tx, *, schema: str = "public") -> dict:
235
248
  }
236
249
 
237
250
 
238
- def ensure_dirty_indexes(tx, *, schema: str = "public", out=None) -> dict:
251
+ def ensure_dirty_indexes(tx, *, schema: str = None, out=None) -> dict:
239
252
  """Create the sweep's partial index on every ``sys_dirty`` table that
240
253
  lacks it (rebuilding an invalid one), ``CONCURRENTLY`` so a large table
241
254
  keeps taking writes. Each statement runs standalone in autocommit
@@ -248,6 +261,7 @@ def ensure_dirty_indexes(tx, *, schema: str = "public", out=None) -> dict:
248
261
  raise exceptions.DbSchemaLockedError(
249
262
  "Cannot create the sys_dirty indexes: schema is locked."
250
263
  )
264
+ schema = _dirty_schema(tx, schema)
251
265
  sql_helper = _sql_helper()
252
266
  done = {}
253
267
  for table, state in dirty_index_state(tx, schema=schema).items():
@@ -524,7 +538,7 @@ def sweep_stale_dirty_rows(
524
538
  context,
525
539
  *,
526
540
  tables: Optional[Iterable[str]] = None,
527
- schema: str = "public",
541
+ schema: str = None,
528
542
  limit: int = DEFAULT_SWEEP_LIMIT,
529
543
  stale_minutes: int = DEFAULT_STALE_MINUTES,
530
544
  requeue_minutes: Optional[int] = None,
@@ -28,6 +28,19 @@ class BaseHandler:
28
28
  and error handling hooks that can be shared across different handler types.
29
29
  """
30
30
 
31
+ # Multi-tenant installations (velocity.aws.tenancy): an object that maps
32
+ # the caller, a queue message's tenant key, or "every active tenant" to
33
+ # a TenantScope. None (the default) turns all tenant handling off.
34
+ tenant_resolver = None
35
+ # The TenantScope of the invocation being served (None outside one).
36
+ current_tenant = None
37
+ # With a resolver: the queue actions that are the installation's own.
38
+ # Only these may be enqueued, or run from a queue record, with no tenant.
39
+ system_actions = ()
40
+ # With a resolver: the engine refuses statements outside a tenant scope
41
+ # (engine.require_tenant()). False opts out.
42
+ tenant_required = True
43
+
31
44
  def __init__(
32
45
  self,
33
46
  aws_event: Dict[str, Any],
@@ -52,6 +65,12 @@ class BaseHandler:
52
65
  self.context_factory = context_factory
53
66
  self.context = None
54
67
  self._update_lambda_ca_certificates()
68
+ if getattr(self, "tenant_resolver", None) is not None:
69
+ # Fail closed at start: a tenant handler that cannot keep
70
+ # tenants apart is never served (velocity.aws.tenancy).
71
+ from velocity.aws import tenancy as _tenancy
72
+
73
+ _tenancy.check_handler(self)
55
74
 
56
75
  def create_context(self, *, args, postdata, response, session):
57
76
  self.context = self.context_factory.create(
@@ -62,6 +81,17 @@ class BaseHandler:
62
81
  response=response,
63
82
  session=session,
64
83
  )
84
+ if getattr(self, "tenant_resolver", None) is not None:
85
+ # What this invocation enqueues carries its tenant, or is one of
86
+ # the declared system actions (Context.enqueue refuses the rest).
87
+ from velocity.aws import tenancy as _tenancy
88
+
89
+ try:
90
+ self.context._tenant_system_actions = _tenancy.system_actions_of(self)
91
+ except Exception as error:
92
+ raise _tenancy.TenantConfigurationError(
93
+ "The context of a tenant handler must accept its enqueue " "policy."
94
+ ) from error
65
95
  return self.context
66
96
 
67
97
  def _update_lambda_ca_certificates(self):
@@ -415,6 +445,19 @@ class BaseHandler:
415
445
  if "job_id" not in fields and isinstance(fields.get("payload"), dict):
416
446
  fields["job_id"] = fields["payload"].get("job_id")
417
447
  fields.setdefault("source", self.resolve_failure_source())
448
+ if getattr(self, "tenant_resolver", None) is not None or (
449
+ type(exception).__name__ == "TenantJobRefused"
450
+ ):
451
+ # The failure sink is shared by every tenant: it gets the
452
+ # tenant key and the payload's field names, never its values.
453
+ from velocity.aws import tenancy as _tenancy
454
+
455
+ fields["payload"] = _tenancy.withheld_payload(
456
+ fields.get("payload"), getattr(self, "current_tenant", None)
457
+ )
458
+ # ... and the exception's type and SQLSTATE, not its message
459
+ # (a driver's DETAIL line quotes row values).
460
+ exception = _tenancy.redacted_exception(exception)
418
461
  failures.report(self.aws_context, exception, **fields)
419
462
  except Exception:
420
463
  logger.warning("Failure reporting failed", exc_info=True)
@@ -19,6 +19,7 @@ from velocity.aws.ssm_config import get_stage as config_get_stage
19
19
  from velocity.logging import get_logger
20
20
  from velocity.cache import memo
21
21
  from velocity.db.core.lazy_engine import LazyEngine
22
+ from velocity.db.core import tenant as _tenant_scope
22
23
  from velocity.db import exceptions as db_exceptions
23
24
  from velocity.aws.job_lifecycle import (
24
25
  CLAIM_COLUMN,
@@ -852,8 +853,22 @@ class Context:
852
853
 
853
854
  entries = []
854
855
  job_rows = []
856
+ # Inside a tenant scope the message says whose work it is; the worker
857
+ # re-enters that tenant before running it (velocity.aws.tenancy).
858
+ tenant_key = _tenant_scope.message_key(transaction)
859
+ system_actions = getattr(self, "_tenant_system_actions", None)
860
+ if system_actions is not None and tenant_key is None:
861
+ # A tenant handler: work with no tenant must be declared.
862
+ if action not in system_actions:
863
+ raise db_exceptions.DbTenantError(
864
+ f"enqueue({action!r}) outside a tenant: on a handler with "
865
+ "a tenant_resolver a message carries its tenant, or its "
866
+ "action is listed in the handler's system_actions."
867
+ )
855
868
  for item in payload:
856
869
  message = {"action": action, "payload": item}
870
+ if tenant_key is not None:
871
+ message["tenant"] = tenant_key
857
872
  # SEC-7: a full UUID — job ids are bearer handles for the job
858
873
  # status/download/cancel actions, so they must not be guessable.
859
874
  id = str(uuid.uuid4())
@@ -90,6 +90,20 @@ class LambdaHandler(BaseHandler):
90
90
  )
91
91
 
92
92
  def beforeAction(self, tx, context):
93
+ if getattr(self, "tenant_resolver", None) is None:
94
+ return self._before_action(tx, context)
95
+ # Until the tenant is known the request runs in one named privileged
96
+ # scope; it ends at resolution, or here when there is no tenant
97
+ # (velocity.aws.tenancy).
98
+ from velocity.aws import tenancy as _tenancy
99
+
100
+ _tenancy.begin_pre_resolution(self, tx)
101
+ try:
102
+ return self._before_action(tx, context)
103
+ finally:
104
+ _tenancy.end_pre_resolution(self)
105
+
106
+ def _before_action(self, tx, context):
93
107
  # Enhanced activity tracking is supplied by the WebHandler mixin; a bare
94
108
  # LambdaHandler (no WebHandler) simply skips it rather than crashing.
95
109
  enhanced = getattr(self, "_enhanced_before_action", None)
@@ -199,6 +213,19 @@ class LambdaHandler(BaseHandler):
199
213
 
200
214
  self._record_identity_mode(session, masquerade)
201
215
 
216
+ if getattr(self, "tenant_resolver", None) is not None:
217
+ # The one choke point: the caller's tenant is resolved here, once,
218
+ # before the user row is read or any action runs; the rest of the
219
+ # request runs as that tenant (velocity.aws.tenancy).
220
+ from velocity.aws import tenancy as _tenancy
221
+
222
+ _tenancy.enter_request_scope(self, tx, context, session, masquerade)
223
+ # Activity tracking starts now, where the request belongs: in
224
+ # the tenant's schema, or (no tenant) in the shared table.
225
+ start = getattr(self, "_start_deferred_activity", None)
226
+ if callable(start):
227
+ start(tx, context)
228
+
202
229
  logger.info(
203
230
  "Starting action",
204
231
  extra={
@@ -488,6 +515,16 @@ class LambdaHandler(BaseHandler):
488
515
  logger.debug("ending handler.onError done")
489
516
 
490
517
  def serve(self, tx):
518
+ if getattr(self, "tenant_resolver", None) is None:
519
+ return self._serve_request(tx)
520
+ # A tenant resolved by this request never outlives it, and one left
521
+ # by an earlier invocation on this container never reaches it.
522
+ from velocity.aws import tenancy as _tenancy
523
+
524
+ with _tenancy.invocation_boundary(self, tx):
525
+ return self._serve_request(tx)
526
+
527
+ def _serve_request(self, tx):
491
528
  response = Response()
492
529
  req_params = self.aws_event.get("queryStringParameters") or {}
493
530
  local_context = self.create_context(
@@ -22,6 +22,8 @@ This package provides mixins for common Lambda handler patterns:
22
22
  notices; and the queue side (program jobs, outbox drain, timers)
23
23
  - SchedulerMixin: run-schedules, the once-a-minute tick over sys_schedules
24
24
  (velocity.aws.scheduler)
25
+ - ImportMixin: the import-* actions (template, upload, preview, apply,
26
+ report, batches) over velocity.misc.staged_import
25
27
  """
26
28
 
27
29
  from .web_handler import WebHandler, ButtonHandler
@@ -39,6 +41,7 @@ from .action_names import action_names
39
41
  from .permissions_actions import PERMISSION_ACTIONS, PermissionsMixin
40
42
  from .tiles_actions import TILE_ACTIONS, TilesMixin
41
43
  from .analytics_actions import ANALYTICS_ACTIONS, AnalyticsMixin
44
+ from .import_actions import IMPORT_ACTIONS, ImportMixin
42
45
  from .workflow_actions import (
43
46
  WORKFLOW_ACTIONS,
44
47
  WORKFLOW_QUEUE_ACTIONS,
@@ -66,6 +69,8 @@ __all__ = [
66
69
  "TILE_ACTIONS",
67
70
  "AnalyticsMixin",
68
71
  "ANALYTICS_ACTIONS",
72
+ "ImportMixin",
73
+ "IMPORT_ACTIONS",
69
74
  "WorkflowMixin",
70
75
  "WorkflowQueueMixin",
71
76
  "WORKFLOW_ACTIONS",
@@ -148,6 +148,15 @@ def _bump_sys_id_sequence(tx, table):
148
148
  )
149
149
 
150
150
 
151
+ def _table_schema(tx):
152
+ """The schema a bare table name means for ``tx``: its tenant's schema
153
+ (velocity.db.tenancy) when it runs in one, else ``public``."""
154
+ from velocity.db.core.tenant import TenantScope
155
+
156
+ scope = getattr(tx, "_tenant", None)
157
+ return scope.schema if isinstance(scope, TenantScope) else "public"
158
+
159
+
151
160
  def _sys_id_fk_references(tx, table):
152
161
  """Foreign keys pointing AT this table's sys_id, as
153
162
  (referencing_table, referencing_column) pairs. A multi-column FK yields
@@ -162,12 +171,12 @@ def _sys_id_fk_references(tx, table):
162
171
  from pg_constraint con
163
172
  join pg_class rel on rel.oid = con.confrelid
164
173
  join pg_namespace nsp on nsp.oid = rel.relnamespace
165
- where con.contype = 'f' and nsp.nspname = 'public'
174
+ where con.contype = 'f' and nsp.nspname = %s
166
175
  and rel.relname = %s
167
176
  and (select att.attname from pg_attribute att
168
177
  where att.attrelid = con.confrelid
169
178
  and att.attnum = con.confkey[1]) = 'sys_id'""",
170
- (table,),
179
+ (_table_schema(tx), table),
171
180
  ).all()
172
181
 
173
182
 
@@ -1146,7 +1155,30 @@ class DataServiceMixin:
1146
1155
  except Exception:
1147
1156
  raw = None
1148
1157
  mode = str(raw or self.client_sql_mode or STRICT).strip().lower()
1149
- return mode if mode in MODES else STRICT
1158
+ mode = mode if mode in MODES else STRICT
1159
+ if mode != STRICT:
1160
+ self._refuse_legacy_sql_in_a_tenant()
1161
+ return mode
1162
+
1163
+ def _refuse_legacy_sql_in_a_tenant(self):
1164
+ """The legacy client SQL mode hands client-written SQL fragments to
1165
+ the database. In a multi-tenant installation SQL is how a session
1166
+ leaves its tenant, so the mode is refused outright: whenever this
1167
+ handler has a ``tenant_resolver``, and whenever a tenant scope is
1168
+ in effect (velocity.aws.tenancy)."""
1169
+ from velocity.db.core import tenant as _tenant_scope
1170
+
1171
+ if (
1172
+ getattr(self, "tenant_resolver", None) is None
1173
+ and _tenant_scope.current_scope() is None
1174
+ ):
1175
+ return
1176
+ from velocity.aws.tenancy import TenantConfigurationError
1177
+
1178
+ raise TenantConfigurationError(
1179
+ "The legacy client SQL mode (DataClientSqlMode / client_sql_mode) "
1180
+ "is not available with tenants: only the strict mode is."
1181
+ )
1150
1182
 
1151
1183
  def _client_guard(self):
1152
1184
  return ClientInputGuard(self._client_sql_mode())
@@ -1194,7 +1226,7 @@ class DataServiceMixin:
1194
1226
  # grant tables, the tile tables — ``authz.protect_tables``): no
1195
1227
  # generic action writes them, whatever the lists above or
1196
1228
  # ``authz_mode`` say.
1197
- if self._table_operation != authz.DATA_READ and authz.is_protected(table):
1229
+ if self._table_operation != authz.DATA_READ and authz.is_protected(table, self):
1198
1230
  return False
1199
1231
  # Confidential tables (``authz.confidential_tables``: the security
1200
1232
  # audit) are read through generic Data only where the handler opted
@@ -1398,9 +1430,12 @@ class DataServiceMixin:
1398
1430
  handler defines it, otherwise ``OnActionDefault`` (a missing or unknown
1399
1431
  action, or a raw ``OnAction*`` name, which dispatch does not pass
1400
1432
  through)."""
1433
+ # Resolved for every request: with tenants, anything but the strict
1434
+ # mode is refused here, before authentication or any action.
1435
+ mode = self.client_sql_mode_effective
1401
1436
  setter = getattr(context, "set_client_sql_mode", None)
1402
1437
  if callable(setter):
1403
- setter(self.client_sql_mode_effective)
1438
+ setter(mode)
1404
1439
  allowed = self.client_actions
1405
1440
  if allowed is not None:
1406
1441
  if isinstance(allowed, str):
@@ -3827,11 +3862,11 @@ class DataServiceMixin:
3827
3862
  ordinal_position
3828
3863
  FROM information_schema.columns
3829
3864
  WHERE table_name = %s
3830
- AND table_schema = 'public'
3865
+ AND table_schema = %s
3831
3866
  ORDER BY ordinal_position
3832
3867
  """
3833
3868
 
3834
- schema_data = tx.execute(schema_query, [table_name])
3869
+ schema_data = tx.execute(schema_query, [table_name, _table_schema(tx)])
3835
3870
 
3836
3871
  if not schema_data:
3837
3872
  raise ValueError(