supertable 2.0.2__tar.gz → 2.0.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- supertable-2.0.3/LICENSE +103 -0
- {supertable-2.0.2 → supertable-2.0.3}/PKG-INFO +26 -4
- {supertable-2.0.2 → supertable-2.0.3}/README.md +25 -3
- {supertable-2.0.2 → supertable-2.0.3}/pyproject.toml +1 -1
- {supertable-2.0.2 → supertable-2.0.3}/setup.py +1 -1
- {supertable-2.0.2 → supertable-2.0.3}/supertable/__init__.py +1 -1
- supertable-2.0.3/supertable/audit/admin.py +206 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/logger.py +85 -22
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/writer_redis.py +5 -11
- {supertable-2.0.2 → supertable-2.0.3}/supertable/config/settings.py +5 -2
- {supertable-2.0.2 → supertable-2.0.3}/supertable/data_writer.py +3 -2
- {supertable-2.0.2 → supertable-2.0.3}/supertable/meta_reader.py +6 -5
- {supertable-2.0.2 → supertable-2.0.3}/supertable/monitoring_writer.py +4 -1
- {supertable-2.0.2 → supertable-2.0.3}/supertable/rbac/role_manager.py +3 -2
- {supertable-2.0.2 → supertable-2.0.3}/supertable/rbac/tests/test_rbac.py +38 -38
- {supertable-2.0.2 → supertable-2.0.3}/supertable/rbac/user_manager.py +2 -1
- {supertable-2.0.2 → supertable-2.0.3}/supertable/redis_catalog.py +157 -249
- {supertable-2.0.2 → supertable-2.0.3}/supertable/redis_infra.py +15 -25
- supertable-2.0.3/supertable/redis_keys.py +286 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/service_registry.py +10 -8
- {supertable-2.0.2 → supertable-2.0.3}/supertable/staging_area.py +2 -1
- {supertable-2.0.2 → supertable-2.0.3}/supertable/super_pipe.py +2 -1
- {supertable-2.0.2 → supertable-2.0.3}/supertable/tests/test_dedup_on_read_write.py +2 -2
- supertable-2.0.3/supertable/tests/test_redis_key_prefix.py +164 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable.egg-info/PKG-INFO +26 -4
- {supertable-2.0.2 → supertable-2.0.3}/supertable.egg-info/SOURCES.txt +3 -0
- supertable-2.0.2/LICENSE +0 -178
- {supertable-2.0.2 → supertable-2.0.3}/requirements.txt +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/setup.cfg +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/chain.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/consumers.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/crypto.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/events.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/export.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/middleware.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/reader.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/retention.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/tests/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/tests/test_chain.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/tests/test_crypto.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/tests/test_emit.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/tests/test_events.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/tests/test_retention.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/audit/writer_parquet.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/config/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/config/defaults.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/config/homedir.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/config/tests/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/config/tests/test_defaults.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/config/tests/test_homedir.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/config/tests/test_settings.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/data_classes.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/data_reader.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/__main__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/check_filter_builder.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/controller.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/data_writer_helpers.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/defaults.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/dummy_data.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/read_parquet_header.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s01_01_01_create_super_table.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s01_01_02_enable_mirroring_formats.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s01_02_create_roles.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s01_03_create_users.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s02_01_write_dummy_data.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s02_02_write_single_data.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s02_03_01_write_staging.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s02_03_02_create_pipe.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s02_04_01_write_monitoring_simple.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s02_04_02_write_monitoring_parallel.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s02_05_write_tombstone.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s03_01_read_data_error.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s03_02_01_read_super_data_ok.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s03_02_02_read_table_data_ok.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s03_03_read_meta.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s03_04_read_staging.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s03_06_01_read_roles.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s03_06_02_read_user.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s03_07_01_estimate_read.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s03_07_02_estimate_files.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s03_08_read_snapshot_history.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s04_01_03_delete_pipe.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s05_01_delete_table.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/quickstart/s05_02_delete_super_table.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/webshop/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/webshop/core.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/webshop/defaults.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/webshop/generate.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/webshop/load.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/demo/webshop/topup.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/engine/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/engine/data_estimator.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/engine/duckdb_lite.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/engine/duckdb_pro.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/engine/engine_common.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/engine/engine_enum.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/engine/executor.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/engine/plan_stats.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/engine/spark_thrift.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/engine/tests/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/engine/tests/conftest.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/engine/tests/test_dedup_read.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/engine/tests/test_engine.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/locking/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/locking/benchmarks/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/locking/benchmarks/benchmark_locking.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/locking/benchmarks/measure_lock_speed.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/locking/benchmarks/measure_lock_time.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/locking/file_lock.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/locking/redis_lock.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/locking/tests/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/locking/tests/test_file_lock.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/locking/tests/test_redis_lock.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/logging.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/mirroring/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/mirroring/mirror_delta.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/mirroring/mirror_formats.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/mirroring/mirror_iceberg.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/mirroring/mirror_parquet.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/plan_extender.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/processing.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/query_plan_manager.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/rbac/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/rbac/access_control.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/rbac/filter_builder.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/rbac/permissions.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/rbac/row_column_security.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/rbac/tests/test_filter_builder.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/rbac/tests/test_rbac_per_table.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/redis_connector.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/simple_table.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/storage/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/storage/azure_storage.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/storage/gcp_storage.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/storage/local_storage.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/storage/minio_storage.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/storage/s3_storage.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/storage/storage_factory.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/storage/storage_interface.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/storage/tests/test_storage.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/super_table.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/tests/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/tests/test_data_reader.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/tests/test_data_writer.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/tests/test_data_writer_comprehensive.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/tests/test_data_writer_tombstones.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/tests/test_meta_reader.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/tests/test_newer_than.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/tests/test_process_delete_only.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/tests/test_processing.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/tests/test_query_sql.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/tests/test_simple_table.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/tests/test_small_file_compaction.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/tests/test_super_table.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/tests/test_supertable_all.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/utils/__init__.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/utils/helper.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/utils/sql_parser.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/utils/tests/test_sql_parser_columns.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable/utils/timer.py +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable.egg-info/dependency_links.txt +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable.egg-info/entry_points.txt +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable.egg-info/requires.txt +0 -0
- {supertable-2.0.2 → supertable-2.0.3}/supertable.egg-info/top_level.txt +0 -0
supertable-2.0.3/LICENSE
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# Functional Source License, Version 1.1, ALv2 Future License
|
|
2
|
+
|
|
3
|
+
## Abbreviation
|
|
4
|
+
|
|
5
|
+
FSL-1.1-ALv2
|
|
6
|
+
|
|
7
|
+
## Notice
|
|
8
|
+
|
|
9
|
+
Copyright 2024-2026 Kladna Soft Kft.
|
|
10
|
+
|
|
11
|
+
## Terms and Conditions
|
|
12
|
+
|
|
13
|
+
### Licensor ("We")
|
|
14
|
+
|
|
15
|
+
The party offering the Software under these Terms and Conditions.
|
|
16
|
+
|
|
17
|
+
### The Software
|
|
18
|
+
|
|
19
|
+
The "Software" is each version of the software that we make available under
|
|
20
|
+
these Terms and Conditions, as indicated by our inclusion of these Terms and
|
|
21
|
+
Conditions with the Software.
|
|
22
|
+
|
|
23
|
+
### License Grant
|
|
24
|
+
|
|
25
|
+
Subject to your compliance with this License Grant and the Patents,
|
|
26
|
+
Redistribution and Trademark clauses below, we hereby grant you the right to
|
|
27
|
+
use, copy, modify, create derivative works, publicly perform, publicly display
|
|
28
|
+
and redistribute the Software for any Permitted Purpose identified below.
|
|
29
|
+
|
|
30
|
+
### Permitted Purpose
|
|
31
|
+
|
|
32
|
+
A Permitted Purpose is any purpose other than a Competing Use. A Competing Use
|
|
33
|
+
means making the Software available to others in a commercial product or
|
|
34
|
+
service that:
|
|
35
|
+
|
|
36
|
+
1. substitutes for the Software;
|
|
37
|
+
2. substitutes for any other product or service we offer using the Software
|
|
38
|
+
that exists as of the date we make the Software available; or
|
|
39
|
+
3. offers the same or substantially similar functionality as the Software.
|
|
40
|
+
|
|
41
|
+
Permitted Purposes specifically include using the Software:
|
|
42
|
+
|
|
43
|
+
1. for your internal use and access;
|
|
44
|
+
2. for non-commercial education;
|
|
45
|
+
3. for non-commercial research; and
|
|
46
|
+
4. in connection with professional services that you provide to a licensee
|
|
47
|
+
using the Software in accordance with these Terms and Conditions.
|
|
48
|
+
|
|
49
|
+
### Patents
|
|
50
|
+
|
|
51
|
+
To the extent your use for a Permitted Purpose would necessarily infringe our
|
|
52
|
+
patents, the license grant above includes a license under our patents. If you
|
|
53
|
+
make a claim against any party that the Software infringes or contributes to
|
|
54
|
+
the infringement of any patent, then your patent license to the Software ends
|
|
55
|
+
immediately.
|
|
56
|
+
|
|
57
|
+
### Redistribution
|
|
58
|
+
|
|
59
|
+
The Terms and Conditions apply to all copies, modifications and derivatives of
|
|
60
|
+
the Software.
|
|
61
|
+
|
|
62
|
+
If you redistribute any copies, modifications or derivatives of the Software,
|
|
63
|
+
you must include a copy of or a link to these Terms and Conditions and not
|
|
64
|
+
remove any copyright notices provided in or with the Software.
|
|
65
|
+
|
|
66
|
+
### Disclaimer
|
|
67
|
+
|
|
68
|
+
THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR
|
|
69
|
+
IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR
|
|
70
|
+
PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT.
|
|
71
|
+
|
|
72
|
+
IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE
|
|
73
|
+
SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES,
|
|
74
|
+
EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE.
|
|
75
|
+
|
|
76
|
+
### Trademarks
|
|
77
|
+
|
|
78
|
+
Except for displaying the License Details and identifying us as the origin of
|
|
79
|
+
the Software, you have no right under these Terms and Conditions to use our
|
|
80
|
+
trademarks, trade names, service marks or product names.
|
|
81
|
+
|
|
82
|
+
## Grant of Future License
|
|
83
|
+
|
|
84
|
+
We hereby irrevocably grant you an additional license to use the Software under
|
|
85
|
+
the Apache License, Version 2.0 that is effective on the second anniversary of
|
|
86
|
+
the date we make the Software available.
|
|
87
|
+
|
|
88
|
+
On or after that date, you may use the Software under the Apache License,
|
|
89
|
+
Version 2.0, in which case the following will apply:
|
|
90
|
+
|
|
91
|
+
Licensed under the Apache License, Version 2.0 (the "License"); you may not use
|
|
92
|
+
this file except in compliance with the License.
|
|
93
|
+
|
|
94
|
+
You may obtain a copy of the License at:
|
|
95
|
+
|
|
96
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
97
|
+
|
|
98
|
+
Unless required by applicable law or agreed to in writing, software distributed
|
|
99
|
+
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
|
100
|
+
CONDITIONS OF ANY KIND, either express or implied.
|
|
101
|
+
|
|
102
|
+
See the License for the specific language governing permissions and limitations
|
|
103
|
+
under the License.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: supertable
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.3
|
|
4
4
|
Summary: SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.
|
|
5
5
|
Author: Levente Kupas
|
|
6
6
|
Author-email: Levente Kupas <lkupas@kladnasoft.com>
|
|
@@ -44,7 +44,7 @@ Dynamic: requires-python
|
|
|
44
44
|
# SuperTable
|
|
45
45
|
|
|
46
46
|

|
|
47
|
-

|
|
48
48
|

|
|
49
49
|
|
|
50
50
|
**SuperTable — versioned data lake library for SQL analytics.**
|
|
@@ -214,6 +214,28 @@ See [docs/00_index.md](docs/00_index.md) for the full table of contents.
|
|
|
214
214
|
|
|
215
215
|
## License
|
|
216
216
|
|
|
217
|
-
|
|
217
|
+
SuperTable is licensed under the **Functional Source License, Version 1.1,
|
|
218
|
+
ALv2 Future License** (`FSL-1.1-ALv2`).
|
|
218
219
|
|
|
219
|
-
|
|
220
|
+
You may use, copy, modify, create derivative works, publicly perform, publicly
|
|
221
|
+
display, and redistribute the software for any permitted purpose other than a
|
|
222
|
+
**Competing Use**.
|
|
223
|
+
|
|
224
|
+
A **Competing Use** means making the software available to others in a
|
|
225
|
+
commercial product or service that:
|
|
226
|
+
|
|
227
|
+
1. substitutes for SuperTable;
|
|
228
|
+
2. substitutes for another product or service offered by Kladna Soft Kft. using
|
|
229
|
+
SuperTable; or
|
|
230
|
+
3. offers the same or substantially similar functionality as SuperTable.
|
|
231
|
+
|
|
232
|
+
Permitted purposes include internal use, non-commercial education,
|
|
233
|
+
non-commercial research, and professional services provided to a licensee using
|
|
234
|
+
the software in accordance with the license.
|
|
235
|
+
|
|
236
|
+
Each version of the software becomes available under the **Apache License 2.0**
|
|
237
|
+
on the second anniversary of the date that version is made available.
|
|
238
|
+
|
|
239
|
+
See [LICENSE](LICENSE) for the full license terms.
|
|
240
|
+
|
|
241
|
+
Copyright © 2024-2026 Kladna Soft Kft.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# SuperTable
|
|
2
2
|
|
|
3
3
|

|
|
4
|
-

|
|
5
5
|

|
|
6
6
|
|
|
7
7
|
**SuperTable — versioned data lake library for SQL analytics.**
|
|
@@ -171,6 +171,28 @@ See [docs/00_index.md](docs/00_index.md) for the full table of contents.
|
|
|
171
171
|
|
|
172
172
|
## License
|
|
173
173
|
|
|
174
|
-
|
|
174
|
+
SuperTable is licensed under the **Functional Source License, Version 1.1,
|
|
175
|
+
ALv2 Future License** (`FSL-1.1-ALv2`).
|
|
175
176
|
|
|
176
|
-
|
|
177
|
+
You may use, copy, modify, create derivative works, publicly perform, publicly
|
|
178
|
+
display, and redistribute the software for any permitted purpose other than a
|
|
179
|
+
**Competing Use**.
|
|
180
|
+
|
|
181
|
+
A **Competing Use** means making the software available to others in a
|
|
182
|
+
commercial product or service that:
|
|
183
|
+
|
|
184
|
+
1. substitutes for SuperTable;
|
|
185
|
+
2. substitutes for another product or service offered by Kladna Soft Kft. using
|
|
186
|
+
SuperTable; or
|
|
187
|
+
3. offers the same or substantially similar functionality as SuperTable.
|
|
188
|
+
|
|
189
|
+
Permitted purposes include internal use, non-commercial education,
|
|
190
|
+
non-commercial research, and professional services provided to a licensee using
|
|
191
|
+
the software in accordance with the license.
|
|
192
|
+
|
|
193
|
+
Each version of the software becomes available under the **Apache License 2.0**
|
|
194
|
+
on the second anniversary of the date that version is made available.
|
|
195
|
+
|
|
196
|
+
See [LICENSE](LICENSE) for the full license terms.
|
|
197
|
+
|
|
198
|
+
Copyright © 2024-2026 Kladna Soft Kft.
|
|
@@ -19,7 +19,7 @@ long_description = readme.read_text(encoding="utf-8") if readme.exists() else ""
|
|
|
19
19
|
|
|
20
20
|
setup(
|
|
21
21
|
name="supertable",
|
|
22
|
-
version="2.0.
|
|
22
|
+
version="2.0.3",
|
|
23
23
|
description="SuperTable — versioned data lake library for SQL analytics on Parquet + Redis.",
|
|
24
24
|
long_description=long_description,
|
|
25
25
|
long_description_content_type="text/markdown",
|
|
@@ -25,7 +25,7 @@ See the ``supertable.demo`` package for runnable end-to-end demos and the
|
|
|
25
25
|
project documentation for the full API surface.
|
|
26
26
|
"""
|
|
27
27
|
|
|
28
|
-
__version__ = "2.0.
|
|
28
|
+
__version__ = "2.0.3"
|
|
29
29
|
|
|
30
30
|
# Re-export the core public surface so users can do ``from supertable import …``
|
|
31
31
|
# instead of remembering submodule paths.
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
# route: supertable.audit.admin
|
|
2
|
+
"""
|
|
3
|
+
Per-organization runtime audit configuration.
|
|
4
|
+
|
|
5
|
+
The Compliance tab in the WebUI (/ui/audit) calls this module via the API
|
|
6
|
+
to read and update per-org audit toggles WITHOUT restarting the process.
|
|
7
|
+
|
|
8
|
+
Persistence
|
|
9
|
+
-----------
|
|
10
|
+
Stored in Redis as a HASH at:
|
|
11
|
+
|
|
12
|
+
supertable:{org}:audit:config
|
|
13
|
+
|
|
14
|
+
Fields
|
|
15
|
+
------
|
|
16
|
+
enabled "true" | "false" master on/off switch (default: env)
|
|
17
|
+
log_queries "true" | "false" record DATA_ACCESS query events
|
|
18
|
+
log_reads "true" | "false" record DATA_ACCESS read events
|
|
19
|
+
hash_chain "true" | "false" tamper-evident hash chaining
|
|
20
|
+
siem_enabled "true" | "false" external SIEM consumer groups
|
|
21
|
+
updated_ms str(int) last update timestamp
|
|
22
|
+
updated_by str actor (username) who toggled
|
|
23
|
+
|
|
24
|
+
Audit-of-the-audit
|
|
25
|
+
------------------
|
|
26
|
+
Every change to this config emits a CONFIG_CHANGE audit event so that
|
|
27
|
+
turning audit OFF is itself recorded — DORA Art. 6 / SOC 2 CC8.1.
|
|
28
|
+
|
|
29
|
+
Compliance: DORA Art. 6 (information security), SOC 2 CC8.1 (change mgmt).
|
|
30
|
+
"""
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import logging
|
|
34
|
+
import time
|
|
35
|
+
from typing import Any, Dict, Optional
|
|
36
|
+
|
|
37
|
+
from supertable import redis_keys as RK
|
|
38
|
+
|
|
39
|
+
logger = logging.getLogger(__name__)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# ---------------------------------------------------------------------------
|
|
43
|
+
# Field schema
|
|
44
|
+
# ---------------------------------------------------------------------------
|
|
45
|
+
|
|
46
|
+
# Boolean fields and their env-var defaults (resolved lazily so settings is
|
|
47
|
+
# only imported when actually needed; avoids import-time side effects).
|
|
48
|
+
_BOOL_FIELDS = (
|
|
49
|
+
"enabled",
|
|
50
|
+
"log_queries",
|
|
51
|
+
"log_reads",
|
|
52
|
+
"hash_chain",
|
|
53
|
+
"siem_enabled",
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
_ENV_DEFAULTS = {
|
|
57
|
+
"enabled": "SUPERTABLE_AUDIT_ENABLED",
|
|
58
|
+
"log_queries": "SUPERTABLE_AUDIT_LOG_QUERIES",
|
|
59
|
+
"log_reads": "SUPERTABLE_AUDIT_LOG_READS",
|
|
60
|
+
"hash_chain": "SUPERTABLE_AUDIT_HASH_CHAIN",
|
|
61
|
+
"siem_enabled": "SUPERTABLE_AUDIT_SIEM_ENABLED",
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _env_default(field: str) -> bool:
|
|
66
|
+
"""Read the env-var default for *field*."""
|
|
67
|
+
from supertable.config.settings import settings as _cfg
|
|
68
|
+
attr = _ENV_DEFAULTS.get(field)
|
|
69
|
+
if not attr:
|
|
70
|
+
return False
|
|
71
|
+
return bool(getattr(_cfg, attr, False))
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _coerce_bool(v: Any) -> Optional[bool]:
|
|
75
|
+
if isinstance(v, bool):
|
|
76
|
+
return v
|
|
77
|
+
if v is None:
|
|
78
|
+
return None
|
|
79
|
+
s = str(v).strip().lower()
|
|
80
|
+
if s in ("1", "true", "yes", "y", "on"):
|
|
81
|
+
return True
|
|
82
|
+
if s in ("0", "false", "no", "n", "off"):
|
|
83
|
+
return False
|
|
84
|
+
return None
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _redis():
|
|
88
|
+
"""Lazy Redis handle. Centralizes the import for testability."""
|
|
89
|
+
from supertable.redis_catalog import RedisCatalog
|
|
90
|
+
return RedisCatalog().r
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
# ---------------------------------------------------------------------------
|
|
94
|
+
# Read
|
|
95
|
+
# ---------------------------------------------------------------------------
|
|
96
|
+
|
|
97
|
+
def get_audit_config(org: str) -> Dict[str, Any]:
|
|
98
|
+
"""Return the effective audit config for *org*.
|
|
99
|
+
|
|
100
|
+
Merges Redis overrides over env-var defaults. Always returns every
|
|
101
|
+
boolean field with a concrete True/False value plus updated_ms /
|
|
102
|
+
updated_by (or empty defaults).
|
|
103
|
+
"""
|
|
104
|
+
out: Dict[str, Any] = {field: _env_default(field) for field in _BOOL_FIELDS}
|
|
105
|
+
out["updated_ms"] = 0
|
|
106
|
+
out["updated_by"] = ""
|
|
107
|
+
|
|
108
|
+
if not org:
|
|
109
|
+
return out
|
|
110
|
+
|
|
111
|
+
try:
|
|
112
|
+
raw = _redis().hgetall(RK.audit_config(org)) or {}
|
|
113
|
+
except Exception as e: # pragma: no cover — non-fatal
|
|
114
|
+
logger.warning("[audit-admin] get_audit_config redis error: %s", e)
|
|
115
|
+
return out
|
|
116
|
+
|
|
117
|
+
for field in _BOOL_FIELDS:
|
|
118
|
+
if field in raw:
|
|
119
|
+
v = _coerce_bool(raw[field])
|
|
120
|
+
if v is not None:
|
|
121
|
+
out[field] = v
|
|
122
|
+
if "updated_ms" in raw:
|
|
123
|
+
try:
|
|
124
|
+
out["updated_ms"] = int(raw["updated_ms"])
|
|
125
|
+
except (TypeError, ValueError):
|
|
126
|
+
pass
|
|
127
|
+
if "updated_by" in raw:
|
|
128
|
+
out["updated_by"] = str(raw["updated_by"])
|
|
129
|
+
|
|
130
|
+
return out
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def is_audit_enabled(org: str) -> bool:
|
|
134
|
+
"""Convenience helper used by the audit logger."""
|
|
135
|
+
return bool(get_audit_config(org).get("enabled", False))
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
# ---------------------------------------------------------------------------
|
|
139
|
+
# Write
|
|
140
|
+
# ---------------------------------------------------------------------------
|
|
141
|
+
|
|
142
|
+
def set_audit_config(
|
|
143
|
+
org: str,
|
|
144
|
+
*,
|
|
145
|
+
enabled: Optional[bool] = None,
|
|
146
|
+
log_queries: Optional[bool] = None,
|
|
147
|
+
log_reads: Optional[bool] = None,
|
|
148
|
+
hash_chain: Optional[bool] = None,
|
|
149
|
+
siem_enabled: Optional[bool] = None,
|
|
150
|
+
updated_by: str = "",
|
|
151
|
+
) -> Dict[str, Any]:
|
|
152
|
+
"""Update per-org audit config. Only fields that are not None are written.
|
|
153
|
+
|
|
154
|
+
Returns the new effective config (post-merge).
|
|
155
|
+
"""
|
|
156
|
+
if not org:
|
|
157
|
+
raise ValueError("organization is required")
|
|
158
|
+
|
|
159
|
+
incoming: Dict[str, Any] = {
|
|
160
|
+
"enabled": enabled,
|
|
161
|
+
"log_queries": log_queries,
|
|
162
|
+
"log_reads": log_reads,
|
|
163
|
+
"hash_chain": hash_chain,
|
|
164
|
+
"siem_enabled": siem_enabled,
|
|
165
|
+
}
|
|
166
|
+
mapping: Dict[str, str] = {}
|
|
167
|
+
for field, val in incoming.items():
|
|
168
|
+
if val is None:
|
|
169
|
+
continue
|
|
170
|
+
mapping[field] = "true" if bool(val) else "false"
|
|
171
|
+
|
|
172
|
+
now_ms = int(time.time() * 1000)
|
|
173
|
+
mapping["updated_ms"] = str(now_ms)
|
|
174
|
+
if updated_by:
|
|
175
|
+
mapping["updated_by"] = str(updated_by)
|
|
176
|
+
|
|
177
|
+
try:
|
|
178
|
+
_redis().hset(RK.audit_config(org), mapping=mapping)
|
|
179
|
+
except Exception as e:
|
|
180
|
+
logger.error("[audit-admin] set_audit_config redis error: %s", e)
|
|
181
|
+
raise
|
|
182
|
+
|
|
183
|
+
new_cfg = get_audit_config(org)
|
|
184
|
+
|
|
185
|
+
# Emit a CONFIG_CHANGE audit event for the change itself.
|
|
186
|
+
# Use a try/except so a misconfigured audit subsystem cannot block the
|
|
187
|
+
# config write that just succeeded.
|
|
188
|
+
try:
|
|
189
|
+
from supertable.audit import emit, EventCategory, Actions, Severity, make_detail
|
|
190
|
+
# Build a detail string showing only the fields that were touched.
|
|
191
|
+
touched = {k: v for k, v in incoming.items() if v is not None}
|
|
192
|
+
emit(
|
|
193
|
+
category=EventCategory.CONFIG_CHANGE,
|
|
194
|
+
action=getattr(Actions, "CONFIG_UPDATE", "config.update"),
|
|
195
|
+
organization=org,
|
|
196
|
+
actor_username=updated_by or "",
|
|
197
|
+
actor_id="",
|
|
198
|
+
resource_type="audit_config",
|
|
199
|
+
resource_id="audit:config",
|
|
200
|
+
detail=make_detail(**{k: ("true" if bool(v) else "false") for k, v in touched.items()}),
|
|
201
|
+
severity=Severity.WARNING,
|
|
202
|
+
)
|
|
203
|
+
except Exception as e: # pragma: no cover — non-fatal
|
|
204
|
+
logger.debug("[audit-admin] config-change audit emit failed: %s", e)
|
|
205
|
+
|
|
206
|
+
return new_cfg
|
|
@@ -35,17 +35,19 @@ logger = logging.getLogger(__name__)
|
|
|
35
35
|
|
|
36
36
|
@dataclass
|
|
37
37
|
class AuditConfig:
|
|
38
|
-
|
|
38
|
+
# Default to OFF. Real values come from env (from_settings) and are
|
|
39
|
+
# overridden per-organization via the Redis-backed admin layer.
|
|
40
|
+
enabled: bool = False
|
|
39
41
|
batch_size: int = 1000
|
|
40
42
|
flush_interval_sec: int = 60
|
|
41
43
|
redis_stream_ttl_hours: int = 24
|
|
42
44
|
redis_stream_maxlen: int = 100_000
|
|
43
|
-
hash_chain: bool =
|
|
44
|
-
log_queries: bool =
|
|
45
|
-
log_reads: bool =
|
|
45
|
+
hash_chain: bool = False
|
|
46
|
+
log_queries: bool = False
|
|
47
|
+
log_reads: bool = False
|
|
46
48
|
alert_webhook: str = ""
|
|
47
49
|
fernet_key: str = ""
|
|
48
|
-
siem_enabled: bool =
|
|
50
|
+
siem_enabled: bool = False
|
|
49
51
|
siem_max_consumers: int = 10
|
|
50
52
|
|
|
51
53
|
@classmethod
|
|
@@ -53,22 +55,38 @@ class AuditConfig:
|
|
|
53
55
|
try:
|
|
54
56
|
from supertable.config.settings import settings as _cfg
|
|
55
57
|
return cls(
|
|
56
|
-
enabled=getattr(_cfg, "SUPERTABLE_AUDIT_ENABLED",
|
|
58
|
+
enabled=getattr(_cfg, "SUPERTABLE_AUDIT_ENABLED", False),
|
|
57
59
|
batch_size=getattr(_cfg, "SUPERTABLE_AUDIT_BATCH_SIZE", 1000),
|
|
58
60
|
flush_interval_sec=getattr(_cfg, "SUPERTABLE_AUDIT_FLUSH_INTERVAL_SEC", 60),
|
|
59
61
|
redis_stream_ttl_hours=getattr(_cfg, "SUPERTABLE_AUDIT_REDIS_STREAM_TTL_HOURS", 24),
|
|
60
62
|
redis_stream_maxlen=getattr(_cfg, "SUPERTABLE_AUDIT_REDIS_STREAM_MAXLEN", 100_000),
|
|
61
|
-
hash_chain=getattr(_cfg, "SUPERTABLE_AUDIT_HASH_CHAIN",
|
|
62
|
-
log_queries=getattr(_cfg, "SUPERTABLE_AUDIT_LOG_QUERIES",
|
|
63
|
-
log_reads=getattr(_cfg, "SUPERTABLE_AUDIT_LOG_READS",
|
|
63
|
+
hash_chain=getattr(_cfg, "SUPERTABLE_AUDIT_HASH_CHAIN", False),
|
|
64
|
+
log_queries=getattr(_cfg, "SUPERTABLE_AUDIT_LOG_QUERIES", False),
|
|
65
|
+
log_reads=getattr(_cfg, "SUPERTABLE_AUDIT_LOG_READS", False),
|
|
64
66
|
alert_webhook=getattr(_cfg, "SUPERTABLE_AUDIT_ALERT_WEBHOOK", ""),
|
|
65
67
|
fernet_key=getattr(_cfg, "SUPERTABLE_AUDIT_FERNET_KEY", ""),
|
|
66
|
-
siem_enabled=getattr(_cfg, "SUPERTABLE_AUDIT_SIEM_ENABLED",
|
|
68
|
+
siem_enabled=getattr(_cfg, "SUPERTABLE_AUDIT_SIEM_ENABLED", False),
|
|
67
69
|
siem_max_consumers=getattr(_cfg, "SUPERTABLE_AUDIT_SIEM_MAX_CONSUMERS", 10),
|
|
68
70
|
)
|
|
69
71
|
except Exception:
|
|
70
72
|
return cls()
|
|
71
73
|
|
|
74
|
+
def with_overrides(self, overrides: Dict[str, Any]) -> "AuditConfig":
|
|
75
|
+
"""Return a copy with select fields replaced from a dict (e.g., from
|
|
76
|
+
the Redis-backed audit:config HASH). Unknown keys are ignored."""
|
|
77
|
+
from dataclasses import replace
|
|
78
|
+
kw: Dict[str, Any] = {}
|
|
79
|
+
for k in (
|
|
80
|
+
"enabled",
|
|
81
|
+
"hash_chain",
|
|
82
|
+
"log_queries",
|
|
83
|
+
"log_reads",
|
|
84
|
+
"siem_enabled",
|
|
85
|
+
):
|
|
86
|
+
if k in overrides and overrides[k] is not None:
|
|
87
|
+
kw[k] = bool(overrides[k])
|
|
88
|
+
return replace(self, **kw) if kw else self
|
|
89
|
+
|
|
72
90
|
|
|
73
91
|
# ---------------------------------------------------------------------------
|
|
74
92
|
# NullAuditLogger (no-op when auditing is disabled)
|
|
@@ -386,32 +404,77 @@ class AuditLogger:
|
|
|
386
404
|
# Singleton cache (one logger per organization)
|
|
387
405
|
# ---------------------------------------------------------------------------
|
|
388
406
|
|
|
389
|
-
_LOGGERS: Dict[str, AuditLogger] = {}
|
|
407
|
+
_LOGGERS: Dict[str, "AuditLogger | NullAuditLogger"] = {}
|
|
390
408
|
_LOGGERS_LOCK = threading.Lock()
|
|
391
|
-
_CONFIG: Optional[AuditConfig] = None
|
|
392
409
|
|
|
410
|
+
# Per-org config cache: org → (config, expires_at_seconds).
|
|
411
|
+
# Resolved against env defaults + Redis override (supertable:{org}:audit:config).
|
|
412
|
+
_ORG_CFG_CACHE: Dict[str, "tuple[AuditConfig, float]"] = {}
|
|
413
|
+
_ORG_CFG_TTL_S: float = 30.0 # toggle takes effect within this many seconds
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _resolve_config_for(organization: str) -> AuditConfig:
|
|
417
|
+
"""Resolve the effective AuditConfig for *organization*.
|
|
418
|
+
|
|
419
|
+
Merges the Redis override at ``supertable:{org}:audit:config`` over the
|
|
420
|
+
env-var defaults. Cached for _ORG_CFG_TTL_S seconds.
|
|
421
|
+
"""
|
|
422
|
+
now = time.time()
|
|
423
|
+
cached = _ORG_CFG_CACHE.get(organization)
|
|
424
|
+
if cached is not None and cached[1] > now:
|
|
425
|
+
return cached[0]
|
|
393
426
|
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
427
|
+
base = AuditConfig.from_settings()
|
|
428
|
+
try:
|
|
429
|
+
from supertable.audit.admin import get_audit_config as _get_redis_cfg
|
|
430
|
+
overrides = _get_redis_cfg(organization)
|
|
431
|
+
except Exception:
|
|
432
|
+
overrides = {}
|
|
433
|
+
|
|
434
|
+
cfg = base.with_overrides(overrides) if overrides else base
|
|
435
|
+
_ORG_CFG_CACHE[organization] = (cfg, now + _ORG_CFG_TTL_S)
|
|
436
|
+
return cfg
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def invalidate_audit_config_cache(organization: Optional[str] = None) -> None:
|
|
440
|
+
"""Drop the cached config for *organization* (or all orgs). Called by the
|
|
441
|
+
admin endpoint after a toggle so the change takes effect immediately."""
|
|
442
|
+
if organization is None:
|
|
443
|
+
_ORG_CFG_CACHE.clear()
|
|
444
|
+
else:
|
|
445
|
+
_ORG_CFG_CACHE.pop(organization, None)
|
|
399
446
|
|
|
400
447
|
|
|
401
448
|
def get_audit_logger(organization: str) -> "AuditLogger | NullAuditLogger":
|
|
402
449
|
"""Return a cached AuditLogger for the organization.
|
|
403
450
|
|
|
404
|
-
Thread-safe.
|
|
451
|
+
Thread-safe. When auditing is disabled (env default OFF, or per-org
|
|
452
|
+
override set to ``enabled=false``), returns a NullAuditLogger. When the
|
|
453
|
+
toggle is flipped from ON→OFF, the previously-running real logger is
|
|
454
|
+
stopped and replaced with a NullAuditLogger on the next call.
|
|
405
455
|
"""
|
|
406
|
-
config =
|
|
407
|
-
if not config.enabled:
|
|
408
|
-
return NullAuditLogger()
|
|
456
|
+
config = _resolve_config_for(organization)
|
|
409
457
|
|
|
410
458
|
with _LOGGERS_LOCK:
|
|
411
459
|
existing = _LOGGERS.get(organization)
|
|
412
|
-
|
|
460
|
+
|
|
461
|
+
if not config.enabled:
|
|
462
|
+
# If we previously had a real logger, drain & stop it.
|
|
463
|
+
if existing is not None and not isinstance(existing, NullAuditLogger):
|
|
464
|
+
try:
|
|
465
|
+
existing.stop()
|
|
466
|
+
except Exception as e:
|
|
467
|
+
logger.warning("[audit] graceful stop failed for %s: %s", organization, e)
|
|
468
|
+
_LOGGERS[organization] = NullAuditLogger()
|
|
469
|
+
elif existing is None:
|
|
470
|
+
_LOGGERS[organization] = NullAuditLogger()
|
|
471
|
+
return _LOGGERS[organization]
|
|
472
|
+
|
|
473
|
+
# Audit is enabled. Reuse a real logger if we have one.
|
|
474
|
+
if existing is not None and not isinstance(existing, NullAuditLogger):
|
|
413
475
|
return existing
|
|
414
476
|
|
|
477
|
+
# Either no logger or the cached one was a Null — create a real one.
|
|
415
478
|
audit_logger = AuditLogger(organization, config)
|
|
416
479
|
_LOGGERS[organization] = audit_logger
|
|
417
480
|
return audit_logger
|
|
@@ -21,21 +21,15 @@ import logging
|
|
|
21
21
|
import time
|
|
22
22
|
from typing import Any, Dict, List, Optional, Tuple
|
|
23
23
|
|
|
24
|
+
from supertable import redis_keys as RK
|
|
25
|
+
|
|
24
26
|
logger = logging.getLogger(__name__)
|
|
25
27
|
|
|
26
28
|
|
|
27
29
|
# ---------------------------------------------------------------------------
|
|
28
|
-
# Key helpers
|
|
30
|
+
# Key helpers — delegate to supertable.redis_keys for namespace consistency
|
|
29
31
|
# ---------------------------------------------------------------------------
|
|
30
32
|
|
|
31
|
-
def _stream_key(org: str) -> str:
|
|
32
|
-
return f"supertable:{org}:audit:stream"
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
def _chain_head_key(org: str, instance_id: str) -> str:
|
|
36
|
-
return f"supertable:{org}:audit:chain_head:{instance_id}"
|
|
37
|
-
|
|
38
|
-
|
|
39
33
|
ARCHIVAL_GROUP = "__archival__"
|
|
40
34
|
|
|
41
35
|
|
|
@@ -55,8 +49,8 @@ class RedisAuditWriter:
|
|
|
55
49
|
self._org = org
|
|
56
50
|
self._instance_id = instance_id
|
|
57
51
|
self._maxlen = maxlen
|
|
58
|
-
self._stream =
|
|
59
|
-
self._chain_key =
|
|
52
|
+
self._stream = RK.audit_stream(org)
|
|
53
|
+
self._chain_key = RK.audit_chain_head(org, instance_id)
|
|
60
54
|
self._ensure_stream()
|
|
61
55
|
|
|
62
56
|
def _ensure_stream(self) -> None:
|
|
@@ -260,7 +260,10 @@ class Settings:
|
|
|
260
260
|
SUPERTABLE_SUPER_META_CACHE_TTL_S: Optional[float] = None # SUPERTABLE_SUPER_META_CACHE_TTL_S
|
|
261
261
|
|
|
262
262
|
# ── Audit ────────────────────────────────────────────────────────
|
|
263
|
-
|
|
263
|
+
# Audit is OFF by default. Enable per-organization in the WebUI
|
|
264
|
+
# /ui/audit → Compliance tab (persisted at supertable:{org}:audit:config),
|
|
265
|
+
# or globally via the SUPERTABLE_AUDIT_ENABLED env var.
|
|
266
|
+
SUPERTABLE_AUDIT_ENABLED: bool = False # SUPERTABLE_AUDIT_ENABLED
|
|
264
267
|
SUPERTABLE_AUDIT_RETENTION_DAYS: int = 2555 # SUPERTABLE_AUDIT_RETENTION_DAYS (~7 years)
|
|
265
268
|
SUPERTABLE_AUDIT_BATCH_SIZE: int = 1000 # SUPERTABLE_AUDIT_BATCH_SIZE
|
|
266
269
|
SUPERTABLE_AUDIT_FLUSH_INTERVAL_SEC: int = 60 # SUPERTABLE_AUDIT_FLUSH_INTERVAL_SEC
|
|
@@ -508,7 +511,7 @@ def _build_settings() -> Settings:
|
|
|
508
511
|
SUPERTABLE_SUPER_META_CACHE_TTL_S=meta_ttl,
|
|
509
512
|
|
|
510
513
|
# ── Audit ────────────────────────────────────────────────────
|
|
511
|
-
SUPERTABLE_AUDIT_ENABLED=_env_bool("SUPERTABLE_AUDIT_ENABLED",
|
|
514
|
+
SUPERTABLE_AUDIT_ENABLED=_env_bool("SUPERTABLE_AUDIT_ENABLED", False),
|
|
512
515
|
SUPERTABLE_AUDIT_RETENTION_DAYS=_env_int("SUPERTABLE_AUDIT_RETENTION_DAYS", 2555),
|
|
513
516
|
SUPERTABLE_AUDIT_BATCH_SIZE=_env_int("SUPERTABLE_AUDIT_BATCH_SIZE", 1000),
|
|
514
517
|
SUPERTABLE_AUDIT_FLUSH_INTERVAL_SEC=_env_int("SUPERTABLE_AUDIT_FLUSH_INTERVAL_SEC", 60),
|
|
@@ -13,6 +13,7 @@ from polars import DataFrame
|
|
|
13
13
|
from supertable.config.defaults import logger
|
|
14
14
|
from supertable.monitoring_writer import MonitoringWriter # async monitoring
|
|
15
15
|
from supertable.super_table import SuperTable
|
|
16
|
+
from supertable import redis_keys as RK
|
|
16
17
|
from supertable.simple_table import SimpleTable
|
|
17
18
|
from supertable.utils.timer import Timer
|
|
18
19
|
from supertable.processing import (
|
|
@@ -455,8 +456,8 @@ class DataWriter:
|
|
|
455
456
|
else:
|
|
456
457
|
schema_json = "{}"
|
|
457
458
|
_org, _sup = self.super_table.organization, self.super_table.super_name
|
|
458
|
-
self.catalog.r.set(
|
|
459
|
-
self.catalog.r.sadd(
|
|
459
|
+
self.catalog.r.set(RK.schema(_org, _sup, simple_name), schema_json)
|
|
460
|
+
self.catalog.r.sadd(RK.table_names(_org, _sup), simple_name)
|
|
460
461
|
except Exception as e:
|
|
461
462
|
logger.debug(f"[data-writer] schema/table_names Redis write failed: {e}")
|
|
462
463
|
|