ethyca-fides 2.64.3b0__py2.py3-none-any.whl → 2.64.5rc3__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ethyca-fides might be problematic. Click here for more details.
- {ethyca_fides-2.64.3b0.dist-info → ethyca_fides-2.64.5rc3.dist-info}/METADATA +3 -3
- {ethyca_fides-2.64.3b0.dist-info → ethyca_fides-2.64.5rc3.dist-info}/RECORD +245 -237
- fides/_version.py +3 -3
- fides/api/api/v1/api.py +0 -2
- fides/api/db/base.py +3 -6
- fides/api/db/database.py +2 -27
- fides/api/graph/config.py +9 -16
- fides/api/models/attachment.py +3 -15
- fides/api/models/comment.py +5 -23
- fides/api/models/connectionconfig.py +0 -11
- fides/api/models/db_cache.py +1 -1
- fides/api/models/detection_discovery/core.py +15 -15
- fides/api/models/fides_user_respondent_email_verification.py +2 -27
- fides/api/models/manual_tasks/__init__.py +14 -0
- fides/api/models/manual_tasks/manual_task.py +120 -0
- fides/api/models/manual_tasks/manual_task_config.py +136 -0
- fides/api/models/manual_tasks/manual_task_log.py +104 -0
- fides/api/models/tcf_publisher_restrictions.py +4 -16
- fides/api/schemas/manual_tasks/__init__.py +0 -0
- fides/api/schemas/manual_tasks/manual_task_config.py +311 -0
- fides/api/schemas/manual_tasks/manual_task_schemas.py +79 -0
- fides/api/schemas/manual_tasks/manual_task_status.py +151 -0
- fides/api/service/connectors/query_configs/bigquery_query_config.py +22 -44
- fides/api/service/connectors/query_configs/query_config.py +2 -5
- fides/api/util/connection_util.py +2 -25
- fides/common/api/v1/urn_registry.py +0 -4
- fides/service/manual_tasks/__init__.py +0 -0
- fides/service/manual_tasks/manual_task_config_service.py +370 -0
- fides/service/manual_tasks/manual_task_service.py +294 -0
- fides/service/manual_tasks/utils.py +185 -0
- fides/ui-build/static/admin/404.html +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{1040-3def3c62138371bb.js → 1040-c1c1372a7f909aef.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/1100-3fdbdf211c3c2a5b.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{1817-a3813209d7a6d28d.js → 1817-96182c1558f80b63.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/2430-b480401d44c55416.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{2921-81a48418f77bc9a7.js → 2921-f5608275555bd7d9.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{3450-3a1f0e80ead8d1b0.js → 3450-272f26c102f3510d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/3505-192986c86dc47869.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3513-a563133845dc990f.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/3670-2abd9b2f17770872.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{3855-b85ad4e52433d519.js → 3855-beb58821d1ddba89.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{3872-09f81435f7905d76.js → 3872-82482e55e69b5a93.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{3923-565a852f322fdadc.js → 3923-a54c286a2ba0a47a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/3983-17ae9c232bddc413.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{401-c3f586a23a061526.js → 401-d2ce0a5a9120e056.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/4060-3486b45081151b69.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{409-e0d39af1d9ce8ff2.js → 409-86f4f687105917fb.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/4121-f0aecb2abd384945.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{4230-d80ae2430aca784a.js → 4230-63abbdfb9e9016b9.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{431-c74dab231c8ac968.js → 431-77d59d43e90058ca.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/4481-d181a9db72984adf.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{5309-0e44dbd914896514.js → 5309-10f68cf805817cfb.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{5574-917c5ff63d222308.js → 5574-b6db9d62362e72d9.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/6060-cb1ab5be7067bf7b.js +4 -0
- fides/ui-build/static/admin/_next/static/chunks/6277-ccdb50f676a1b336.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/6659-b2088f525bf13c17.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/6662-42940a2b00933e79.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6853-8d0a099f61c758d1.js → 6853-8a1b8e1c8b249f2f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{6882-13d6f8d95b1e404f.js → 6882-6c94583bffe85ba7.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/69-943b19d39da339d9.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{6954-882b87698c5ed4fb.js → 6954-dc3540389daf94da.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/7553-e7ae268701f3dcfe.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{79-3f742fe4efd9893f.js → 79-26ccd45dfd6653a7.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{796-83a8bbdcdb67f2ba.js → 796-38c0f7e6755ad359.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/8433-1e065c55c8da73b0.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9046-6995482a030e323c.js → 9046-bbce3c73af16daf9.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{905-ffdbd0b14167e8bd.js → 905-742074a074be1055.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{9226-8a3be36ad1a9c1e7.js → 9226-5e0ce31cfdedd5ee.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{9392.9a948112de74781b.js → 9392.25024e070026343d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{9676.cc515c853b8cf578.js → 9676.e60a53f1f5890847.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/9767-96ed554a043c3c4d.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/{9951-b5b77bfcc8efb493.js → 9951-d3d5d0fe4c4edb86.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/c78d26b1-88a3e1bacb2a03c2.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{404-488f8f03fe0ffbc7.js → 404-d41660858638adee.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{_app-030b20295bb61f96.js → _app-6b3539f8d82ce9ae.js} +89 -92
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{manual-1e88ac28bc7a41b6.js → manual-7c5e1e845372c99b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems/{multiple-f2c9451fffaaa529.js → multiple-87e78f52af21c4a9.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems-68955d8441e60668.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/configure/{add-vendors-b17d160147365cf3.js → add-vendors-6374a3a7747df964.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{configure-8e168d78acdf0cfe.js → configure-0512151e2ed0f4c1.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{[id]-5ca2467de7986929.js → [id]-09435f4ab2deafd2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-experience/{new-06bb3b0bf097fcdb.js → new-a0039f216fb3eb93.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-experience-3c11fecc2797ab68.js → privacy-experience-cd036518b5d4efbe.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{[id]-92bc6c7b82a679b4.js → [id]-0e534580abf670be.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/privacy-notices/{new-a3da3243526b7ecb.js → new-bb1790eb87b63109.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{privacy-notices-b7d82386e7521041.js → privacy-notices-67e9846f877a3f38.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{properties-20a2029b7e7fc895.js → properties-69a92b6ddbc2bae2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/consent/{reporting-0ce299131db4c3e5.js → reporting-d07d05f4b898a5ed.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{consent-9f6a7a231bba17b7.js → consent-23e886d692ab6d1a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/[projectUrn]/{[resourceUrn]-11d52f1570759c4d.js → [resourceUrn]-c623d6f1a61c8ea9.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/projects/{[projectUrn]-fd705968b357e99a.js → [projectUrn]-6c766c766dc97c5c.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{projects-45b585deee0b2371.js → projects-28937a8da3d73145.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/resources/{[resourceUrn]-b83afa5565d0c84e.js → [resourceUrn]-57bd5cdf784f059f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-catalog/[systemId]/{resources-d8db234a44a2ddf4.js → resources-51d99174c8006eb5.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{data-catalog-2810fa2b519a076b.js → data-catalog-627fbf19dce88e81.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/[monitorId]/{[systemId]-e861699a8866c64b.js → [systemId]-ed1629b05519a370.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/action-center/{[monitorId]-5dd2fbf33e228f9c.js → [monitorId]-ad307bfc5f51fd9d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{action-center-806cae6bc128cd38.js → action-center-07cbd61ede6e18ac.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{activity-487285bd5eca2595.js → activity-6a2aaed8d0e66d82.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/detection/{[resourceUrn]-393e20924c83373e.js → [resourceUrn]-06edce289876dea1.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{detection-8733807dad4bc96e.js → detection-faf326a6200637d0.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/discovery/{[resourceUrn]-14bd7500362ff224.js → [resourceUrn]-64acf269256ee74f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/data-discovery/{discovery-9e7dfd5a6acc2e8f.js → discovery-8c3e4be6d36da66d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{datamap-c9509d72c538d22b.js → datamap-b576a94b583a7940.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/[collectionName]/{[...subfieldNames]-4e8a436297a055b2.js → [...subfieldNames]-4912858ffde4621a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/[datasetId]/{[collectionName]-6ce02295bb7f5b6d.js → [collectionName]-df21dd7ca0f35718.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/{[datasetId]-9eaa907437fde063.js → [datasetId]-5ce28a329e8f667f.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/new-5287c76ecf600281.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{dataset-b328595abf20ea5d.js → dataset-c6fab547396df6ac.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{[id]-81ab412e337d2888.js → [id]-d04c9925d324eee1.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection/{new-34dfc172165dbb1c.js → new-656abd09ea5ee39a.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection-23fbec0590c8d192.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/{index-de954b741cbca022.js → index-b66687e8194495a2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations/{[id]-16dd6eff8f5dcc81.js → [id]-906bc5f05702efb0.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations-024a1facb9be04d0.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{[id]-42edcab11e8b5c3c.js → [id]-b74db8488e2e4b58.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/messaging/{add-template-333e54ac8c3ad57a.js → add-template-a607018ff097b6c2.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{messaging-8e1e6f3782983225.js → messaging-4a1a04c5179d2053.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/ant-components-1567c9770a5f05aa.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{AntForm-920338760f5a71b5.js → AntForm-4949fe7f1815462b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikAntFormItem-7462fb387a9de3f8.js → FormikAntFormItem-6085f73850302d55.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikControlled-6148fce7a4e2e9ca.js → FormikControlled-a3158e8217c13850.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/form-experiments/{FormikField-30efd8c937bf19e4.js → FormikField-8576f1ef5c67d87d.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/{forms-c44de83e0952d1e0.js → forms-341e67462b5e3352.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/table-migration-6f0e64f0c52bd68f.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-6971c7773dbf9b51.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-303bd2182da03088.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-831572cc7f42615f.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/{configure-1d981663e1a84166.js → configure-d24230b890b13762.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-569ff31eff637034.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/properties/{[id]-b08a69b1c460c7fe.js → [id]-2bedc5013e13ab52.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/properties/{add-property-a5d1c65ec21df69d.js → add-property-9d68bd70299dd945.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{properties-3a75c6ed8308d126.js → properties-c3116b6bfe2e695e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/reporting/{datamap-dda59a7105ee609b.js → datamap-6dfa7091b99d8321.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/about/{alpha-6773158ba6ccf4b0.js → alpha-9751059905bba190.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{about-2e046d177d52465c.js → about-ad29d77012ec9dba.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent/[configuration_id]/{[purpose_id]-91f2ec72f9654cbd.js → [purpose_id]-32e3190c1ad00b40.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent-13a17a4ace7293d1.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{custom-fields-94d97e3eb964c494.js → custom-fields-d2c0aac32f5d2930.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{domain-records-2c7ecff0a8a74c42.js → domain-records-a10674380f94d014.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/domains-78449b0e02bced88.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/email-templates-7d299c4cb3199036.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{locations-8d4383584c72eb5a.js → locations-cc7d629433fa6d44.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/organization-a9fa55c40fa570a6.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/{regulations-29892065d99ff113.js → regulations-64a813cb2741683b.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/test-datasets-547b0cfe1e49e6d2.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/{[id]-8fa8a2f238e08791.js → [id]-fb3d094e4f4585f6.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{systems-acdbdd3dfd21162f.js → systems-c2df5b7b0596a9cb.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-261deb6fb3e51cb3.js +1 -0
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/{new-a2524414e968f862.js → new-b124cc24b930c9e1.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/user-management/profile/{[id]-87ed17fa1d9f8f72.js → [id]-fd2ff6b13052c54e.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/{user-management-6b4e0764bb8816b8.js → user-management-fa052b0439920ef6.js} +1 -1
- fides/ui-build/static/admin/_next/static/chunks/{webpack-da78c536f3d86d06.js → webpack-e61d457474e00565.js} +1 -1
- fides/ui-build/static/admin/_next/static/css/{1994066ec907b7df.css → c693338e3bc8dcc6.css} +1 -1
- fides/ui-build/static/admin/_next/static/g740PFm_oIYCWNhoD6-r0/_buildManifest.js +1 -0
- fides/ui-build/static/admin/add-systems/manual.html +1 -1
- fides/ui-build/static/admin/add-systems/multiple.html +1 -1
- fides/ui-build/static/admin/add-systems.html +1 -1
- fides/ui-build/static/admin/consent/configure/add-vendors.html +1 -1
- fides/ui-build/static/admin/consent/configure.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-experience.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/[id].html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices/new.html +1 -1
- fides/ui-build/static/admin/consent/privacy-notices.html +1 -1
- fides/ui-build/static/admin/consent/properties.html +1 -1
- fides/ui-build/static/admin/consent/reporting.html +1 -1
- fides/ui-build/static/admin/consent.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn]/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects/[projectUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/projects.html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-catalog/[systemId]/resources.html +1 -1
- fides/ui-build/static/admin/data-catalog.html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId]/[systemId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center/[monitorId].html +1 -1
- fides/ui-build/static/admin/data-discovery/action-center.html +1 -1
- fides/ui-build/static/admin/data-discovery/activity.html +1 -1
- fides/ui-build/static/admin/data-discovery/detection/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/detection.html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery/[resourceUrn].html +1 -1
- fides/ui-build/static/admin/data-discovery/discovery.html +1 -1
- fides/ui-build/static/admin/datamap.html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName]/[...subfieldNames].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId]/[collectionName].html +1 -1
- fides/ui-build/static/admin/dataset/[datasetId].html +1 -1
- fides/ui-build/static/admin/dataset/new.html +1 -1
- fides/ui-build/static/admin/dataset.html +1 -1
- fides/ui-build/static/admin/datastore-connection/[id].html +1 -1
- fides/ui-build/static/admin/datastore-connection/new.html +1 -1
- fides/ui-build/static/admin/datastore-connection.html +1 -1
- fides/ui-build/static/admin/index.html +1 -1
- fides/ui-build/static/admin/integrations/[id].html +1 -1
- fides/ui-build/static/admin/integrations.html +1 -1
- fides/ui-build/static/admin/lib/fides-ext-gpp.js +1 -1
- fides/ui-build/static/admin/lib/fides-headless.js +1 -1
- fides/ui-build/static/admin/lib/fides-preview.js +1 -1
- fides/ui-build/static/admin/lib/fides-tcf.js +2 -2
- fides/ui-build/static/admin/lib/fides.js +2 -2
- fides/ui-build/static/admin/login/[provider].html +1 -1
- fides/ui-build/static/admin/login.html +1 -1
- fides/ui-build/static/admin/messaging/[id].html +1 -1
- fides/ui-build/static/admin/messaging/add-template.html +1 -1
- fides/ui-build/static/admin/messaging.html +1 -1
- fides/ui-build/static/admin/poc/ant-components.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/AntForm.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikAntFormItem.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikControlled.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikField.html +1 -1
- fides/ui-build/static/admin/poc/form-experiments/FormikSpreadField.html +1 -1
- fides/ui-build/static/admin/poc/forms.html +1 -1
- fides/ui-build/static/admin/poc/table-migration.html +1 -1
- fides/ui-build/static/admin/privacy-requests/[id].html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/messaging.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure/storage.html +1 -1
- fides/ui-build/static/admin/privacy-requests/configure.html +1 -1
- fides/ui-build/static/admin/privacy-requests.html +1 -1
- fides/ui-build/static/admin/properties/[id].html +1 -1
- fides/ui-build/static/admin/properties/add-property.html +1 -1
- fides/ui-build/static/admin/properties.html +1 -1
- fides/ui-build/static/admin/reporting/datamap.html +1 -1
- fides/ui-build/static/admin/settings/about/alpha.html +1 -1
- fides/ui-build/static/admin/settings/about.html +1 -1
- fides/ui-build/static/admin/settings/consent/[configuration_id]/[purpose_id].html +1 -1
- fides/ui-build/static/admin/settings/consent.html +1 -1
- fides/ui-build/static/admin/settings/custom-fields.html +1 -1
- fides/ui-build/static/admin/settings/domain-records.html +1 -1
- fides/ui-build/static/admin/settings/domains.html +1 -1
- fides/ui-build/static/admin/settings/email-templates.html +1 -1
- fides/ui-build/static/admin/settings/locations.html +1 -1
- fides/ui-build/static/admin/settings/organization.html +1 -1
- fides/ui-build/static/admin/settings/regulations.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id]/test-datasets.html +1 -1
- fides/ui-build/static/admin/systems/configure/[id].html +1 -1
- fides/ui-build/static/admin/systems.html +1 -1
- fides/ui-build/static/admin/taxonomy.html +1 -1
- fides/ui-build/static/admin/user-management/new.html +1 -1
- fides/ui-build/static/admin/user-management/profile/[id].html +1 -1
- fides/ui-build/static/admin/user-management.html +1 -1
- fides/api/alembic/migrations/versions/41a634d8c669_manual_task_restrict_deletes.py +0 -257
- fides/api/alembic/migrations/versions/6a76a1fa4f3f_add_manual_task_instance_table.py +0 -256
- fides/api/alembic/migrations/versions/aadfe83c5644_add_manual_task_to_connectiontype_enum.py +0 -46
- fides/api/api/v1/endpoints/partitioning_endpoints.py +0 -108
- fides/api/models/manual_task.py +0 -965
- fides/api/schemas/partitioning/__init__.py +0 -17
- fides/api/schemas/partitioning/bigquery_time_based_partitioning.py +0 -31
- fides/api/schemas/partitioning/time_based_partitioning.py +0 -1376
- fides/ui-build/static/admin/_next/static/chunks/1169-76bbada4f3d16538.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/1807-3beab149351d5ded.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/3615-5e2d062d684b8fa1.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/4121-de4ca969faf2b9f4.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/570-c99f07161bd339cd.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6084-40d5d94561a6093e.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/6662-6a915300505fd9c0.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7476-aaf8970dbbbe4864.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/7630-c558dc3a199a633d.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/787-9c751615f5816094.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/827-c6fe34fb336467ae.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9014-eeae6f581158e645.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9767-1dca308466dce863.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/9826-303b14ef4fc7ab4a.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/add-systems-a8e0bc7b6674f47e.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/dataset/new-55243db34cf77b7e.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/datastore-connection-cdb77886d7fd4f44.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/integrations-40a3cd14264796e2.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/ant-components-3f62dd959a039fe9.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/poc/table-migration-7852aa60090c8c9a.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/[id]-11dd6152bf6607cc.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/messaging-c50c2ee7d7d2a585.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests/configure/storage-282de19599d67aaf.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/privacy-requests-3f962ade5df86380.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/consent-0a87f29768425a37.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/domains-b9e77c75c6b77c88.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/email-templates-bb60c397a03558d8.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/settings/organization-4b835393f5274379.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/systems/configure/[id]/test-datasets-9669fa6b20545530.js +0 -1
- fides/ui-build/static/admin/_next/static/chunks/pages/taxonomy-c9510a1eb612323d.js +0 -1
- fides/ui-build/static/admin/_next/static/ezJVaZ_Oi_0J2-wPEOvj1/_buildManifest.js +0 -1
- {ethyca_fides-2.64.3b0.dist-info → ethyca_fides-2.64.5rc3.dist-info}/WHEEL +0 -0
- {ethyca_fides-2.64.3b0.dist-info → ethyca_fides-2.64.5rc3.dist-info}/entry_points.txt +0 -0
- {ethyca_fides-2.64.3b0.dist-info → ethyca_fides-2.64.5rc3.dist-info}/licenses/LICENSE +0 -0
- {ethyca_fides-2.64.3b0.dist-info → ethyca_fides-2.64.5rc3.dist-info}/top_level.txt +0 -0
- /fides/ui-build/static/admin/_next/static/{ezJVaZ_Oi_0J2-wPEOvj1 → g740PFm_oIYCWNhoD6-r0}/_ssgManifest.js +0 -0
|
@@ -1,1376 +0,0 @@
|
|
|
1
|
-
# pylint: disable=too-many-branches, too-many-statements, too-many-return-statements, too-many-lines
|
|
2
|
-
import re
|
|
3
|
-
from datetime import datetime, timedelta
|
|
4
|
-
from enum import Enum
|
|
5
|
-
from typing import List, Optional, Set, Tuple
|
|
6
|
-
|
|
7
|
-
from pydantic import Field, field_validator, model_validator
|
|
8
|
-
from pydantic.json_schema import SkipJsonSchema
|
|
9
|
-
from sqlalchemy import and_, column, func, text
|
|
10
|
-
from sqlalchemy.sql import ColumnElement
|
|
11
|
-
|
|
12
|
-
from fides.api.schemas.base_class import FidesSchema
|
|
13
|
-
|
|
14
|
-
# ------------------------------------------------------------------
|
|
15
|
-
# Regular expression patterns reused across validation helpers.
|
|
16
|
-
# ------------------------------------------------------------------
|
|
17
|
-
|
|
18
|
-
NOW_LITERAL_REGEX = r"^NOW\(\)$"
|
|
19
|
-
TODAY_LITERAL_REGEX = r"^TODAY\(\)$"
|
|
20
|
-
TIMESTAMP_TODAY_LITERAL_REGEX = r"^TIMESTAMP\(TODAY\(\)\)$"
|
|
21
|
-
|
|
22
|
-
# Arithmetic offset pattern, e.g. "NOW() - 30 DAYS" or "TODAY() + 2 WEEKS"
|
|
23
|
-
ARITHMETIC_OFFSET_REGEX = (
|
|
24
|
-
r"^(NOW|TODAY)\(\)\s*([+-])\s*(\d+)\s+"
|
|
25
|
-
r"(DAY|DAYS|WEEK|WEEKS|MONTH|MONTHS|YEAR|YEARS)$"
|
|
26
|
-
)
|
|
27
|
-
|
|
28
|
-
# TIMESTAMP function with TODAY() and optional offset, e.g. "TIMESTAMP(TODAY() - 30 DAYS)"
|
|
29
|
-
TIMESTAMP_TODAY_OFFSET_REGEX = (
|
|
30
|
-
r"^TIMESTAMP\(TODAY\(\)\s*([+-])\s*(\d+)\s+"
|
|
31
|
-
r"(DAY|DAYS|WEEK|WEEKS|MONTH|MONTHS|YEAR|YEARS)\)$"
|
|
32
|
-
)
|
|
33
|
-
|
|
34
|
-
# Simple date & datetime literals
|
|
35
|
-
DATE_LITERAL_REGEX = r"^\d{4}-\d{2}-\d{2}$"
|
|
36
|
-
DATETIME_LITERAL_REGEX = r"^\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}$"
|
|
37
|
-
TIMESTAMP_DATE_LITERAL_REGEX = r"^TIMESTAMP\('\d{4}-\d{2}-\d{2}'\)$"
|
|
38
|
-
TIMESTAMP_DATETIME_LITERAL_REGEX = (
|
|
39
|
-
r"^TIMESTAMP\('\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2}'\)$"
|
|
40
|
-
)
|
|
41
|
-
|
|
42
|
-
# Interval pattern (non-capturing) e.g. "7 DAYS", "2 WEEKS"
|
|
43
|
-
INTERVAL_REGEX = r"^\d+\s+(DAY|DAYS|WEEK|WEEKS|MONTH|MONTHS|YEAR|YEARS)$"
|
|
44
|
-
# Same pattern but with capturing groups for value & unit so we can parse quickly
|
|
45
|
-
INTERVAL_CAPTURE_REGEX = r"^(\d+)\s+(DAY|DAYS|WEEK|WEEKS|MONTH|MONTHS|YEAR|YEARS)$"
|
|
46
|
-
|
|
47
|
-
# Offset pattern for day/week units, captures NOW() or TODAY(), numeric value, and unit
|
|
48
|
-
DAY_WEEK_OFFSET_REGEX = r"^(NOW\(\)|TODAY\(\))\s*-\s*(\d+)\s+(DAY|DAYS|WEEK|WEEKS)$"
|
|
49
|
-
|
|
50
|
-
# Offset pattern for month/year units (captures base, numeric value, and unit)
|
|
51
|
-
MONTH_YEAR_OFFSET_REGEX = (
|
|
52
|
-
r"^(NOW\(\)|TODAY\(\))\s*-\s*(\d+)\s+(MONTH|MONTHS|YEAR|YEARS)$"
|
|
53
|
-
)
|
|
54
|
-
|
|
55
|
-
# TIMESTAMP-wrapped offset patterns for day/week units
|
|
56
|
-
TIMESTAMP_DAY_WEEK_OFFSET_REGEX = (
|
|
57
|
-
r"^TIMESTAMP\(TODAY\(\)\s*-\s*(\d+)\s+(DAY|DAYS|WEEK|WEEKS)\)$"
|
|
58
|
-
)
|
|
59
|
-
|
|
60
|
-
# TIMESTAMP-wrapped offset patterns for month/year units
|
|
61
|
-
TIMESTAMP_MONTH_YEAR_OFFSET_REGEX = (
|
|
62
|
-
r"^TIMESTAMP\(TODAY\(\)\s*-\s*(\d+)\s+(MONTH|MONTHS|YEAR|YEARS)\)$"
|
|
63
|
-
)
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
class TimeUnit(str, Enum):
|
|
67
|
-
"""Standardized time units for partitioning."""
|
|
68
|
-
|
|
69
|
-
DAY = "DAY"
|
|
70
|
-
WEEK = "WEEK"
|
|
71
|
-
MONTH = "MONTH"
|
|
72
|
-
YEAR = "YEAR"
|
|
73
|
-
|
|
74
|
-
@classmethod
|
|
75
|
-
def parse(cls, raw: str) -> "TimeUnit":
|
|
76
|
-
"""Parse raw (possibly lowercase or plural) into a TimeUnit."""
|
|
77
|
-
|
|
78
|
-
cleaned = raw.strip().upper().rstrip("S")
|
|
79
|
-
try:
|
|
80
|
-
return cls[cleaned]
|
|
81
|
-
except KeyError as exc:
|
|
82
|
-
raise ValueError(f"Unsupported time unit: {raw}") from exc
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
class TimeBasedPartitioning(FidesSchema):
|
|
86
|
-
"""
|
|
87
|
-
Allows you to partition data based on time ranges using various
|
|
88
|
-
time expressions and intervals.
|
|
89
|
-
"""
|
|
90
|
-
|
|
91
|
-
# Generic time-based partitioning using pure SQLAlchemy constructs.
|
|
92
|
-
# Uses datetime.timedelta for all time calculations and SQLAlchemy expressions.
|
|
93
|
-
|
|
94
|
-
field: str = Field(
|
|
95
|
-
description="Column name to partition on (e.g., `created_at`, `updated_at`)",
|
|
96
|
-
)
|
|
97
|
-
|
|
98
|
-
start: Optional[str] = Field(
|
|
99
|
-
default=None,
|
|
100
|
-
description="Start time expression. Supported formats: `NOW()`, `TODAY()`, `TIMESTAMP(TODAY())`, `YYYY-MM-DD`, `YYYY-MM-DD HH:MM:SS`, or arithmetic expressions like `NOW() - 30 DAYS`, `TIMESTAMP(TODAY() - 30 DAYS)`.",
|
|
101
|
-
)
|
|
102
|
-
|
|
103
|
-
end: Optional[str] = Field(
|
|
104
|
-
default=None,
|
|
105
|
-
description="End time expression. Same formats as start field.",
|
|
106
|
-
)
|
|
107
|
-
|
|
108
|
-
interval: Optional[str] = Field(
|
|
109
|
-
description="Interval expression defining the size of each partition in `DAY(S)`, `WEEK(S)`, `MONTH(S)`, or `YEAR(S)`.",
|
|
110
|
-
default=None,
|
|
111
|
-
)
|
|
112
|
-
|
|
113
|
-
# Determines whether the *first* slice produced by this partition spec
|
|
114
|
-
# includes the lower bound (>=) or is exclusive (>). Default is inclusive
|
|
115
|
-
# and the value is *not* part of the public schema – it is manipulated by
|
|
116
|
-
# helper utilities when combining multiple adjacent partitions.
|
|
117
|
-
inclusive_start: SkipJsonSchema[bool] = Field(default=True, repr=False)
|
|
118
|
-
|
|
119
|
-
# Example of using model_config with json_schema_extra for schema-level examples
|
|
120
|
-
model_config = {
|
|
121
|
-
"json_schema_extra": {
|
|
122
|
-
"examples": [
|
|
123
|
-
{
|
|
124
|
-
"field": "created_at",
|
|
125
|
-
"start": "NOW() - 30 DAYS",
|
|
126
|
-
"end": "NOW()",
|
|
127
|
-
"interval": "7 DAYS",
|
|
128
|
-
},
|
|
129
|
-
{
|
|
130
|
-
"field": "order_date",
|
|
131
|
-
"start": "2024-01-01",
|
|
132
|
-
"end": "2024-12-31",
|
|
133
|
-
"interval": "1 MONTH",
|
|
134
|
-
},
|
|
135
|
-
{
|
|
136
|
-
"field": "event_timestamp",
|
|
137
|
-
"start": "TODAY() - 1 YEAR",
|
|
138
|
-
"interval": "1 WEEK",
|
|
139
|
-
},
|
|
140
|
-
{
|
|
141
|
-
"field": "created_at",
|
|
142
|
-
"start": "TIMESTAMP(TODAY() - 7 DAYS)",
|
|
143
|
-
"end": "TIMESTAMP(TODAY())",
|
|
144
|
-
"interval": "1 DAY",
|
|
145
|
-
},
|
|
146
|
-
]
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
# ---------------------------------------------------------------------
|
|
151
|
-
# Validators
|
|
152
|
-
# ---------------------------------------------------------------------
|
|
153
|
-
|
|
154
|
-
@field_validator("start", "end")
|
|
155
|
-
@classmethod
|
|
156
|
-
def validate_time_expression(cls, value: Optional[str]) -> Optional[str]:
|
|
157
|
-
"""Validate time expressions."""
|
|
158
|
-
if value is None:
|
|
159
|
-
return value
|
|
160
|
-
v = value.strip().upper()
|
|
161
|
-
patterns = [
|
|
162
|
-
NOW_LITERAL_REGEX,
|
|
163
|
-
TODAY_LITERAL_REGEX,
|
|
164
|
-
TIMESTAMP_TODAY_LITERAL_REGEX,
|
|
165
|
-
ARITHMETIC_OFFSET_REGEX,
|
|
166
|
-
TIMESTAMP_TODAY_OFFSET_REGEX,
|
|
167
|
-
DATE_LITERAL_REGEX,
|
|
168
|
-
DATETIME_LITERAL_REGEX,
|
|
169
|
-
TIMESTAMP_DATE_LITERAL_REGEX,
|
|
170
|
-
TIMESTAMP_DATETIME_LITERAL_REGEX,
|
|
171
|
-
]
|
|
172
|
-
|
|
173
|
-
if not any(re.match(pattern, v) for pattern in patterns):
|
|
174
|
-
raise ValueError(f"Unsupported time expression: {v}")
|
|
175
|
-
return v
|
|
176
|
-
|
|
177
|
-
@field_validator("interval")
|
|
178
|
-
@classmethod
|
|
179
|
-
def validate_interval(cls, value: Optional[str]) -> Optional[str]:
|
|
180
|
-
"""Validate interval expressions."""
|
|
181
|
-
|
|
182
|
-
if value is None:
|
|
183
|
-
return value
|
|
184
|
-
v = value.strip().upper()
|
|
185
|
-
if not re.match(INTERVAL_REGEX, v):
|
|
186
|
-
raise ValueError(f"Invalid interval format: {v}")
|
|
187
|
-
|
|
188
|
-
# Store normalized uppercase interval
|
|
189
|
-
return v
|
|
190
|
-
|
|
191
|
-
@model_validator(mode="after")
|
|
192
|
-
def validate_bounds_and_interval(self) -> "TimeBasedPartitioning":
|
|
193
|
-
"""Ensure at least one bound is provided."""
|
|
194
|
-
# At least one bound must be supplied.
|
|
195
|
-
if self.start is None and self.end is None:
|
|
196
|
-
raise ValueError("At least one of 'start' or 'end' must be provided.")
|
|
197
|
-
|
|
198
|
-
return self
|
|
199
|
-
|
|
200
|
-
# ---------------------------------------------------------------------
|
|
201
|
-
# Interval helpers
|
|
202
|
-
# ---------------------------------------------------------------------
|
|
203
|
-
|
|
204
|
-
def _split_interval(self) -> Tuple[int, TimeUnit]:
|
|
205
|
-
"""Convert an interval string into a value and unit pair."""
|
|
206
|
-
|
|
207
|
-
if self.interval is None:
|
|
208
|
-
raise ValueError("Interval must be specified before it can be parsed.")
|
|
209
|
-
|
|
210
|
-
normalized = str(self.interval).strip().upper()
|
|
211
|
-
match = re.match(INTERVAL_CAPTURE_REGEX, normalized)
|
|
212
|
-
if not match:
|
|
213
|
-
raise ValueError(f"Invalid interval format: {self.interval}")
|
|
214
|
-
|
|
215
|
-
value = int(match.group(1))
|
|
216
|
-
unit = TimeUnit.parse(match.group(2))
|
|
217
|
-
return value, unit
|
|
218
|
-
|
|
219
|
-
def _timedelta_from_value_unit(self, value: int, unit: TimeUnit) -> timedelta:
|
|
220
|
-
"""Translate value/unit pairs into a `timedelta`."""
|
|
221
|
-
|
|
222
|
-
if unit is TimeUnit.WEEK:
|
|
223
|
-
return timedelta(weeks=value)
|
|
224
|
-
if unit is TimeUnit.DAY:
|
|
225
|
-
return timedelta(days=value)
|
|
226
|
-
|
|
227
|
-
raise ValueError(
|
|
228
|
-
"Only 'day' and 'week' units can be converted to timedelta – "
|
|
229
|
-
f"received '{unit}'."
|
|
230
|
-
)
|
|
231
|
-
|
|
232
|
-
def _timedelta_to_interval(self, time_delta: timedelta) -> str:
|
|
233
|
-
"""Return a SQL `INTERVAL` clause for the supplied `timedelta`."""
|
|
234
|
-
total_days = int(time_delta.total_seconds() / 86400) # 86400 seconds in a day
|
|
235
|
-
|
|
236
|
-
# Keep it in weeks if the interval is in weeks
|
|
237
|
-
if self.interval and "WEEK" in self.interval:
|
|
238
|
-
if total_days % 7 == 0 and total_days >= 7:
|
|
239
|
-
return self.format_interval(total_days // 7, TimeUnit.WEEK)
|
|
240
|
-
|
|
241
|
-
return self.format_interval(total_days, TimeUnit.DAY)
|
|
242
|
-
|
|
243
|
-
def _parse_time_expression(self, expr: str) -> ColumnElement:
|
|
244
|
-
"""Convert time expression to SQLAlchemy expression."""
|
|
245
|
-
expr = expr.strip().upper()
|
|
246
|
-
|
|
247
|
-
if expr == "NOW()":
|
|
248
|
-
return func.current_timestamp()
|
|
249
|
-
|
|
250
|
-
# Handle TODAY() direct
|
|
251
|
-
if expr == "TODAY()":
|
|
252
|
-
return func.current_date()
|
|
253
|
-
|
|
254
|
-
# Handle TIMESTAMP(TODAY()) - Today at 00:00:00 as timestamp
|
|
255
|
-
if expr == "TIMESTAMP(TODAY())":
|
|
256
|
-
return func.timestamp(func.current_date())
|
|
257
|
-
|
|
258
|
-
# Handle TIMESTAMP(TODAY() - N UNIT) patterns
|
|
259
|
-
timestamp_offset_match = re.match(TIMESTAMP_TODAY_OFFSET_REGEX, expr)
|
|
260
|
-
if timestamp_offset_match:
|
|
261
|
-
operator, value_str, unit_raw = timestamp_offset_match.groups()
|
|
262
|
-
value = int(value_str)
|
|
263
|
-
unit_raw = unit_raw.upper()
|
|
264
|
-
|
|
265
|
-
# Build the TODAY() - N UNIT expression inside TIMESTAMP()
|
|
266
|
-
base_date = func.current_date()
|
|
267
|
-
|
|
268
|
-
# Build INTERVAL text
|
|
269
|
-
if TimeUnit.parse(unit_raw) == TimeUnit.WEEK:
|
|
270
|
-
interval_text = self._timedelta_to_interval(timedelta(weeks=value))
|
|
271
|
-
elif TimeUnit.parse(unit_raw) == TimeUnit.DAY:
|
|
272
|
-
interval_text = self._timedelta_to_interval(timedelta(days=value))
|
|
273
|
-
else: # MONTH / YEAR
|
|
274
|
-
interval_text = self.format_interval(value, TimeUnit.parse(unit_raw))
|
|
275
|
-
|
|
276
|
-
delta_expr = text(interval_text)
|
|
277
|
-
|
|
278
|
-
# Apply offset to base date, then convert to timestamp
|
|
279
|
-
date_with_offset = (
|
|
280
|
-
base_date - delta_expr if operator == "-" else base_date + delta_expr
|
|
281
|
-
)
|
|
282
|
-
return func.timestamp(date_with_offset)
|
|
283
|
-
|
|
284
|
-
# Handle arithmetic on NOW() and TODAY() via a single pattern
|
|
285
|
-
arithmetic_match = re.match(ARITHMETIC_OFFSET_REGEX, expr)
|
|
286
|
-
|
|
287
|
-
if arithmetic_match:
|
|
288
|
-
func_name, operator, value_str, unit_raw = arithmetic_match.groups()
|
|
289
|
-
|
|
290
|
-
value = int(value_str)
|
|
291
|
-
unit_raw = unit_raw.upper()
|
|
292
|
-
|
|
293
|
-
# Resolve base expression for NOW() vs TODAY()
|
|
294
|
-
base_expr = (
|
|
295
|
-
func.current_timestamp() if func_name == "NOW" else func.current_date()
|
|
296
|
-
)
|
|
297
|
-
|
|
298
|
-
# Build INTERVAL text
|
|
299
|
-
if TimeUnit.parse(unit_raw) == TimeUnit.WEEK:
|
|
300
|
-
interval_text = self._timedelta_to_interval(timedelta(weeks=value))
|
|
301
|
-
elif TimeUnit.parse(unit_raw) == TimeUnit.DAY:
|
|
302
|
-
interval_text = self._timedelta_to_interval(timedelta(days=value))
|
|
303
|
-
else: # MONTH / YEAR
|
|
304
|
-
interval_text = self.format_interval(value, TimeUnit.parse(unit_raw))
|
|
305
|
-
|
|
306
|
-
delta_expr = text(interval_text)
|
|
307
|
-
|
|
308
|
-
return base_expr - delta_expr if operator == "-" else base_expr + delta_expr
|
|
309
|
-
|
|
310
|
-
# Handle date literals
|
|
311
|
-
if re.match(DATE_LITERAL_REGEX, expr):
|
|
312
|
-
date_str = expr.split()[0] # Remove time part if present
|
|
313
|
-
return func.DATE(date_str)
|
|
314
|
-
|
|
315
|
-
# Handle datetime literals
|
|
316
|
-
if re.match(DATETIME_LITERAL_REGEX, expr):
|
|
317
|
-
# Use TIMESTAMP function to convert string to timestamp for compatibility with interval arithmetic
|
|
318
|
-
return func.timestamp(text(f"'{expr}'"))
|
|
319
|
-
|
|
320
|
-
# Handle TIMESTAMP('date') literals
|
|
321
|
-
if re.match(TIMESTAMP_DATE_LITERAL_REGEX, expr):
|
|
322
|
-
# Extract the date from TIMESTAMP('2020-01-01') format
|
|
323
|
-
date_match = re.search(r"'(\d{4}-\d{2}-\d{2})'", expr)
|
|
324
|
-
if date_match:
|
|
325
|
-
date_str = date_match.group(1)
|
|
326
|
-
return func.timestamp(func.date(date_str))
|
|
327
|
-
|
|
328
|
-
# Handle TIMESTAMP('datetime') literals
|
|
329
|
-
if re.match(TIMESTAMP_DATETIME_LITERAL_REGEX, expr):
|
|
330
|
-
# Extract the datetime from TIMESTAMP('2020-01-01 12:30:45') format
|
|
331
|
-
datetime_match = re.search(
|
|
332
|
-
r"'(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})'", expr
|
|
333
|
-
)
|
|
334
|
-
if datetime_match:
|
|
335
|
-
datetime_str = datetime_match.group(1)
|
|
336
|
-
return func.timestamp(text(f"'{datetime_str}'"))
|
|
337
|
-
|
|
338
|
-
raise ValueError(f"Unsupported time expression: {expr}")
|
|
339
|
-
|
|
340
|
-
def _calculate_total_duration(self) -> timedelta:
|
|
341
|
-
"""Calculate total duration between start and end as timedelta.
|
|
342
|
-
|
|
343
|
-
Supported patterns
|
|
344
|
-
------------------
|
|
345
|
-
1. `NOW() - N DAY|DAYS|WEEK|WEEKS to NOW()`
|
|
346
|
-
2. `TODAY() - N DAY|DAYS|WEEK|WEEKS to TODAY()`
|
|
347
|
-
3. `TIMESTAMP(TODAY() - N DAY|DAYS|WEEK|WEEKS) to TIMESTAMP(TODAY())`
|
|
348
|
-
4. Both `start` and `end` are literal `YYYY-MM-DD` strings.
|
|
349
|
-
"""
|
|
350
|
-
start_str = self.start or ""
|
|
351
|
-
end_str = self.end or ""
|
|
352
|
-
|
|
353
|
-
# Dynamic offsets expressed relative to NOW() or TODAY()
|
|
354
|
-
start_offset_match = re.match(DAY_WEEK_OFFSET_REGEX, start_str)
|
|
355
|
-
end_offset_match = re.match(DAY_WEEK_OFFSET_REGEX, end_str)
|
|
356
|
-
|
|
357
|
-
# Additional pattern for MONTH/YEAR offsets so day/week interval slices can still be generated
|
|
358
|
-
start_monthyear_match = re.match(MONTH_YEAR_OFFSET_REGEX, start_str)
|
|
359
|
-
end_monthyear_match = re.match(MONTH_YEAR_OFFSET_REGEX, end_str)
|
|
360
|
-
|
|
361
|
-
# TIMESTAMP-wrapped offset patterns
|
|
362
|
-
start_timestamp_day_week_match = re.match(
|
|
363
|
-
TIMESTAMP_DAY_WEEK_OFFSET_REGEX, start_str
|
|
364
|
-
)
|
|
365
|
-
end_timestamp_day_week_match = re.match(
|
|
366
|
-
TIMESTAMP_DAY_WEEK_OFFSET_REGEX, end_str
|
|
367
|
-
)
|
|
368
|
-
start_timestamp_month_year_match = re.match(
|
|
369
|
-
TIMESTAMP_MONTH_YEAR_OFFSET_REGEX, start_str
|
|
370
|
-
)
|
|
371
|
-
end_timestamp_month_year_match = re.match(
|
|
372
|
-
TIMESTAMP_MONTH_YEAR_OFFSET_REGEX, end_str
|
|
373
|
-
)
|
|
374
|
-
|
|
375
|
-
# Case 1: start is offset, end is the base (NOW()/TODAY())
|
|
376
|
-
if start_offset_match and end_str == start_offset_match.group(1):
|
|
377
|
-
value = int(start_offset_match.group(2))
|
|
378
|
-
unit = start_offset_match.group(3)
|
|
379
|
-
|
|
380
|
-
return (
|
|
381
|
-
timedelta(weeks=value)
|
|
382
|
-
if unit in ["WEEK", "WEEKS"]
|
|
383
|
-
else timedelta(days=value)
|
|
384
|
-
)
|
|
385
|
-
|
|
386
|
-
# Case 1b: TIMESTAMP patterns - start is TIMESTAMP(TODAY() - N UNIT), end is TIMESTAMP(TODAY())
|
|
387
|
-
if start_timestamp_day_week_match and end_str == "TIMESTAMP(TODAY())":
|
|
388
|
-
value = int(start_timestamp_day_week_match.group(1))
|
|
389
|
-
unit = start_timestamp_day_week_match.group(2)
|
|
390
|
-
|
|
391
|
-
return (
|
|
392
|
-
timedelta(weeks=value)
|
|
393
|
-
if unit in ["WEEK", "WEEKS"]
|
|
394
|
-
else timedelta(days=value)
|
|
395
|
-
)
|
|
396
|
-
|
|
397
|
-
# Case 2: both start and end are offsets from the same base (NOW()/TODAY())
|
|
398
|
-
# E.g. start = "NOW() - 1000 DAYS", end = "NOW() - 500 DAYS"
|
|
399
|
-
if start_offset_match and end_offset_match:
|
|
400
|
-
# Ensure both reference the same base function (NOW vs TODAY) and unit type
|
|
401
|
-
if start_offset_match.group(1) == end_offset_match.group(
|
|
402
|
-
1
|
|
403
|
-
) and start_offset_match.group(3) == end_offset_match.group(3):
|
|
404
|
-
start_val = int(start_offset_match.group(2))
|
|
405
|
-
end_val = int(end_offset_match.group(2))
|
|
406
|
-
|
|
407
|
-
if start_val <= end_val:
|
|
408
|
-
raise ValueError(
|
|
409
|
-
"`start` offset must be greater (further in the past) than `end` offset"
|
|
410
|
-
)
|
|
411
|
-
|
|
412
|
-
unit = start_offset_match.group(3)
|
|
413
|
-
|
|
414
|
-
# The starting offset (start_val) represents the full duration back
|
|
415
|
-
# from NOW()/TODAY(); we iterate from that point down toward the
|
|
416
|
-
# end offset, generating slices sized by `interval`. Therefore the
|
|
417
|
-
# total duration for fixed-length slicing must be the *start* offset
|
|
418
|
-
# – not the difference between start & end.
|
|
419
|
-
|
|
420
|
-
total = start_val
|
|
421
|
-
|
|
422
|
-
return (
|
|
423
|
-
timedelta(weeks=total)
|
|
424
|
-
if unit in ["WEEK", "WEEKS"]
|
|
425
|
-
else timedelta(days=total)
|
|
426
|
-
)
|
|
427
|
-
|
|
428
|
-
# Case 2b: both start and end are TIMESTAMP offsets
|
|
429
|
-
# E.g. start = "TIMESTAMP(TODAY() - 14 DAYS)", end = "TIMESTAMP(TODAY() - 7 DAYS)"
|
|
430
|
-
if start_timestamp_day_week_match and end_timestamp_day_week_match:
|
|
431
|
-
start_val = int(start_timestamp_day_week_match.group(1))
|
|
432
|
-
end_val = int(end_timestamp_day_week_match.group(1))
|
|
433
|
-
start_unit = start_timestamp_day_week_match.group(2)
|
|
434
|
-
end_unit = end_timestamp_day_week_match.group(2)
|
|
435
|
-
|
|
436
|
-
# Ensure same unit type
|
|
437
|
-
if start_unit == end_unit:
|
|
438
|
-
if start_val <= end_val:
|
|
439
|
-
raise ValueError(
|
|
440
|
-
"`start` offset must be greater (further in the past) than `end` offset"
|
|
441
|
-
)
|
|
442
|
-
|
|
443
|
-
# For backwards iteration from the dynamic base reference point (TODAY/NOW),
|
|
444
|
-
# use the *start* offset as total duration so we iterate down toward the end bound
|
|
445
|
-
total = start_val
|
|
446
|
-
|
|
447
|
-
return (
|
|
448
|
-
timedelta(weeks=total)
|
|
449
|
-
if start_unit in ["WEEK", "WEEKS"]
|
|
450
|
-
else timedelta(days=total)
|
|
451
|
-
)
|
|
452
|
-
|
|
453
|
-
# Case 3: MONTH/YEAR offsets with DAY/WEEK interval requirements
|
|
454
|
-
if start_monthyear_match:
|
|
455
|
-
base_func = start_monthyear_match.group(1)
|
|
456
|
-
start_val_raw = int(start_monthyear_match.group(2))
|
|
457
|
-
start_unit_raw = start_monthyear_match.group(3)
|
|
458
|
-
|
|
459
|
-
# Helper: convert raw month/year units into calendar months
|
|
460
|
-
def _units_to_months(value: int, unit_raw: str) -> int:
|
|
461
|
-
return value * (12 if unit_raw.startswith("YEAR") else 1)
|
|
462
|
-
|
|
463
|
-
# If end is just the base (TODAY()/NOW())
|
|
464
|
-
if end_str == base_func:
|
|
465
|
-
total_months = _units_to_months(start_val_raw, start_unit_raw)
|
|
466
|
-
# If end is another offset with same base
|
|
467
|
-
elif end_monthyear_match and end_monthyear_match.group(1) == base_func:
|
|
468
|
-
end_val_raw = int(end_monthyear_match.group(2))
|
|
469
|
-
end_unit_raw = end_monthyear_match.group(3)
|
|
470
|
-
|
|
471
|
-
start_months = _units_to_months(start_val_raw, start_unit_raw)
|
|
472
|
-
end_months = _units_to_months(end_val_raw, end_unit_raw)
|
|
473
|
-
|
|
474
|
-
if start_months <= end_months:
|
|
475
|
-
raise ValueError("`start` offset must be greater than `end` offset")
|
|
476
|
-
|
|
477
|
-
# Use *start* offset as total duration so generator walks down toward end bound
|
|
478
|
-
total_months = start_months
|
|
479
|
-
else:
|
|
480
|
-
total_months = None
|
|
481
|
-
|
|
482
|
-
if total_months is not None:
|
|
483
|
-
# Approximate months as 4 weeks each for timedelta purposes
|
|
484
|
-
return timedelta(weeks=total_months * 4)
|
|
485
|
-
|
|
486
|
-
# Case 3b: TIMESTAMP MONTH/YEAR offsets with DAY/WEEK interval requirements
|
|
487
|
-
if start_timestamp_month_year_match:
|
|
488
|
-
start_val_raw = int(start_timestamp_month_year_match.group(1))
|
|
489
|
-
start_unit_raw = start_timestamp_month_year_match.group(2)
|
|
490
|
-
|
|
491
|
-
# Helper: convert raw month/year units into calendar months
|
|
492
|
-
def _units_to_months(value: int, unit_raw: str) -> int:
|
|
493
|
-
return value * (12 if unit_raw.startswith("YEAR") else 1)
|
|
494
|
-
|
|
495
|
-
# If end is TIMESTAMP(TODAY())
|
|
496
|
-
if end_str == "TIMESTAMP(TODAY())":
|
|
497
|
-
total_months = _units_to_months(start_val_raw, start_unit_raw)
|
|
498
|
-
# If end is another TIMESTAMP offset
|
|
499
|
-
elif end_timestamp_month_year_match:
|
|
500
|
-
end_val_raw = int(end_timestamp_month_year_match.group(1))
|
|
501
|
-
end_unit_raw = end_timestamp_month_year_match.group(2)
|
|
502
|
-
|
|
503
|
-
start_months = _units_to_months(start_val_raw, start_unit_raw)
|
|
504
|
-
end_months = _units_to_months(end_val_raw, end_unit_raw)
|
|
505
|
-
|
|
506
|
-
if start_months <= end_months:
|
|
507
|
-
raise ValueError("`start` offset must be greater than `end` offset")
|
|
508
|
-
|
|
509
|
-
# Use *start* offset as total duration
|
|
510
|
-
total_months = start_months
|
|
511
|
-
else:
|
|
512
|
-
total_months = None
|
|
513
|
-
|
|
514
|
-
if total_months is not None:
|
|
515
|
-
# Approximate months as 4 weeks each for timedelta purposes
|
|
516
|
-
return timedelta(weeks=total_months * 4)
|
|
517
|
-
|
|
518
|
-
# Static literal YYYY-MM-DD date ranges or YYYY-MM-DD HH:MM:SS datetime ranges
|
|
519
|
-
if _is_date_or_datetime_literal(start_str) and _is_date_or_datetime_literal(
|
|
520
|
-
end_str
|
|
521
|
-
):
|
|
522
|
-
start_date = _date_or_datetime_value(start_str)
|
|
523
|
-
end_date = _date_or_datetime_value(end_str)
|
|
524
|
-
|
|
525
|
-
if end_date <= start_date:
|
|
526
|
-
raise ValueError(
|
|
527
|
-
f"End date must be after start date: {self.start} to {self.end}"
|
|
528
|
-
)
|
|
529
|
-
|
|
530
|
-
return end_date - start_date
|
|
531
|
-
|
|
532
|
-
# Unsupported combination
|
|
533
|
-
raise ValueError(
|
|
534
|
-
f"Cannot calculate intervals for start='{self.start}' end='{self.end}'"
|
|
535
|
-
)
|
|
536
|
-
|
|
537
|
-
# ---------------------------------------------------------------------
|
|
538
|
-
# Slice generators
|
|
539
|
-
# ---------------------------------------------------------------------
|
|
540
|
-
|
|
541
|
-
def _generate_fixed_length_slices(
|
|
542
|
-
self,
|
|
543
|
-
interval_time_delta: timedelta,
|
|
544
|
-
total_duration: timedelta,
|
|
545
|
-
) -> List[ColumnElement]:
|
|
546
|
-
"""
|
|
547
|
-
Generate non-overlapping conditions for fixed-length (day/week) intervals.
|
|
548
|
-
"""
|
|
549
|
-
|
|
550
|
-
# First slice start is always inclusive (>=)
|
|
551
|
-
def _range_condition(
|
|
552
|
-
start_exp: ColumnElement, end_exp: ColumnElement, is_first: bool
|
|
553
|
-
) -> ColumnElement:
|
|
554
|
-
"""Return an AND clause with configurable start inclusivity."""
|
|
555
|
-
op_inclusive = is_first
|
|
556
|
-
op = (
|
|
557
|
-
column(self.field) >= start_exp
|
|
558
|
-
if op_inclusive
|
|
559
|
-
else column(self.field) > start_exp
|
|
560
|
-
)
|
|
561
|
-
return and_(op, column(self.field) <= end_exp)
|
|
562
|
-
|
|
563
|
-
conditions: List[ColumnElement] = []
|
|
564
|
-
|
|
565
|
-
# Dynamic (NOW()/TODAY()) vs static ranges
|
|
566
|
-
start_str = self.start or ""
|
|
567
|
-
end_str = self.end or ""
|
|
568
|
-
|
|
569
|
-
has_now = "NOW()" in start_str or "NOW()" in end_str
|
|
570
|
-
has_today = "TODAY()" in start_str or "TODAY()" in end_str
|
|
571
|
-
has_timestamp = "TIMESTAMP(" in start_str or "TIMESTAMP(" in end_str
|
|
572
|
-
has_timestamp_literal = (
|
|
573
|
-
_is_timestamp_date_literal(start_str)
|
|
574
|
-
or _is_timestamp_date_literal(end_str)
|
|
575
|
-
or _is_timestamp_datetime_literal(start_str)
|
|
576
|
-
or _is_timestamp_datetime_literal(end_str)
|
|
577
|
-
)
|
|
578
|
-
|
|
579
|
-
if has_now or has_today:
|
|
580
|
-
# Treat both NOW() and TODAY() as dynamic, using the appropriate base SQL
|
|
581
|
-
base_expr = func.current_timestamp() if has_now else func.current_date()
|
|
582
|
-
end_expr = self._parse_time_expression(end_str)
|
|
583
|
-
|
|
584
|
-
# Helper function to apply timestamp wrapping when TIMESTAMP patterns are used
|
|
585
|
-
def _maybe_wrap_timestamp(expr: ColumnElement) -> ColumnElement:
|
|
586
|
-
if (
|
|
587
|
-
has_timestamp and not has_now and not has_timestamp_literal
|
|
588
|
-
): # TIMESTAMP(TODAY()) patterns but not NOW() and not TIMESTAMP('date') literals
|
|
589
|
-
return func.timestamp(expr)
|
|
590
|
-
return expr
|
|
591
|
-
|
|
592
|
-
# Determine the relative offset (timedelta) that represents the user
|
|
593
|
-
# supplied `end` bound so we do not generate slices beyond it. If the
|
|
594
|
-
# end expression is simply NOW()/TODAY(), the offset is zero. If it
|
|
595
|
-
# is an expression like "NOW() - 500 DAYS" we extract the numeric
|
|
596
|
-
# offset so we can stop the fixed-length iteration once we reach it.
|
|
597
|
-
|
|
598
|
-
end_offset_bound: timedelta = timedelta(0)
|
|
599
|
-
|
|
600
|
-
dynamic_end_match = re.match(DAY_WEEK_OFFSET_REGEX, end_str)
|
|
601
|
-
timestamp_end_match = re.match(TIMESTAMP_DAY_WEEK_OFFSET_REGEX, end_str)
|
|
602
|
-
|
|
603
|
-
if dynamic_end_match is not None:
|
|
604
|
-
numeric_val = int(dynamic_end_match.group(2))
|
|
605
|
-
unit_raw = dynamic_end_match.group(3)
|
|
606
|
-
end_offset_bound = (
|
|
607
|
-
timedelta(weeks=numeric_val)
|
|
608
|
-
if TimeUnit.parse(unit_raw) == TimeUnit.WEEK
|
|
609
|
-
else timedelta(days=numeric_val)
|
|
610
|
-
)
|
|
611
|
-
elif timestamp_end_match is not None:
|
|
612
|
-
numeric_val = int(timestamp_end_match.group(1))
|
|
613
|
-
unit_raw = timestamp_end_match.group(2)
|
|
614
|
-
end_offset_bound = (
|
|
615
|
-
timedelta(weeks=numeric_val)
|
|
616
|
-
if TimeUnit.parse(unit_raw) == TimeUnit.WEEK
|
|
617
|
-
else timedelta(days=numeric_val)
|
|
618
|
-
)
|
|
619
|
-
|
|
620
|
-
# Extend dynamic_end_match for month/year offsets as weeks approximation
|
|
621
|
-
if dynamic_end_match is None and timestamp_end_match is None:
|
|
622
|
-
dynamic_end_match = re.match(MONTH_YEAR_OFFSET_REGEX, end_str)
|
|
623
|
-
timestamp_end_match = re.match(
|
|
624
|
-
TIMESTAMP_MONTH_YEAR_OFFSET_REGEX, end_str
|
|
625
|
-
)
|
|
626
|
-
|
|
627
|
-
def _offset_to_timedelta(match_obj: Optional[re.Match]) -> timedelta:
|
|
628
|
-
"""Convert an offset regex match into a timedelta (weeks/days).
|
|
629
|
-
|
|
630
|
-
For month/year units we approximate: month≈4 weeks, year≈52 weeks.
|
|
631
|
-
"""
|
|
632
|
-
|
|
633
|
-
if match_obj is None:
|
|
634
|
-
return timedelta(0)
|
|
635
|
-
|
|
636
|
-
numeric_val = int(match_obj.group(2))
|
|
637
|
-
unit_raw = match_obj.group(3)
|
|
638
|
-
|
|
639
|
-
unit_enum = TimeUnit.parse(unit_raw)
|
|
640
|
-
|
|
641
|
-
if unit_enum is TimeUnit.WEEK:
|
|
642
|
-
return timedelta(weeks=numeric_val)
|
|
643
|
-
if unit_enum is TimeUnit.DAY:
|
|
644
|
-
return timedelta(days=numeric_val)
|
|
645
|
-
|
|
646
|
-
# Approximate month/year for slicing purposes
|
|
647
|
-
if unit_enum is TimeUnit.MONTH:
|
|
648
|
-
return timedelta(weeks=numeric_val * 4)
|
|
649
|
-
if unit_enum is TimeUnit.YEAR:
|
|
650
|
-
return timedelta(weeks=numeric_val * 52)
|
|
651
|
-
|
|
652
|
-
return timedelta(0)
|
|
653
|
-
|
|
654
|
-
def _timestamp_offset_to_timedelta(
|
|
655
|
-
match_obj: Optional[re.Match],
|
|
656
|
-
) -> timedelta:
|
|
657
|
-
"""Convert a TIMESTAMP offset regex match into a timedelta (weeks/days).
|
|
658
|
-
|
|
659
|
-
For month/year units we approximate: month≈4 weeks, year≈52 weeks.
|
|
660
|
-
"""
|
|
661
|
-
|
|
662
|
-
if match_obj is None:
|
|
663
|
-
return timedelta(0)
|
|
664
|
-
|
|
665
|
-
numeric_val = int(match_obj.group(1))
|
|
666
|
-
unit_raw = match_obj.group(2)
|
|
667
|
-
|
|
668
|
-
unit_enum = TimeUnit.parse(unit_raw)
|
|
669
|
-
|
|
670
|
-
if unit_enum is TimeUnit.WEEK:
|
|
671
|
-
return timedelta(weeks=numeric_val)
|
|
672
|
-
if unit_enum is TimeUnit.DAY:
|
|
673
|
-
return timedelta(days=numeric_val)
|
|
674
|
-
|
|
675
|
-
# Approximate month/year for slicing purposes
|
|
676
|
-
if unit_enum is TimeUnit.MONTH:
|
|
677
|
-
return timedelta(weeks=numeric_val * 4)
|
|
678
|
-
if unit_enum is TimeUnit.YEAR:
|
|
679
|
-
return timedelta(weeks=numeric_val * 52)
|
|
680
|
-
|
|
681
|
-
return timedelta(0)
|
|
682
|
-
|
|
683
|
-
# Update end_offset_bound for month/year patterns if not already set
|
|
684
|
-
if end_offset_bound == timedelta(0):
|
|
685
|
-
if dynamic_end_match is not None:
|
|
686
|
-
end_offset_bound = _offset_to_timedelta(dynamic_end_match)
|
|
687
|
-
elif timestamp_end_match is not None:
|
|
688
|
-
end_offset_bound = _timestamp_offset_to_timedelta(
|
|
689
|
-
timestamp_end_match
|
|
690
|
-
)
|
|
691
|
-
|
|
692
|
-
# Iterate from the starting offset (total_duration) backward toward
|
|
693
|
-
# the end offset, generating non-overlapping slices until just before
|
|
694
|
-
# we would cross the end bound.
|
|
695
|
-
|
|
696
|
-
current_offset = total_duration
|
|
697
|
-
first_interval = self.inclusive_start
|
|
698
|
-
|
|
699
|
-
while current_offset > end_offset_bound:
|
|
700
|
-
start_offset = current_offset
|
|
701
|
-
end_offset = current_offset - interval_time_delta
|
|
702
|
-
|
|
703
|
-
# If subtracting the interval would step past the end bound,
|
|
704
|
-
# clamp the end_offset to the bound so the final slice lands
|
|
705
|
-
# exactly on the user-provided end expression.
|
|
706
|
-
end_offset = max(end_offset, end_offset_bound)
|
|
707
|
-
|
|
708
|
-
# Clamp negative end offsets to zero (aligns with base_expr)
|
|
709
|
-
if end_offset.total_seconds() < 0:
|
|
710
|
-
end_offset = timedelta(0)
|
|
711
|
-
|
|
712
|
-
# Build expressions first – needed for equality check
|
|
713
|
-
interval_start = (
|
|
714
|
-
_maybe_wrap_timestamp(base_expr)
|
|
715
|
-
if start_offset.total_seconds() == 0
|
|
716
|
-
else _maybe_wrap_timestamp(
|
|
717
|
-
base_expr - text(self._timedelta_to_interval(start_offset))
|
|
718
|
-
)
|
|
719
|
-
)
|
|
720
|
-
|
|
721
|
-
interval_end = (
|
|
722
|
-
end_expr
|
|
723
|
-
if end_offset.total_seconds() == 0
|
|
724
|
-
else _maybe_wrap_timestamp(
|
|
725
|
-
base_expr - text(self._timedelta_to_interval(end_offset))
|
|
726
|
-
)
|
|
727
|
-
)
|
|
728
|
-
|
|
729
|
-
# Skip zero-length slice
|
|
730
|
-
if str(interval_start) == str(interval_end):
|
|
731
|
-
break
|
|
732
|
-
|
|
733
|
-
conditions.append(
|
|
734
|
-
_range_condition(interval_start, interval_end, first_interval)
|
|
735
|
-
)
|
|
736
|
-
first_interval = False
|
|
737
|
-
current_offset = end_offset
|
|
738
|
-
else:
|
|
739
|
-
# Static literal or dynamic expressions already parsed – walk forward from start
|
|
740
|
-
start_expr = self._parse_time_expression(start_str)
|
|
741
|
-
end_expr = self._parse_time_expression(end_str)
|
|
742
|
-
|
|
743
|
-
# Helper function to apply timestamp wrapping when TIMESTAMP patterns are used
|
|
744
|
-
def _maybe_wrap_timestamp(expr: ColumnElement) -> ColumnElement:
|
|
745
|
-
if (
|
|
746
|
-
has_timestamp and not has_now and not has_timestamp_literal
|
|
747
|
-
): # TIMESTAMP(TODAY()) patterns but not NOW() and not TIMESTAMP('date') literals
|
|
748
|
-
return func.timestamp(expr)
|
|
749
|
-
return expr
|
|
750
|
-
|
|
751
|
-
current_offset = timedelta(0)
|
|
752
|
-
first_interval = self.inclusive_start
|
|
753
|
-
|
|
754
|
-
while current_offset < total_duration:
|
|
755
|
-
interval_start = (
|
|
756
|
-
start_expr
|
|
757
|
-
if current_offset.total_seconds() == 0
|
|
758
|
-
else _maybe_wrap_timestamp(
|
|
759
|
-
start_expr + text(self._timedelta_to_interval(current_offset))
|
|
760
|
-
)
|
|
761
|
-
)
|
|
762
|
-
|
|
763
|
-
next_offset = current_offset + interval_time_delta
|
|
764
|
-
interval_end = (
|
|
765
|
-
end_expr
|
|
766
|
-
if next_offset >= total_duration
|
|
767
|
-
else _maybe_wrap_timestamp(
|
|
768
|
-
start_expr + text(self._timedelta_to_interval(next_offset))
|
|
769
|
-
)
|
|
770
|
-
)
|
|
771
|
-
|
|
772
|
-
conditions.append(
|
|
773
|
-
_range_condition(interval_start, interval_end, first_interval)
|
|
774
|
-
)
|
|
775
|
-
first_interval = False
|
|
776
|
-
current_offset = next_offset
|
|
777
|
-
|
|
778
|
-
return conditions
|
|
779
|
-
|
|
780
|
-
def _generate_calendar_slices(
|
|
781
|
-
self,
|
|
782
|
-
field_column: ColumnElement,
|
|
783
|
-
value: int,
|
|
784
|
-
unit: TimeUnit,
|
|
785
|
-
) -> List[ColumnElement]:
|
|
786
|
-
"""Generate interval conditions for month/year units where start & end are literals."""
|
|
787
|
-
|
|
788
|
-
start_str = self.start or ""
|
|
789
|
-
end_str = self.end or ""
|
|
790
|
-
|
|
791
|
-
if not (_is_date_literal(start_str) and _is_date_literal(end_str)):
|
|
792
|
-
# Fallback to single clause; complex dynamic expressions not supported yet.
|
|
793
|
-
start_expr = (
|
|
794
|
-
None if self.start is None else self._parse_time_expression(start_str)
|
|
795
|
-
)
|
|
796
|
-
end_expr = (
|
|
797
|
-
None if self.end is None else self._parse_time_expression(end_str)
|
|
798
|
-
)
|
|
799
|
-
|
|
800
|
-
if start_expr is not None and end_expr is not None:
|
|
801
|
-
return [and_(field_column >= start_expr, field_column <= end_expr)]
|
|
802
|
-
if start_expr is not None:
|
|
803
|
-
return [field_column >= start_expr]
|
|
804
|
-
if end_expr is not None:
|
|
805
|
-
return [field_column <= end_expr]
|
|
806
|
-
|
|
807
|
-
# Both start/end are literals – rely on SQL to handle calendar nuances
|
|
808
|
-
start_date = _date_value(start_str)
|
|
809
|
-
end_date = _date_value(end_str)
|
|
810
|
-
|
|
811
|
-
total_units = (
|
|
812
|
-
(end_date.year - start_date.year) * 12 + (end_date.month - start_date.month)
|
|
813
|
-
if unit is TimeUnit.MONTH
|
|
814
|
-
else end_date.year - start_date.year
|
|
815
|
-
)
|
|
816
|
-
|
|
817
|
-
# If the end day is greater than start day, ensure we include the partial unit
|
|
818
|
-
if unit is TimeUnit.MONTH and end_date.day > start_date.day:
|
|
819
|
-
total_units += 1
|
|
820
|
-
|
|
821
|
-
iterations = (total_units + value - 1) // value # ceil division
|
|
822
|
-
|
|
823
|
-
base_expr = func.DATE(start_date.strftime("%Y-%m-%d"))
|
|
824
|
-
|
|
825
|
-
def offset_expr(multiplier: int) -> ColumnElement:
|
|
826
|
-
if multiplier == 0:
|
|
827
|
-
return base_expr
|
|
828
|
-
offset = multiplier * value
|
|
829
|
-
return base_expr + text(self.format_interval(offset, unit))
|
|
830
|
-
|
|
831
|
-
conditions: List[ColumnElement] = []
|
|
832
|
-
inclusive_start = self.inclusive_start # Respect spec setting
|
|
833
|
-
for i in range(iterations):
|
|
834
|
-
start_exp = offset_expr(i)
|
|
835
|
-
|
|
836
|
-
# Determine end expression for this slice
|
|
837
|
-
next_multiplier = i + 1
|
|
838
|
-
end_exp = (
|
|
839
|
-
func.DATE(end_date.strftime("%Y-%m-%d"))
|
|
840
|
-
if next_multiplier * value >= total_units
|
|
841
|
-
else offset_expr(next_multiplier)
|
|
842
|
-
)
|
|
843
|
-
|
|
844
|
-
conditions.append(
|
|
845
|
-
and_(
|
|
846
|
-
(
|
|
847
|
-
(field_column >= start_exp)
|
|
848
|
-
if inclusive_start
|
|
849
|
-
else (field_column > start_exp)
|
|
850
|
-
),
|
|
851
|
-
field_column <= end_exp,
|
|
852
|
-
)
|
|
853
|
-
)
|
|
854
|
-
inclusive_start = False
|
|
855
|
-
|
|
856
|
-
return conditions
|
|
857
|
-
|
|
858
|
-
def _generate_dynamic_month_year_slices(
|
|
859
|
-
self,
|
|
860
|
-
field_column: ColumnElement,
|
|
861
|
-
value: int,
|
|
862
|
-
unit: TimeUnit,
|
|
863
|
-
) -> Optional[List[ColumnElement]]:
|
|
864
|
-
"""Generate slices for dynamic ranges expressed in months or years.
|
|
865
|
-
|
|
866
|
-
Supported patterns
|
|
867
|
-
-----------------
|
|
868
|
-
1. `NOW() - N <unit>s TO NOW()` (timestamp based)
|
|
869
|
-
2. `TODAY() - N <unit>s TO TODAY()` (date based)
|
|
870
|
-
3. `TIMESTAMP(TODAY() - N <unit>s) TO TIMESTAMP(TODAY())` (timestamp based)
|
|
871
|
-
|
|
872
|
-
The helper chooses `CURRENT_TIMESTAMP` or `CURRENT_DATE` automatically
|
|
873
|
-
based on which pattern is matched.
|
|
874
|
-
"""
|
|
875
|
-
|
|
876
|
-
start_str = self.start or ""
|
|
877
|
-
end_str = self.end or ""
|
|
878
|
-
|
|
879
|
-
# --- START PATTERN MATCHES -------------------------------------------------
|
|
880
|
-
# - Scenario A: start is offset, end is TODAY()/NOW()
|
|
881
|
-
# - Scenario B: start and end are both offsets from the same base (NOW/TODAY)
|
|
882
|
-
# - Scenario C: start is TIMESTAMP offset, end is TIMESTAMP(TODAY())
|
|
883
|
-
# - Scenario D: start and end are both TIMESTAMP offsets
|
|
884
|
-
|
|
885
|
-
start_match = re.match(MONTH_YEAR_OFFSET_REGEX, start_str)
|
|
886
|
-
start_timestamp_match = re.match(TIMESTAMP_MONTH_YEAR_OFFSET_REGEX, start_str)
|
|
887
|
-
|
|
888
|
-
if start_match is None and start_timestamp_match is None:
|
|
889
|
-
return None
|
|
890
|
-
|
|
891
|
-
end_match = re.match(MONTH_YEAR_OFFSET_REGEX, end_str)
|
|
892
|
-
end_timestamp_match = re.match(TIMESTAMP_MONTH_YEAR_OFFSET_REGEX, end_str)
|
|
893
|
-
|
|
894
|
-
# Determine which pattern we're using and extract values
|
|
895
|
-
if start_match is not None:
|
|
896
|
-
# Traditional NOW()/TODAY() pattern
|
|
897
|
-
base_func_name = start_match.group(1) # NOW or TODAY
|
|
898
|
-
start_units_raw = int(start_match.group(2))
|
|
899
|
-
start_unit_enum = TimeUnit.parse(start_match.group(3))
|
|
900
|
-
use_current_date = base_func_name == "TODAY()"
|
|
901
|
-
is_timestamp_pattern = False
|
|
902
|
-
elif start_timestamp_match is not None:
|
|
903
|
-
# TIMESTAMP pattern
|
|
904
|
-
start_units_raw = int(start_timestamp_match.group(1))
|
|
905
|
-
start_unit_enum = TimeUnit.parse(start_timestamp_match.group(2))
|
|
906
|
-
use_current_date = True # TIMESTAMP(TODAY()) uses CURRENT_DATE as base
|
|
907
|
-
is_timestamp_pattern = True
|
|
908
|
-
else:
|
|
909
|
-
# This shouldn't happen due to the earlier check, but makes mypy happy
|
|
910
|
-
return None
|
|
911
|
-
|
|
912
|
-
# Helper to convert raw units (possibly years) into desired interval unit count
|
|
913
|
-
def _convert_units(raw_units: int, raw_unit_enum: TimeUnit) -> Optional[int]:
|
|
914
|
-
if raw_unit_enum is unit:
|
|
915
|
-
return raw_units
|
|
916
|
-
if raw_unit_enum is TimeUnit.YEAR and unit is TimeUnit.MONTH:
|
|
917
|
-
return raw_units * 12
|
|
918
|
-
return None # unsupported mismatch
|
|
919
|
-
|
|
920
|
-
conditions: List[ColumnElement] = []
|
|
921
|
-
|
|
922
|
-
# Scenario A: `... TO NOW()/TODAY()` or `... TO TIMESTAMP(TODAY())`
|
|
923
|
-
# Break down complex condition for pylint
|
|
924
|
-
traditional_to_base = not is_timestamp_pattern and end_match is None
|
|
925
|
-
traditional_to_zero_offset = (
|
|
926
|
-
not is_timestamp_pattern
|
|
927
|
-
and end_match is not None
|
|
928
|
-
and end_match.group(1) == base_func_name
|
|
929
|
-
and int(end_match.group(2)) == 0
|
|
930
|
-
)
|
|
931
|
-
timestamp_to_today = is_timestamp_pattern and end_str == "TIMESTAMP(TODAY())"
|
|
932
|
-
|
|
933
|
-
if traditional_to_base or traditional_to_zero_offset or timestamp_to_today:
|
|
934
|
-
total_units = _convert_units(start_units_raw, start_unit_enum)
|
|
935
|
-
if total_units is None or total_units % value != 0:
|
|
936
|
-
return None
|
|
937
|
-
|
|
938
|
-
iterations = total_units // value
|
|
939
|
-
inclusive_start = self.inclusive_start
|
|
940
|
-
base_expr = (
|
|
941
|
-
func.current_date() if use_current_date else func.current_timestamp()
|
|
942
|
-
)
|
|
943
|
-
|
|
944
|
-
for i in range(iterations):
|
|
945
|
-
start_offset_units = total_units - (i * value)
|
|
946
|
-
end_offset_units = start_offset_units - value
|
|
947
|
-
|
|
948
|
-
start_expr = base_expr - text(
|
|
949
|
-
self.format_interval(start_offset_units, unit)
|
|
950
|
-
)
|
|
951
|
-
if end_offset_units == 0:
|
|
952
|
-
if is_timestamp_pattern:
|
|
953
|
-
end_expr = func.timestamp(base_expr)
|
|
954
|
-
else:
|
|
955
|
-
end_expr = base_expr
|
|
956
|
-
else:
|
|
957
|
-
end_expr = base_expr - text(
|
|
958
|
-
self.format_interval(end_offset_units, unit)
|
|
959
|
-
)
|
|
960
|
-
|
|
961
|
-
start_op = (
|
|
962
|
-
field_column >= start_expr
|
|
963
|
-
if inclusive_start
|
|
964
|
-
else field_column > start_expr
|
|
965
|
-
)
|
|
966
|
-
conditions.append(and_(start_op, field_column <= end_expr))
|
|
967
|
-
inclusive_start = False
|
|
968
|
-
|
|
969
|
-
return conditions
|
|
970
|
-
|
|
971
|
-
# Scenario B: both start and end are offsets (traditional or TIMESTAMP)
|
|
972
|
-
if (
|
|
973
|
-
not is_timestamp_pattern
|
|
974
|
-
and end_match is not None
|
|
975
|
-
and end_match.group(1) == base_func_name
|
|
976
|
-
) or (is_timestamp_pattern and end_timestamp_match is not None):
|
|
977
|
-
|
|
978
|
-
if is_timestamp_pattern and end_timestamp_match is not None:
|
|
979
|
-
end_units_raw = int(end_timestamp_match.group(1))
|
|
980
|
-
end_unit_enum = TimeUnit.parse(end_timestamp_match.group(2))
|
|
981
|
-
elif end_match is not None:
|
|
982
|
-
end_units_raw = int(end_match.group(2))
|
|
983
|
-
end_unit_enum = TimeUnit.parse(end_match.group(3))
|
|
984
|
-
else:
|
|
985
|
-
# This shouldn't happen due to the earlier check, but makes mypy happy
|
|
986
|
-
return None
|
|
987
|
-
|
|
988
|
-
start_units_converted = _convert_units(start_units_raw, start_unit_enum)
|
|
989
|
-
end_units_converted = _convert_units(end_units_raw, end_unit_enum)
|
|
990
|
-
if (
|
|
991
|
-
start_units_converted is None
|
|
992
|
-
or end_units_converted is None
|
|
993
|
-
or start_units_converted <= end_units_converted
|
|
994
|
-
):
|
|
995
|
-
return None
|
|
996
|
-
|
|
997
|
-
total_units = start_units_converted - end_units_converted
|
|
998
|
-
if total_units % value != 0:
|
|
999
|
-
return None
|
|
1000
|
-
|
|
1001
|
-
iterations = total_units // value
|
|
1002
|
-
inclusive_start = self.inclusive_start
|
|
1003
|
-
base_expr = (
|
|
1004
|
-
func.current_date() if use_current_date else func.current_timestamp()
|
|
1005
|
-
)
|
|
1006
|
-
|
|
1007
|
-
for i in range(iterations):
|
|
1008
|
-
slice_start_units = start_units_converted - (i * value)
|
|
1009
|
-
slice_end_units = slice_start_units - value
|
|
1010
|
-
|
|
1011
|
-
start_expr = base_expr - text(
|
|
1012
|
-
self.format_interval(slice_start_units, unit)
|
|
1013
|
-
)
|
|
1014
|
-
if is_timestamp_pattern:
|
|
1015
|
-
end_expr = func.timestamp(
|
|
1016
|
-
base_expr - text(self.format_interval(slice_end_units, unit))
|
|
1017
|
-
)
|
|
1018
|
-
else:
|
|
1019
|
-
end_expr = base_expr - text(
|
|
1020
|
-
self.format_interval(slice_end_units, unit)
|
|
1021
|
-
)
|
|
1022
|
-
|
|
1023
|
-
start_op = (
|
|
1024
|
-
field_column >= start_expr
|
|
1025
|
-
if inclusive_start
|
|
1026
|
-
else field_column > start_expr
|
|
1027
|
-
)
|
|
1028
|
-
conditions.append(and_(start_op, field_column <= end_expr))
|
|
1029
|
-
inclusive_start = False
|
|
1030
|
-
|
|
1031
|
-
return conditions
|
|
1032
|
-
|
|
1033
|
-
return None # Fallback to other generation paths
|
|
1034
|
-
|
|
1035
|
-
def _generate_literal_to_now_month_year_slices(
|
|
1036
|
-
self,
|
|
1037
|
-
field_column: ColumnElement,
|
|
1038
|
-
value: int,
|
|
1039
|
-
unit: TimeUnit,
|
|
1040
|
-
) -> List[ColumnElement]:
|
|
1041
|
-
"""Generate partitions when start is a YYYY-MM-DD literal and end
|
|
1042
|
-
is NOW() or TODAY() with month/year interval.
|
|
1043
|
-
|
|
1044
|
-
The logic mirrors _generate_calendar_slices, but the end
|
|
1045
|
-
bound is dynamic so we calculate the number of complete units between
|
|
1046
|
-
start and the current date/time at runtime.
|
|
1047
|
-
"""
|
|
1048
|
-
|
|
1049
|
-
assert self.start is not None # for mypy
|
|
1050
|
-
start_date = _date_value(self.start)
|
|
1051
|
-
|
|
1052
|
-
# Choose appropriate SQL base expressions depending on TODAY/NOW/TIMESTAMP(TODAY())
|
|
1053
|
-
end_str = str(self.end)
|
|
1054
|
-
end_is_today = end_str == "TODAY()"
|
|
1055
|
-
end_is_timestamp_today = end_str == "TIMESTAMP(TODAY())"
|
|
1056
|
-
|
|
1057
|
-
if end_is_timestamp_today:
|
|
1058
|
-
base_now_expr = func.timestamp(func.current_date())
|
|
1059
|
-
else:
|
|
1060
|
-
base_now_expr = (
|
|
1061
|
-
func.current_date() if end_is_today else func.current_timestamp()
|
|
1062
|
-
)
|
|
1063
|
-
|
|
1064
|
-
# Calculate total units between start literal and "now"
|
|
1065
|
-
now_dt = datetime.utcnow()
|
|
1066
|
-
|
|
1067
|
-
if unit is TimeUnit.MONTH:
|
|
1068
|
-
total_units = (now_dt.year - start_date.year) * 12 + (
|
|
1069
|
-
now_dt.month - start_date.month
|
|
1070
|
-
)
|
|
1071
|
-
if now_dt.day > start_date.day:
|
|
1072
|
-
total_units += 1
|
|
1073
|
-
else: # TimeUnit.YEAR
|
|
1074
|
-
total_units = now_dt.year - start_date.year
|
|
1075
|
-
if (now_dt.month, now_dt.day) > (start_date.month, start_date.day):
|
|
1076
|
-
total_units += 1
|
|
1077
|
-
|
|
1078
|
-
iterations = (total_units + value - 1) // value # ceil division
|
|
1079
|
-
|
|
1080
|
-
base_expr = func.DATE(start_date.strftime("%Y-%m-%d"))
|
|
1081
|
-
|
|
1082
|
-
def offset_expr(multiplier: int) -> ColumnElement:
|
|
1083
|
-
if multiplier == 0:
|
|
1084
|
-
return base_expr
|
|
1085
|
-
offset = multiplier * value
|
|
1086
|
-
return base_expr + text(self.format_interval(offset, unit))
|
|
1087
|
-
|
|
1088
|
-
conditions: List[ColumnElement] = []
|
|
1089
|
-
inclusive_start = self.inclusive_start # First slice always inclusive
|
|
1090
|
-
|
|
1091
|
-
for i in range(iterations):
|
|
1092
|
-
start_exp = offset_expr(i)
|
|
1093
|
-
|
|
1094
|
-
next_mult = i + 1
|
|
1095
|
-
end_exp = (
|
|
1096
|
-
base_now_expr
|
|
1097
|
-
if next_mult * value >= total_units
|
|
1098
|
-
else offset_expr(next_mult)
|
|
1099
|
-
)
|
|
1100
|
-
|
|
1101
|
-
start_op = (
|
|
1102
|
-
field_column >= start_exp
|
|
1103
|
-
if inclusive_start
|
|
1104
|
-
else field_column > start_exp
|
|
1105
|
-
)
|
|
1106
|
-
conditions.append(and_(start_op, field_column <= end_exp))
|
|
1107
|
-
inclusive_start = False
|
|
1108
|
-
|
|
1109
|
-
return conditions
|
|
1110
|
-
|
|
1111
|
-
# ---------------------------------------------------------------------
|
|
1112
|
-
# Public methods
|
|
1113
|
-
# ---------------------------------------------------------------------
|
|
1114
|
-
|
|
1115
|
-
def format_interval(self, value: int, unit: TimeUnit) -> str:
|
|
1116
|
-
"""Return a SQL snippet representing time interval.
|
|
1117
|
-
|
|
1118
|
-
Default implementation emits the ANSI-ish style used by BigQuery and
|
|
1119
|
-
many other engines: `INTERVAL <N> DAY` or `INTERVAL <N> WEEK`.
|
|
1120
|
-
|
|
1121
|
-
Subclasses may override if their dialect differs (e.g., Postgres prefers
|
|
1122
|
-
`INTERVAL '<N> day'`).
|
|
1123
|
-
"""
|
|
1124
|
-
unit_str = unit.value
|
|
1125
|
-
return f"INTERVAL {value} {unit_str}"
|
|
1126
|
-
|
|
1127
|
-
def generate_expressions(self) -> List[ColumnElement]:
|
|
1128
|
-
"""
|
|
1129
|
-
Generate SQLAlchemy WHERE conditions for time-based or open-ended partitioning.
|
|
1130
|
-
This is the main entry point for the partition generation logic.
|
|
1131
|
-
"""
|
|
1132
|
-
|
|
1133
|
-
field_column: ColumnElement = column(self.field)
|
|
1134
|
-
|
|
1135
|
-
# Open-ended scenarios
|
|
1136
|
-
if self.start is None and self.end is not None:
|
|
1137
|
-
assert self.end is not None # for mypy
|
|
1138
|
-
end_expr = self._parse_time_expression(self.end)
|
|
1139
|
-
return [field_column <= end_expr]
|
|
1140
|
-
|
|
1141
|
-
if self.end is None and self.start is not None:
|
|
1142
|
-
start_expr = self._parse_time_expression(self.start)
|
|
1143
|
-
return [field_column >= start_expr]
|
|
1144
|
-
|
|
1145
|
-
if self.start is None or self.end is None:
|
|
1146
|
-
raise ValueError(
|
|
1147
|
-
"Either both start and end must be provided, or one open-bound with the other bound specified."
|
|
1148
|
-
)
|
|
1149
|
-
|
|
1150
|
-
if self.interval is None:
|
|
1151
|
-
# No interval specified - return a single condition covering the entire range
|
|
1152
|
-
start_expr = self._parse_time_expression(self.start)
|
|
1153
|
-
end_expr = self._parse_time_expression(self.end)
|
|
1154
|
-
start_op = (
|
|
1155
|
-
field_column >= start_expr
|
|
1156
|
-
if self.inclusive_start
|
|
1157
|
-
else field_column > start_expr
|
|
1158
|
-
)
|
|
1159
|
-
return [and_(start_op, field_column <= end_expr)]
|
|
1160
|
-
|
|
1161
|
-
# Determine interval value and unit (Enum)
|
|
1162
|
-
interval_value, interval_unit = self._split_interval()
|
|
1163
|
-
|
|
1164
|
-
if interval_unit in (TimeUnit.MONTH, TimeUnit.YEAR):
|
|
1165
|
-
end_str = self.end or ""
|
|
1166
|
-
|
|
1167
|
-
# Try dynamic slice generation for various NOW()/TODAY() offset scenarios.
|
|
1168
|
-
dynamic_expressions = self._generate_dynamic_month_year_slices(
|
|
1169
|
-
field_column, interval_value, interval_unit
|
|
1170
|
-
)
|
|
1171
|
-
if dynamic_expressions is not None:
|
|
1172
|
-
return dynamic_expressions
|
|
1173
|
-
|
|
1174
|
-
# Literal start -> NOW()/TODAY()/TIMESTAMP(TODAY()) end
|
|
1175
|
-
if _is_date_literal(self.start) and end_str in (
|
|
1176
|
-
"NOW()",
|
|
1177
|
-
"TODAY()",
|
|
1178
|
-
"TIMESTAMP(TODAY())",
|
|
1179
|
-
):
|
|
1180
|
-
return self._generate_literal_to_now_month_year_slices(
|
|
1181
|
-
field_column, interval_value, interval_unit
|
|
1182
|
-
)
|
|
1183
|
-
|
|
1184
|
-
# Fallback to calendar slicing when both bounds are literals
|
|
1185
|
-
return self._generate_calendar_slices(
|
|
1186
|
-
field_column, interval_value, interval_unit
|
|
1187
|
-
)
|
|
1188
|
-
|
|
1189
|
-
# day/week (fixed length) path
|
|
1190
|
-
interval_time_delta = self._timedelta_from_value_unit(
|
|
1191
|
-
interval_value, interval_unit
|
|
1192
|
-
)
|
|
1193
|
-
total_duration = self._calculate_total_duration()
|
|
1194
|
-
|
|
1195
|
-
return self._generate_fixed_length_slices(
|
|
1196
|
-
interval_time_delta,
|
|
1197
|
-
total_duration,
|
|
1198
|
-
)
|
|
1199
|
-
|
|
1200
|
-
def generate_where_clauses(self) -> List[str]:
|
|
1201
|
-
"""
|
|
1202
|
-
Generate SQLAlchemy WHERE conditions for time-based partitioning.
|
|
1203
|
-
This needs to be implemented by subclasses to generate the
|
|
1204
|
-
dialect-specific WHERE clauses.
|
|
1205
|
-
"""
|
|
1206
|
-
|
|
1207
|
-
raise NotImplementedError("generate_where_clauses not implemented")
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
def validate_partitioning_list(partitionings: List["TimeBasedPartitioning"]) -> None:
|
|
1211
|
-
"""Validate that multiple TimeBasedPartitioning objects do not define overlapping
|
|
1212
|
-
ranges (inclusive) and that all partitioning objects use the same field name.
|
|
1213
|
-
Only specs whose provided bounds are literal YYYY-MM-DD strings participate in
|
|
1214
|
-
overlap validation; any bound that is None is treated as open-ended (-infinity
|
|
1215
|
-
for `start` or +infinity for `end`). Specs with dynamic expressions like `NOW()`
|
|
1216
|
-
are skipped because we cannot resolve them at validation time.
|
|
1217
|
-
"""
|
|
1218
|
-
|
|
1219
|
-
# Validate that all partitions use the same field
|
|
1220
|
-
if len(partitionings) > 1:
|
|
1221
|
-
first_field = partitionings[0].field
|
|
1222
|
-
for i, partition in enumerate(partitionings[1:], 1):
|
|
1223
|
-
if partition.field != first_field:
|
|
1224
|
-
raise ValueError(
|
|
1225
|
-
f"All partitioning specifications must use the same field. "
|
|
1226
|
-
f"Expected '{first_field}' but found '{partition.field}' at index {i}."
|
|
1227
|
-
)
|
|
1228
|
-
|
|
1229
|
-
def _materialize(
|
|
1230
|
-
p: "TimeBasedPartitioning",
|
|
1231
|
-
) -> Optional[Tuple[datetime, datetime, "TimeBasedPartitioning"]]:
|
|
1232
|
-
if p.start is not None and not _is_date_literal(p.start):
|
|
1233
|
-
return None # skip – dynamic expression
|
|
1234
|
-
if p.end is not None and not _is_date_literal(p.end):
|
|
1235
|
-
return None # skip – dynamic expression
|
|
1236
|
-
|
|
1237
|
-
start_dt = _date_value(p.start) if p.start is not None else datetime.min
|
|
1238
|
-
end_dt = _date_value(p.end) if p.end is not None else datetime.max
|
|
1239
|
-
return (start_dt, end_dt, p)
|
|
1240
|
-
|
|
1241
|
-
materialized = [_materialize(p) for p in partitionings]
|
|
1242
|
-
comparable_specs = [m for m in materialized if m is not None]
|
|
1243
|
-
|
|
1244
|
-
# Sort by start datetime so consecutive comparison works
|
|
1245
|
-
comparable_specs.sort(key=lambda tup: tup[0])
|
|
1246
|
-
|
|
1247
|
-
previous_end: Optional[datetime] = None
|
|
1248
|
-
for start_dt, end_dt, spec in comparable_specs:
|
|
1249
|
-
if previous_end is not None:
|
|
1250
|
-
if start_dt <= previous_end:
|
|
1251
|
-
raise ValueError(
|
|
1252
|
-
"Partitioning specifications overlap or touch: '{}' - '{}' overlaps with a previous range.".format(
|
|
1253
|
-
spec.start or "-infinity", spec.end or "+infinity"
|
|
1254
|
-
)
|
|
1255
|
-
)
|
|
1256
|
-
previous_end = end_dt
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
def _is_date_literal(expr: Optional[str]) -> bool:
|
|
1260
|
-
"""Return True if the expression is a simple YYYY-MM-DD literal."""
|
|
1261
|
-
if expr is None:
|
|
1262
|
-
return False
|
|
1263
|
-
return bool(re.match(DATE_LITERAL_REGEX, expr.strip()))
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
def _is_datetime_literal(expr: Optional[str]) -> bool:
|
|
1267
|
-
"""Return True if the expression is a YYYY-MM-DD HH:MM:SS literal."""
|
|
1268
|
-
if expr is None:
|
|
1269
|
-
return False
|
|
1270
|
-
return bool(re.match(DATETIME_LITERAL_REGEX, expr.strip()))
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
def _is_timestamp_date_literal(expr: Optional[str]) -> bool:
|
|
1274
|
-
"""Return True if the expression is a TIMESTAMP('YYYY-MM-DD') literal."""
|
|
1275
|
-
if expr is None:
|
|
1276
|
-
return False
|
|
1277
|
-
return bool(re.match(TIMESTAMP_DATE_LITERAL_REGEX, expr.strip()))
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
def _is_timestamp_datetime_literal(expr: Optional[str]) -> bool:
|
|
1281
|
-
"""Return True if the expression is a TIMESTAMP('YYYY-MM-DD HH:MM:SS') literal."""
|
|
1282
|
-
if expr is None:
|
|
1283
|
-
return False
|
|
1284
|
-
return bool(re.match(TIMESTAMP_DATETIME_LITERAL_REGEX, expr.strip()))
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
def _is_date_or_datetime_literal(expr: Optional[str]) -> bool:
|
|
1288
|
-
"""Return True if the expression is either a date or datetime literal."""
|
|
1289
|
-
return (
|
|
1290
|
-
_is_date_literal(expr)
|
|
1291
|
-
or _is_datetime_literal(expr)
|
|
1292
|
-
or _is_timestamp_date_literal(expr)
|
|
1293
|
-
or _is_timestamp_datetime_literal(expr)
|
|
1294
|
-
)
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
def _date_value(expr: str) -> datetime:
|
|
1298
|
-
"""Convert a YYYY-MM-DD literal into a datetime object (at midnight)."""
|
|
1299
|
-
return datetime.strptime(expr.strip(), "%Y-%m-%d")
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
def _datetime_value(expr: str) -> datetime:
|
|
1303
|
-
"""Convert a YYYY-MM-DD HH:MM:SS literal into a datetime object."""
|
|
1304
|
-
return datetime.strptime(expr.strip(), "%Y-%m-%d %H:%M:%S")
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
def _timestamp_literal_value(expr: str) -> datetime:
|
|
1308
|
-
"""Extract datetime from TIMESTAMP('date') or TIMESTAMP('datetime') literal."""
|
|
1309
|
-
if _is_timestamp_date_literal(expr):
|
|
1310
|
-
# Extract date from TIMESTAMP('2024-01-01') format
|
|
1311
|
-
date_match = re.search(r"'(\d{4}-\d{2}-\d{2})'", expr)
|
|
1312
|
-
if date_match:
|
|
1313
|
-
date_str = date_match.group(1)
|
|
1314
|
-
return datetime.strptime(date_str, "%Y-%m-%d")
|
|
1315
|
-
elif _is_timestamp_datetime_literal(expr):
|
|
1316
|
-
# Extract datetime from TIMESTAMP('2024-01-01 12:30:45') format
|
|
1317
|
-
datetime_match = re.search(r"'(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})'", expr)
|
|
1318
|
-
if datetime_match:
|
|
1319
|
-
datetime_str = datetime_match.group(1)
|
|
1320
|
-
return datetime.strptime(datetime_str, "%Y-%m-%d %H:%M:%S")
|
|
1321
|
-
|
|
1322
|
-
raise ValueError(f"Invalid TIMESTAMP literal: {expr}")
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
def _date_or_datetime_value(expr: str) -> datetime:
|
|
1326
|
-
"""Convert a date or datetime literal into a datetime object."""
|
|
1327
|
-
if _is_date_literal(expr):
|
|
1328
|
-
return _date_value(expr)
|
|
1329
|
-
if _is_datetime_literal(expr):
|
|
1330
|
-
return _datetime_value(expr)
|
|
1331
|
-
if _is_timestamp_date_literal(expr) or _is_timestamp_datetime_literal(expr):
|
|
1332
|
-
return _timestamp_literal_value(expr)
|
|
1333
|
-
raise ValueError(f"Expression is not a date or datetime literal: {expr}")
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
# Required external keys for a time-based partitioning spec. Internal helper
|
|
1337
|
-
# attributes like `inclusive_start` are intentionally excluded.
|
|
1338
|
-
TIME_BASED_REQUIRED_KEYS: Set[str] = {"field", "start", "end", "interval"}
|
|
1339
|
-
|
|
1340
|
-
# ------------------------------------------------------------------
|
|
1341
|
-
# Utilities for working with lists of partition specs
|
|
1342
|
-
# ------------------------------------------------------------------
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
def combine_partitions(parts: List["TimeBasedPartitioning"]) -> List[ColumnElement]:
|
|
1346
|
-
"""Return a combined list of SQLAlchemy expressions for an list
|
|
1347
|
-
of `TimeBasedPartitioning` objects, ensuring adjacent specs do not
|
|
1348
|
-
overlap at the boundary row.
|
|
1349
|
-
|
|
1350
|
-
If two consecutive specs share a boundary (`prev.end == curr.start`) the
|
|
1351
|
-
current spec's first slice is made exclusive by toggling its internal
|
|
1352
|
-
`inclusive_start` flag.
|
|
1353
|
-
"""
|
|
1354
|
-
|
|
1355
|
-
combined: List[ColumnElement] = []
|
|
1356
|
-
|
|
1357
|
-
validate_partitioning_list(parts)
|
|
1358
|
-
|
|
1359
|
-
for idx, spec in enumerate(parts):
|
|
1360
|
-
p = spec.model_copy(deep=True) # avoid mutating caller's object
|
|
1361
|
-
|
|
1362
|
-
if idx > 0 and p.start is not None and parts[idx - 1].end is not None:
|
|
1363
|
-
if p.start == parts[idx - 1].end:
|
|
1364
|
-
p.inclusive_start = False
|
|
1365
|
-
|
|
1366
|
-
combined.extend(p.generate_expressions())
|
|
1367
|
-
|
|
1368
|
-
return combined
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
__all__ = [
|
|
1372
|
-
"TIME_BASED_REQUIRED_KEYS",
|
|
1373
|
-
"TimeBasedPartitioning",
|
|
1374
|
-
"combine_partitions",
|
|
1375
|
-
"validate_partitioning_list",
|
|
1376
|
-
]
|