cosmotech-acceleration-library 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

cosmotech/csm_data/commands/store/dump_to_s3.py

@@ -22,7 +22,7 @@ VALID_TYPES = (
 @click.option(
     "--store-folder",
     envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
-    help=T("csm-data.commands.store.dump_to_s3.parameters.store_folder"),
+    help=T("csm_data.commands.store.dump_to_s3.parameters.store_folder"),
     metavar="PATH",
     type=str,
     show_envvar=True,
@@ -31,13 +31,13 @@ VALID_TYPES = (
 @click.option(
     "--output-type",
     default="sqlite",
-    help=T("csm-data.commands.store.dump_to_s3.parameters.output_type"),
+    help=T("csm_data.commands.store.dump_to_s3.parameters.output_type"),
     type=click.Choice(VALID_TYPES, case_sensitive=False),
 )
 @click.option(
     "--bucket-name",
     envvar="CSM_DATA_BUCKET_NAME",
-    help=T("csm-data.commands.store.dump_to_s3.parameters.bucket_name"),
+    help=T("csm_data.commands.store.dump_to_s3.parameters.bucket_name"),
     metavar="BUCKET",
     type=str,
     show_envvar=True,
@@ -47,7 +47,7 @@ VALID_TYPES = (
     "--prefix",
     "file_prefix",
     envvar="CSM_DATA_BUCKET_PREFIX",
-    help=T("csm-data.commands.store.dump_to_s3.parameters.prefix"),
+    help=T("csm_data.commands.store.dump_to_s3.parameters.prefix"),
     metavar="PREFIX",
     type=str,
     show_envvar=True,
@@ -56,14 +56,14 @@ VALID_TYPES = (
 @click.option(
     "--use-ssl/--no-ssl",
     default=True,
-    help=T("csm-data.commands.store.dump_to_s3.parameters.use_ssl"),
+    help=T("csm_data.commands.store.dump_to_s3.parameters.use_ssl"),
     type=bool,
     is_flag=True,
 )
 @click.option(
     "--s3-url",
     "endpoint_url",
-    help=T("csm-data.commands.store.dump_to_s3.parameters.s3_url"),
+    help=T("csm_data.commands.store.dump_to_s3.parameters.s3_url"),
     type=str,
     required=True,
     show_envvar=True,
@@ -73,7 +73,7 @@ VALID_TYPES = (
 @click.option(
     "--access-id",
     "access_id",
-    help=T("csm-data.commands.store.dump_to_s3.parameters.access_id"),
+    help=T("csm_data.commands.store.dump_to_s3.parameters.access_id"),
     type=str,
     required=True,
     show_envvar=True,
@@ -83,7 +83,7 @@ VALID_TYPES = (
 @click.option(
     "--secret-key",
     "secret_key",
-    help=T("csm-data.commands.store.dump_to_s3.parameters.secret_key"),
+    help=T("csm_data.commands.store.dump_to_s3.parameters.secret_key"),
     type=str,
     required=True,
     show_envvar=True,
@@ -92,14 +92,14 @@ VALID_TYPES = (
 )
 @click.option(
     "--ssl-cert-bundle",
-    help=T("csm-data.commands.store.dump_to_s3.parameters.ssl_cert_bundle"),
+    help=T("csm_data.commands.store.dump_to_s3.parameters.ssl_cert_bundle"),
     type=str,
     show_envvar=True,
     metavar="PATH",
     envvar="CSM_S3_CA_BUNDLE",
 )
 @web_help("csm-data/store/dump-to-s3")
-@translate_help("csm-data.commands.store.dump_to_s3.description")
+@translate_help("csm_data.commands.store.dump_to_s3.description")
 def dump_to_s3(
     store_folder,
     bucket_name: str,

cosmotech/csm_data/commands/store/list_tables.py

@@ -12,11 +12,11 @@ from cosmotech.orchestrator.utils.translate import T
 
 @click.command()
 @web_help("csm-data/store/list-tables")
-@translate_help("csm-data.commands.store.list_tables.description")
+@translate_help("csm_data.commands.store.list_tables.description")
 @click.option(
     "--store-folder",
     envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
-    help=T("csm-data.commands.store.list_tables.parameters.store_folder"),
+    help=T("csm_data.commands.store.list_tables.parameters.store_folder"),
     metavar="PATH",
     type=str,
     show_envvar=True,
@@ -24,7 +24,7 @@ from cosmotech.orchestrator.utils.translate import T
 )
 @click.option(
     "--schema/--no-schema",
-    help=T("csm-data.commands.store.list_tables.parameters.schema"),
+    help=T("csm_data.commands.store.list_tables.parameters.schema"),
     is_flag=True,
     type=bool,
     default=False,

cosmotech/csm_data/commands/store/load_csv_folder.py

@@ -12,11 +12,11 @@ from cosmotech.orchestrator.utils.translate import T
 
 @click.command()
 @web_help("csm-data/store/load-csv-folder")
-@translate_help("csm-data.commands.store.load_csv_folder.description")
+@translate_help("csm_data.commands.store.load_csv_folder.description")
 @click.option(
     "--store-folder",
     envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
-    help=T("csm-data.commands.store.load_csv_folder.parameters.store_folder"),
+    help=T("csm_data.commands.store.load_csv_folder.parameters.store_folder"),
     metavar="PATH",
     type=str,
     show_envvar=True,
@@ -25,7 +25,7 @@ from cosmotech.orchestrator.utils.translate import T
 @click.option(
     "--csv-folder",
     envvar="CSM_DATASET_ABSOLUTE_PATH",
-    help=T("csm-data.commands.store.load_csv_folder.parameters.csv_folder"),
+    help=T("csm_data.commands.store.load_csv_folder.parameters.csv_folder"),
     metavar="PATH",
     type=str,
     show_envvar=True,

cosmotech/csm_data/commands/store/load_from_singlestore.py

@@ -12,12 +12,12 @@ from cosmotech.orchestrator.utils.translate import T
 
 @click.command()
 @web_help("csm-data/store/load-from-singlestore")
-@translate_help("csm-data.commands.store.load_from_singlestore.description")
+@translate_help("csm_data.commands.store.load_from_singlestore.description")
 @click.option(
     "--singlestore-host",
     "single_store_host",
     envvar="SINGLE_STORE_HOST",
-    help=T("csm-data.commands.store.load_from_singlestore.parameters.singlestore_host"),
+    help=T("csm_data.commands.store.load_from_singlestore.parameters.singlestore_host"),
     type=str,
     show_envvar=True,
     required=True,
@@ -25,7 +25,7 @@ from cosmotech.orchestrator.utils.translate import T
 @click.option(
     "--singlestore-port",
     "single_store_port",
-    help=T("csm-data.commands.store.load_from_singlestore.parameters.singlestore_port"),
+    help=T("csm_data.commands.store.load_from_singlestore.parameters.singlestore_port"),
     envvar="SINGLE_STORE_PORT",
     show_envvar=True,
     required=False,
@@ -34,7 +34,7 @@ from cosmotech.orchestrator.utils.translate import T
 @click.option(
     "--singlestore-db",
     "single_store_db",
-    help=T("csm-data.commands.store.load_from_singlestore.parameters.singlestore_db"),
+    help=T("csm_data.commands.store.load_from_singlestore.parameters.singlestore_db"),
     envvar="SINGLE_STORE_DB",
     show_envvar=True,
     required=True,
@@ -42,7 +42,7 @@ from cosmotech.orchestrator.utils.translate import T
 @click.option(
     "--singlestore-user",
     "single_store_user",
-    help=T("csm-data.commands.store.load_from_singlestore.parameters.singlestore_user"),
+    help=T("csm_data.commands.store.load_from_singlestore.parameters.singlestore_user"),
     envvar="SINGLE_STORE_USERNAME",
     show_envvar=True,
     required=True,
@@ -50,7 +50,7 @@ from cosmotech.orchestrator.utils.translate import T
 @click.option(
     "--singlestore-password",
     "single_store_password",
-    help=T("csm-data.commands.store.load_from_singlestore.parameters.singlestore_password"),
+    help=T("csm_data.commands.store.load_from_singlestore.parameters.singlestore_password"),
     envvar="SINGLE_STORE_PASSWORD",
     show_envvar=True,
     required=True,
@@ -58,7 +58,7 @@ from cosmotech.orchestrator.utils.translate import T
 @click.option(
     "--singlestore-tables",
     "single_store_tables",
-    help=T("csm-data.commands.store.load_from_singlestore.parameters.singlestore_tables"),
+    help=T("csm_data.commands.store.load_from_singlestore.parameters.singlestore_tables"),
     envvar="SINGLE_STORE_TABLES",
     show_envvar=True,
     required=True,
@@ -67,7 +67,7 @@ from cosmotech.orchestrator.utils.translate import T
     "--store-folder",
     "store_folder",
     envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
-    help=T("csm-data.commands.store.load_from_singlestore.parameters.store_folder"),
+    help=T("csm_data.commands.store.load_from_singlestore.parameters.store_folder"),
     metavar="PATH",
     type=str,
     show_envvar=True,

cosmotech/csm_data/commands/store/reset.py

@@ -12,11 +12,11 @@ from cosmotech.orchestrator.utils.translate import T
 
 @click.command()
 @web_help("csm-data/store/reset")
-@translate_help("csm-data.commands.store.reset.description")
+@translate_help("csm_data.commands.store.reset.description")
 @click.option(
     "--store-folder",
     envvar="CSM_PARAMETERS_ABSOLUTE_PATH",
-    help=T("csm-data.commands.store.reset.parameters.store_folder"),
+    help=T("csm_data.commands.store.reset.parameters.store_folder"),
     metavar="PATH",
     type=str,
     show_envvar=True,

cosmotech/csm_data/commands/store/store.py

@@ -22,7 +22,7 @@ from cosmotech.orchestrator.utils.translate import T
 
 @click.group()
 @web_help("csm-data/store")
-@translate_help("csm-data.commands.store.description")
+@translate_help("csm_data.commands.store.store.description")
 def store():
     pass
 

cosmotech/csm_data/main.py

@@ -24,11 +24,12 @@ from cosmotech.orchestrator.utils.translate import T
 def print_version(ctx, param, value):
     if not value or ctx.resilient_parsing:
         return
-    click.echo(f"Cosmo Tech Data Interface {__version__}")
+    click.echo(T("csm_data.commons.version.message").format(version=__version__))
     ctx.exit()
 
 
-@click.group("csm-data")
+@click.group("csm-data", invoke_without_command=True)
+@click.pass_context
 @click_log.simple_verbosity_option(LOGGER, "--log-level", envvar="LOG_LEVEL", show_envvar=True)
 @click.option(
     "--version",
@@ -36,12 +37,13 @@ def print_version(ctx, param, value):
     callback=print_version,
     expose_value=False,
     is_eager=True,
-    help="Print version number and return.",
+    help=T("csm_data.commands.main.parameters.version"),
 )
 @web_help("csm-data")
-@translate_help("csm-data.commands.main.description")
-def main():
-    pass
+@translate_help("csm_data.commands.main.description")
+def main(ctx):
+    if ctx.invoked_subcommand is None:
+        click.echo(T("csm_data.commands.main.content"))
 
 
 main.add_command(api, "api")

cosmotech/csm_data/utils/decorators.py

@@ -59,7 +59,7 @@ def web_help(effective_target="", base_url=WEB_DOCUMENTATION_ROOT):
         @click.option(
             "--web-help",
             is_flag=True,
-            help="Open the web documentation",
+            help=T("csm_data.commons.decorators.parameters.web_help").format(url=documentation_url),
             is_eager=True,
             callback=open_documentation,
         )

cosmotech/translation/csm_data/en-US/csm_data/commands/api/api.yml

@@ -0,0 +1,8 @@
+description: |
+  Cosmo Tech API helper command
+
+  This command will inform you of which connection is available to use for the Cosmo Tech API
+
+  If no connection is available, will list all possible set of parameters and return an error code,
+
+  You can use this command in a csm-orc template to make sure that API connection is available.

cosmotech/translation/csm_data/en-US/csm_data/commands/api/postgres_send_runner_metadata.yml

@@ -0,0 +1,17 @@
+description: |
+  Send runner metadata to a PostgreSQL database.
+
+  Creates or updates a table in PostgreSQL with runner metadata including id, name, last run id, and run template id.
+  The table will be created if it doesn't exist, and existing records will be updated based on the runner id.
+parameters:
+  organization_id: An organization id for the Cosmo Tech API
+  workspace_id: A workspace id for the Cosmo Tech API
+  runner_id: A runner id for the Cosmo Tech API
+  table_prefix: Prefix to add to the table name
+  postgres_host: PostgreSQL host URI
+  postgres_port: PostgreSQL database port
+  postgres_db: PostgreSQL database name
+  postgres_schema: PostgreSQL schema name
+  postgres_user: PostgreSQL connection user name
+  postgres_password: PostgreSQL connection password
+  encode_password: Force encoding of password to percent encoding

cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_load_csv.yml

@@ -0,0 +1,13 @@
+description: |
+  Load data from a runner's RDS database into a CSV file.
+
+  Executes a SQL query against the runner's RDS database and saves the results to a CSV file.
+  By default, it will list all tables in the public schema if no specific query is provided.
+parameters:
+  target_folder: The folder where the csv will be written
+  organization_id: An organization id for the Cosmo Tech API
+  workspace_id: A workspace id for the Cosmo Tech API
+  runner_id: A runner id for the Cosmo Tech API
+  run_id: A run id for the Cosmo Tech API
+  file_name: A file name to write the query results
+  query: SQL query to execute (defaults to listing all tables in public schema)

cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_csv.yml

@@ -0,0 +1,12 @@
+description: |
+  Send CSV files to a runner's RDS database.
+
+  Takes all CSV files from a source folder and sends their content to the runner's RDS database.
+  Each CSV file will be sent to a table named after the file (without the .csv extension).
+  The table name will be prefixed with "CD_" in the database.
+parameters:
+  source_folder: The folder containing csvs to send
+  organization_id: An organization id for the Cosmo Tech API
+  workspace_id: A workspace id for the Cosmo Tech API
+  runner_id: A runner id for the Cosmo Tech API
+  run_id: A run id for the Cosmo Tech API

cosmotech/translation/csm_data/en-US/csm_data/commands/api/rds_send_store.yml

@@ -0,0 +1,12 @@
+description: |
+  Send data from a store to a runner's RDS database.
+
+  Takes all tables from a store and sends their content to the runner's RDS database.
+  Each table will be sent to a table with the same name, prefixed with "CD_" in the database.
+  Null values in rows will be removed before sending.
+parameters:
+  store_folder: The folder containing the store files
+  organization_id: An organization id for the Cosmo Tech API
+  workspace_id: A workspace id for the Cosmo Tech API
+  runner_id: A runner id for the Cosmo Tech API
+  run_id: A run id for the Cosmo Tech API

cosmotech/translation/csm_data/en-US/csm_data/commands/api/run_load_data.yml

@@ -0,0 +1,15 @@
+description: |
+  Download a runner data from the Cosmo Tech API
+  Requires a valid Azure connection either with:
+  - The AZ cli command: az login
+  - A triplet of env var AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET
+parameters:
+  organization_id: The id of an organization in the cosmotech api
+  workspace_id: The id of a workspace in the cosmotech api
+  runner_id: The id of a runner in the cosmotech api
+  parameters_absolute_path: A local folder to store the parameters content
+  dataset_absolute_path: A local folder to store the main dataset content
+  write_json: Whether to write the data in JSON format
+  write_csv: Whether to write the data in CSV format
+  fetch_dataset: Whether to fetch datasets
+  parallel: Whether to fetch datasets in parallel

cosmotech/translation/csm_data/en-US/csm_data/commands/api/runtemplate_load_handler.yml

@@ -0,0 +1,7 @@
+description: |
+  Uses environment variables to download cloud based Template steps
+parameters:
+  organization_id: The id of an organization in the cosmotech api
+  workspace_id: The id of a solution in the cosmotech api
+  run_template_id: The name of the run template in the cosmotech api
+  handler_list: A list of handlers to download (comma separated)

cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_load_files.yml

@@ -0,0 +1,14 @@
+description: |
+  Query a twingraph and loads all the data from it
+
+  Will create 1 csv file per node type / relationship type
+
+  The twingraph must have been populated using the "tdl-send-files" command for this to work correctly
+
+  Requires a valid connection to the API to send the data
+parameters:
+  organization_id: An organization id for the Cosmo Tech API
+  workspace_id: A workspace id for the Cosmo Tech API
+  runner_id: A runner id for the Cosmo Tech API
+  scenario_id: A scenario id for the Cosmo Tech API
+  dir: Path to the directory to write the results to

cosmotech/translation/csm_data/en-US/csm_data/commands/api/tdl_send_files.yml

@@ -0,0 +1,18 @@
+description: |
+  Reads a folder CSVs and send those to the Cosmo Tech API as a Dataset
+
+  CSVs must follow a given format:
+    - Nodes files must have an id column
+    - Relationship files must have id, src and dest columns
+
+  Non-existing relationship (aka dest or src does not point to existing node) won't trigger an error,
+  the relationship will not be created instead.
+
+  Requires a valid connection to the API to send the data
+parameters:
+  api_url: The URI to a Cosmo Tech API instance
+  organization_id: An organization id for the Cosmo Tech API
+  workspace_id: A workspace id for the Cosmo Tech API
+  runner_id: A runner id for the Cosmo Tech API
+  dir: Path to the directory containing csvs to send
+  clear: Flag to clear the target dataset first (if set to True will clear the dataset before sending anything, irreversibly)

cosmotech/translation/csm_data/en-US/csm_data/commands/api/wsf_load_file.yml

@@ -0,0 +1,10 @@
+description: |
+  Download files from a workspace.
+
+  Downloads files from a specified path in a workspace to a local target folder.
+  If the workspace path ends with '/', it will be treated as a folder and all files within will be downloaded.
+parameters:
+  organization_id: An organization id for the Cosmo Tech API
+  workspace_id: A workspace id for the Cosmo Tech API
+  workspace_path: Path inside the workspace to load (end with '/' for a folder)
+  target_folder: Folder in which to send the downloaded file

cosmotech/translation/csm_data/en-US/csm_data/commands/api/wsf_send_file.yml

@@ -0,0 +1,12 @@
+description: |
+  Upload a file to a workspace.
+
+  Uploads a local file to a specified path in a workspace.
+  If the workspace path ends with '/', the file will be uploaded to that folder with its original name.
+  Otherwise, the file will be uploaded with the name specified in the workspace path.
+parameters:
+  organization_id: An organization id for the Cosmo Tech API
+  workspace_id: A workspace id for the Cosmo Tech API
+  file_path: Path to the file to send as a workspace file
+  workspace_path: Path inside the workspace to store the file (end with '/' for a folder)
+  overwrite: Flag to overwrite the target file if it exists

cosmotech/translation/csm_data/en-US/csm_data/commands/main.yml

@@ -0,0 +1,9 @@
+content: |
+  Welcome to the Cosmo Tech Data Interface command line interface (CLI) tool.
+  This tool provides a set of commands to help you manage your data connections and configurations.
+description: |
+  Cosmo Tech Data Interface
+
+  Command toolkit providing quick implementation of data connections to use inside the Cosmo Tech Platform
+parameters:
+  version: Print version number and return.

cosmotech/translation/csm_data/en-US/csm_data/commands/storage/adx_send_data.yml

@@ -0,0 +1,8 @@
+description: "Send data to ADX"
+parameters:
+  adx_uri: The ADX cluster path (URI info can be found into ADX cluster page)
+  adx_ingest_uri: The ADX cluster ingest path (URI info can be found into ADX cluster page)
+  database_name: The targeted database name
+  waiting_ingestion: Toggle waiting for the ingestion results
+  adx_tag: The ADX tag to use for the ingestion
+  store_folder: The folder containing the datastore containing the data to send

cosmotech/translation/csm_data/en-US/csm_data/commands/storage/adx_send_runnerdata.yml

@@ -0,0 +1,15 @@
+description: |
+  Uses environment variables to send content of CSV files to ADX
+  Requires a valid Azure connection either with:
+  - The AZ cli command: az login
+  - A triplet of env var AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_CLIENT_SECRET
+parameters:
+  dataset_absolute_path: A local folder to store the main dataset content
+  parameters_absolute_path: A local folder to store the parameters content
+  runner_id: the Runner Id to add to records
+  adx_uri: the ADX cluster path (URI info can be found into ADX cluster page)
+  adx_ingest_uri: The ADX cluster ingest path (URI info can be found into ADX cluster page)
+  database_name: The targeted database name
+  send_parameters: whether or not to send parameters (parameters path is mandatory then)
+  send_datasets: whether or not to send datasets (parameters path is mandatory then)
+  wait: Toggle waiting for the ingestion results

cosmotech/translation/csm_data/en-US/csm_data/commands/storage/az_storage_upload.yml

@@ -0,0 +1,8 @@
+description: |
+  Upload a folder to an Azure Storage Blob
+parameters:
+  source_folder: The folder/file to upload to the target blob storage
+  recursive: Recursively send the content of every folder inside the starting folder to the blob storage
+  blob_name: The blob name in the Azure Storage service to upload to
+  prefix: A prefix by which all uploaded files should start with in the blob storage
+  az_storage_sas_url: SAS url allowing access to the AZ storage container

cosmotech/translation/csm_data/en-US/csm_data/commands/storage/s3_bucket_delete.yml

@@ -0,0 +1,17 @@
+description: |
+  Delete S3 bucket content to a given folder
+
+  Will delete everything in the bucket unless a prefix is set, then only file following the given prefix will be deleted
+
+  Make use of the boto3 library to access the bucket
+
+  More information is available on this page:
+  [https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html]
+parameters:
+  bucket_name: The bucket on S3 to delete
+  prefix_filter: A prefix by which all deleted files should start in the bucket
+  use_ssl: Use SSL to secure connection to S3
+  s3_url: URL to connect to the S3 system
+  access_id: Identity used to connect to the S3 system
+  secret_key: Secret tied to the ID used to connect to the S3 system
+  ssl_cert_bundle: Path to an alternate CA Bundle to validate SSL connections

cosmotech/translation/csm_data/en-US/csm_data/commands/storage/s3_bucket_download.yml

@@ -0,0 +1,18 @@
+description: |
+  Download S3 bucket content to a given folder
+
+  Will download everything in the bucket unless a prefix is set, then only file following the given prefix will be downloaded
+
+  Make use of the boto3 library to access the bucket
+
+  More information is available on this page:
+  [https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html]
+parameters:
+  target_folder: The folder in which to download the bucket content
+  bucket_name: The bucket on S3 to download
+  prefix_filter: A prefix by which all downloaded files should start in the bucket
+  use_ssl: Use SSL to secure connection to S3
+  s3_url: URL to connect to the S3 system
+  access_id: Identity used to connect to the S3 system
+  secret_key: Secret tied to the ID used to connect to the S3 system
+  ssl_cert_bundle: Path to an alternate CA Bundle to validate SSL connections

cosmotech/translation/csm_data/en-US/csm_data/commands/storage/s3_bucket_upload.yml

@@ -0,0 +1,21 @@
+description: |
+  Upload a folder to a S3 Bucket
+
+  Will upload everything from a given folder to a S3 bucket. If a single file is passed only it will be uploaded, and recursive will be ignored
+
+  Giving a prefix will add it to every upload (finishing the prefix with a "/" will allow to upload in a folder inside the bucket)
+
+  Make use of the boto3 library to access the bucket
+
+  More information is available on this page:
+  [https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html]
+parameters:
+  source_folder: The folder/file to upload to the target bucket
+  recursive: Recursively send the content of every folder inside the starting folder to the bucket
+  bucket_name: The bucket on S3 to upload to
+  prefix: A prefix by which all uploaded files should start with in the bucket
+  use_ssl: Use SSL to secure connection to S3
+  s3_url: URL to connect to the S3 system
+  access_id: Identity used to connect to the S3 system
+  secret_key: Secret tied to the ID used to connect to the S3 system
+  ssl_cert_bundle: Path to an alternate CA Bundle to validate SSL connections

cosmotech/translation/csm_data/en-US/csm_data/commands/storage/storage.yml

@@ -0,0 +1,4 @@
+description: |
+  Storage operations command group
+
+  This group of commands provides functionality for interacting with various storage systems

cosmotech/translation/csm_data/en-US/csm_data/commands/store/dump_to_azure.yml

@@ -0,0 +1,23 @@
+description: |
+  Dump a datastore to a Azure storage account.
+
+  Will upload everything from a given data store to a Azure storage container.
+
+  3 modes currently exists:
+    - sqlite: will dump the data store underlying database as is
+    - csv: will convert every table of the datastore to csv and send them as separate files
+    - parquet: will convert every table of the datastore to parquet and send them as separate files
+
+  Make use of the azure.storage.blob library to access the container
+
+  More information is available on this page:
+  [https://learn.microsoft.com/en-us/azure/storage/blobs/storage-quickstart-blobs-python?tabs=managed-identity%2Croles-azure-portal%2Csign-in-azure-cli&pivots=blob-storage-quickstart-scratch]
+parameters:
+  store_folder: The folder containing the store files
+  output_type: Choose the type of file output to use (sqlite, csv, parquet)
+  account_name: The account name on Azure to upload to
+  container_name: The container name on Azure to upload to
+  prefix: A prefix by which all uploaded files should start with in the container
+  tenant_id: Tenant Identity used to connect to Azure storage system
+  client_id: Client Identity used to connect to Azure storage system
+  client_secret: Client Secret tied to the ID used to connect to Azure storage system

cosmotech/translation/csm_data/en-US/csm_data/commands/store/dump_to_postgresql.yml

@@ -0,0 +1,20 @@
+description: |
+  Running this command will dump your store to a given postgresql database
+
+  Tables names from the store will be prepended with table-prefix in target database
+
+  The postgresql user must have USAGE granted on the schema for this script to work due to the use of the command COPY FROM STDIN
+
+  You can simply give him that grant by running the command:
+  GRANT USAGE ON SCHEMA <schema> TO <username>
+parameters:
+  store_folder: The folder containing the store files
+  table_prefix: Prefix to add to the table name
+  postgres_host: PostgreSQL host URI
+  postgres_port: PostgreSQL database port
+  postgres_db: PostgreSQL database name
+  postgres_schema: PostgreSQL schema name
+  postgres_user: PostgreSQL connection user name
+  postgres_password: PostgreSQL connection password
+  replace: Append data on existing tables
+  encode_password: Force encoding of password to percent encoding

cosmotech/translation/csm_data/en-US/csm_data/commands/store/dump_to_s3.yml

@@ -0,0 +1,26 @@
+description: |
+  Dump a datastore to a S3
+
+  Will upload everything from a given data store to a S3 bucket.
+
+  3 modes currently exists:
+    - sqlite: will dump the data store underlying database as is
+    - csv: will convert every table of the datastore to csv and send them as separate files
+    - parquet: will convert every table of the datastore to parquet and send them as separate files
+
+  Giving a prefix will add it to every upload (finishing the prefix with a "/" will allow to upload in a folder inside the bucket)
+
+  Make use of the boto3 library to access the bucket
+
+  More information is available on this page:
+  [https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html]
+parameters:
+  store_folder: The folder containing the store files
+  output_type: Choose the type of file output to use (sqlite, csv, parquet)
+  bucket_name: The bucket on S3 to upload to
+  prefix: A prefix by which all uploaded files should start with in the bucket
+  use_ssl: Use SSL to secure connection to S3
+  s3_url: URL to connect to the S3 system
+  access_id: Identity used to connect to the S3 system
+  secret_key: Secret tied to the ID used to connect to the S3 system
+  ssl_cert_bundle: Path to an alternate CA Bundle to validate SSL connections

cosmotech/translation/csm_data/en-US/csm_data/commands/store/list_tables.yml

@@ -0,0 +1,5 @@
+description: |
+  Running this command will list the existing tables in your datastore
+parameters:
+  store_folder: The folder containing the store files
+  schema: Display the schema of the tables

cosmotech/translation/csm_data/en-US/csm_data/commands/store/load_csv_folder.yml

@@ -0,0 +1,5 @@
+description: |
+  Running this command will find all csvs in the given folder and put them in the store
+parameters:
+  store_folder: The folder containing the store files
+  csv_folder: The folder containing the csv files to store

cosmotech/translation/csm_data/en-US/csm_data/commands/store/load_from_singlestore.yml

@@ -0,0 +1,16 @@
+description: |
+  Load data from SingleStore tables into the store.
+  Will download everything from a given SingleStore database following some configuration into the store.
+
+  Make use of the singlestoredb to access to SingleStore
+
+  More information is available on this page:
+  [https://docs.singlestore.com/cloud/developer-resources/connect-with-application-development-tools/connect-with-python/connect-using-the-singlestore-python-client/]
+parameters:
+  singlestore_host: SingleStore instance URI
+  singlestore_port: SingleStore port
+  singlestore_db: SingleStore database name
+  singlestore_user: SingleStore connection user name
+  singlestore_password: SingleStore connection password
+  singlestore_tables: SingleStore table names to fetched (separated by comma)
+  store_folder: The folder containing the store files