fluidattacks_dynamo_etl 4.0.5__tar.gz

fluidattacks_dynamo_etl-4.0.5/.envrc

@@ -0,0 +1 @@
+use_flake ".#python311.devShell"

fluidattacks_dynamo_etl-4.0.5/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.4
+Name: fluidattacks_dynamo_etl
+Version: 4.0.5
+Summary: Dynamo ETLs configuration
+Author-email: Product Team <development@fluidattacks.com>
+Requires-Python: >=3.11
+Requires-Dist: boto3 >=1.34.131, <2.0.0
+Requires-Dist: click >=8.1.3, <9.0.0
+Requires-Dist: python-dateutil >=2.9.0.post0, <3.0.0
+Requires-Dist: fa-purity >=2.6.0, <3.0.0
+Requires-Dist: fa_singer_io >=3.0.0, <4.0.0
+Requires-Dist: redshift-client >=10.0.0, <11.0.0
+Requires-Dist: fluidattacks-connection-manager>=4.0.0, <5.0.0
+Requires-Dist: fluidattacks-batch-client>=0.2.2, <1.0.0
+Requires-Dist: fluidattacks-etl-utils >=4.1.0, <5.0.0
+Requires-Dist: fluidattacks-target-warehouse >=3.0.0, <4.0.0
+Requires-Dist: fluidattacks-utils-logger >=3.0.0, <4.0.0
+Requires-Dist: snowflake-client >=5.0.0, <6.0.0

fluidattacks_dynamo_etl-4.0.5/build/default.nix

@@ -0,0 +1,49 @@
+{
+  nixpkgs,
+  builders,
+  scripts,
+  src,
+  system,
+  job_inputs,
+}:
+let
+  build_additionals =
+    bundle:
+    let
+      bin = nixpkgs.writeShellApplication {
+        name = "dynamo-etl";
+        runtimeInputs = [
+          bundle.env.runtime
+          nixpkgs.sops
+          nixpkgs.openssh
+        ];
+        runtimeEnv = {
+          AWS_DEFAULT_REGION = "us-east-1";
+        };
+        text = ''
+          dynamo-etl "''${@}"
+        '';
+      };
+      image = import ./image.nix { inherit nixpkgs bin; };
+      jobs = import ./jobs {
+        inherit
+          nixpkgs
+          job_inputs
+          system
+          bundle
+          image
+          ;
+      };
+    in
+    {
+      inherit bin image;
+      job = bundle.job // jobs;
+    };
+in
+{
+  inherit src;
+  root_path = "observes/etl/dynamo";
+  module_name = "fluidattacks_dynamo_etl";
+  pypi_token_var = "DYNAMO_ETL_TOKEN";
+  override = bundle: bundle // (build_additionals bundle);
+}

fluidattacks_dynamo_etl-4.0.5/build/filter.nix

@@ -0,0 +1,12 @@
+path_filter: src:
+path_filter {
+  root = src;
+  include = [
+    "fluidattacks_dynamo_etl"
+    "tests"
+    "pyproject.toml"
+    "mypy.ini"
+    "ruff.toml"
+    "uv.lock"
+  ];
+}

fluidattacks_dynamo_etl-4.0.5/build/image.nix

@@ -0,0 +1,50 @@
+{ nixpkgs, bin }:
+let
+  tempDir = nixpkgs.runCommand "temp-dir" { } ''
+    mkdir -p $out/tmp
+  '';
+  fluidSetup = nixpkgs.runCommand "fluid-public-ssh" { } ''
+    mkdir -p $out/root/.ssh
+    touch $out/root/.ssh/id_ed25519
+    echo 'gitlab.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf' > $out/root/.ssh/known_hosts
+  '';
+  rootHome = nixpkgs.runCommand "root-home" { } ''
+    mkdir -p $out/etc
+    echo 'root:x:0:0:root user:/root:/bin/sh' > $out/etc/passwd
+    echo 'nobody:x:65534:65534:nobody:/var/empty:/bin/sh' >> $out/etc/passwd
+  '';
+  with_universe_ssh =
+    shell_app:
+    nixpkgs.writeShellApplication {
+      inherit (shell_app) name;
+      runtimeInputs = [ nixpkgs.coreutils ];
+      text = ''
+        echo "$UNIVERSE_SSH_KEY" > /root/.ssh/id_ed25519
+        chmod 1777 /tmp
+        chmod 400 /root/.ssh
+        chmod 400 /root/.ssh/id_ed25519
+        chmod 400 /root/.ssh/known_hosts
+        ${shell_app}/bin/${shell_app.name} "''${@}"
+      '';
+      # UNIVERSE_SSH_KEY should be impure and not set at build time because is a secret
+    };
+  # [NOTICE] to maintain the program agnostic of ssh config as on the build/dev env,
+  # ssh setup must be done on the container derivation
+in
+nixpkgs.dockerTools.buildImage {
+  name = "dynamo_etl";
+  copyToRoot = [
+    nixpkgs.dockerTools.caCertificates
+    nixpkgs.dockerTools.fakeNss
+    (with_universe_ssh bin)
+    tempDir
+    fluidSetup
+    rootHome
+  ];
+  # tempDir: required by mktemp
+  # caCertificates: provably required by requests and internet-communication libraries
+  # fakeNss: provably required by ssh, git or other cli tools
+  config = {
+    Env = [ "PATH=/bin" ];
+  };
+}

fluidattacks_dynamo_etl-4.0.5/build/jobs/default.nix

@@ -0,0 +1,69 @@
+{
+  bundle,
+  image,
+  nixpkgs,
+  job_inputs,
+  system,
+}:
+let
+  determine_integrates_schema =
+    CUE_SCHEMA_DIR:
+    let
+      det-schema = nixpkgs.writeShellApplication {
+        runtimeEnv = { inherit CUE_SCHEMA_DIR; };
+        name = "det-schema";
+        runtimeInputs = [
+          nixpkgs.cue
+          bundle.env.runtime
+        ];
+        text = builtins.readFile ./integrates_schema.sh;
+      };
+    in
+    derivation {
+      inherit system;
+      name = "observes-dynamo-etl-det-integrates-schema";
+      builder = "${nixpkgs.bash}/bin/bash";
+      args = [ "${det-schema}/bin/det-schema" ];
+    };
+in
+{
+  determine_integrates_schema = determine_integrates_schema job_inputs.pinned_integrates_cue_schema;
+  update_integrates_schema = nixpkgs.writeShellApplication {
+    runtimeEnv = {
+      AWS_DEFAULT_REGION = "us-east-1";
+      INTEGRATES_VMS_SCHEMAS = determine_integrates_schema job_inputs.self_integrates_cue_schema;
+    };
+    name = "observes-dynamo-etl-update-integrates-schema";
+    runtimeInputs = [ bundle.env.runtime ];
+    text = "python ${./update_integrates.py}";
+  };
+  update_sifts_schema = nixpkgs.writeShellApplication {
+    runtimeEnv = {
+      AWS_DEFAULT_REGION = "us-east-1";
+      SIFTS_SCHEMAS = ./sifts_state.schema.json;
+    };
+    name = "observes-dynamo-etl-update-sifts-schema";
+    runtimeInputs = [ bundle.env.runtime ];
+    text = "python ${./update_sifts.py}";
+  };
+  deploy_image = nixpkgs.writeShellApplication {
+    name = "observes-dynamo-etl-deploy-image";
+    extraShellCheckFlags = [ "-x" ];
+    runtimeInputs = [
+      nixpkgs.skopeo
+      job_inputs.shell-helpers.helper-aws
+      job_inputs.shell-helpers.helper-sops
+      job_inputs.ecr_deploy_image.env.runtime
+    ];
+    text = ''
+      # shellcheck source=${job_inputs.shell-helpers.helper-aws}/bin/helper-aws
+      source helper-aws aws_login prod_common 3600
+
+      deploy-image deploy \
+        --registry "205810638802.dkr.ecr.us-east-1.amazonaws.com/observes/dynamo_etl" \
+        --region "us-east-1" \
+        --image-tar "${image}" \
+        --version "${bundle.pkg.version}"
+    '';
+  };
+}

fluidattacks_dynamo_etl-4.0.5/build/jobs/integrates_schema.sh

@@ -0,0 +1,8 @@
+# shellcheck shell=bash
+
+BUILDER_PWD="${PWD}"
+cd "${CUE_SCHEMA_DIR}" || exit
+cue eval -O ./tables/integrates_vms/full_schema.cue --out "openapi" > "${BUILDER_PWD}/integrates_schema.openapi.json"
+# the `out` env var comes from the nix builder environment
+# shellcheck disable=SC2154
+dynamo-etl schemas integrates-vms "${BUILDER_PWD}/integrates_schema.openapi.json" > "${out}"

fluidattacks_dynamo_etl-4.0.5/build/jobs/sifts_state.schema.json

@@ -0,0 +1,2 @@
+{"type": "SCHEMA", "stream": "sifts_state", "schema": {"properties": {"code_hash_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}, "version_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}, "prediction_label_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}, "prediction_score_float": {"type": ["null", "number"], "size": "float"}, "created_at_datetime": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic", "format": "date-time"}, "category_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}, "subcategory_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}, "pk_str": {"type": ["string", "null"], "precision": 1000, "metatype": "dynamic"}, "sk_str": {"type": ["string", "null"], "precision": 1000, "metatype": "dynamic"}, "group_name_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}, "commit_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}, "path_str": {"type": ["string", "null"], "precision": 1000, "metatype": "dynamic"}, "start_byte_int": {"type": ["integer", "null"], "size": "normal"}, "end_byte_int": {"type": ["integer", "null"], "size": "normal"}, "start_line_int": {"type": ["integer", "null"], "size": "normal"}, "start_column_int": {"type": ["integer", "null"], "size": "normal"}, "end_line_int": {"type": ["integer", "null"], "size": "normal"}, "end_column_int": {"type": ["integer", "null"], "size": "normal"}, "node_type_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}, "model_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}, "root_nickname_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}, "file_path_str": {"type": ["string", "null"], "precision": 1000, "metatype": "dynamic"}, "snippet_hash_id_str": {"type": ["string", "null"], "precision": 1000, "metatype": "dynamic"}, "analyzed_at_datetime": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic", "format": "date-time"}, "candidate_index_int": {"type": ["integer", "null"], "size": "normal"}, "cost_float": {"type": ["null", "number"], "size": "float"}, "input_tokens_int": {"type": ["integer", "null"], "size": "normal"}, "output_tokens_int": {"type": ["integer", "null"], "size": "normal"}, "total_tokens_int": {"type": ["integer", "null"], "size": "normal"}, "vulnerable_bool": {"type": "boolean"}, "ranking_score_float": {"type": ["null", "number"], "size": "float"}, "reason_str": {"type": ["string", "null"], "precision": 16777216, "metatype": "dynamic"}, "sifts_state____vulnerable_lines_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}, "suggested_criteria_code_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}, "suggested_finding_title_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}, "trace_id_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}, "vulnerability_id_candidate_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}}}, "key_properties": []}
+{"type": "SCHEMA", "stream": "sifts_state____vulnerable_lines_str", "schema": {"properties": {"val_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}, "sid0_str": {"type": ["string", "null"], "precision": 256, "metatype": "dynamic"}, "forward_index_int": {"type": ["integer", "null"], "size": "normal"}, "backward_index_int": {"type": ["integer", "null"], "size": "normal"}}}, "key_properties": []}

fluidattacks_dynamo_etl-4.0.5/build/jobs/update_integrates.py

@@ -0,0 +1,13 @@
+# noqa: INP001
+# this is a script not intented to be a pkg
+from os import environ
+from pathlib import Path
+
+from fa_purity import Cmd, UnitType, Unsafe
+
+from fluidattacks_dynamo_etl.etls.integrates.schema import update_schema_cache
+
+cmd: Cmd[UnitType] = update_schema_cache(Path(environ["INTEGRATES_VMS_SCHEMAS"])).map(
+    lambda r: r.alt(Unsafe.raise_exception).to_union(),
+)
+cmd.compute()

fluidattacks_dynamo_etl-4.0.5/build/jobs/update_sifts.py

@@ -0,0 +1,13 @@
+# noqa: INP001
+# this is a script not intented to be a pkg
+from os import environ
+from pathlib import Path
+
+from fa_purity import Cmd, UnitType, Unsafe
+
+from fluidattacks_dynamo_etl.etls.sifts import update_schema_cache
+
+cmd: Cmd[UnitType] = update_schema_cache(Path(environ["SIFTS_SCHEMAS"])).map(
+    lambda r: r.alt(Unsafe.raise_exception).to_union(),
+)
+cmd.compute()