fluidattacks_target_warehouse 1.0.0__tar.gz

fluidattacks_target_warehouse-1.0.0/.envrc

@@ -0,0 +1,2 @@
+ln -f ../../common/mypy.ini ./mypy.ini
+use_flake ".#python311.devShell"

fluidattacks_target_warehouse-1.0.0/PKG-INFO

@@ -0,0 +1,17 @@
+Metadata-Version: 2.4
+Name: fluidattacks_target_warehouse
+Version: 1.0.0
+Summary: Singer target for fluidattacks warehouse
+Author-email: Product Team <development@fluidattacks.com>
+Requires-Python: >=3.11
+Classifier: License :: OSI Approved :: MIT License
+Requires-Dist: boto3 >=1.17.104, <2.0.0
+Requires-Dist: click >=8.1.3, <9.0.0
+Requires-Dist: fluidattacks-connection-manager >=1.0.0, <2.0.0
+Requires-Dist: fluidattacks-etl-utils >=1.0.0, <2.0.0
+Requires-Dist: fa-purity >=2.1.0, <3.0.0
+Requires-Dist: fa-singer-io >=2.0.2, <4.0.0
+Requires-Dist: mypy-boto3-s3 >=1.28.55, <2.0.0
+Requires-Dist: redshift-client >=8.1.1, <9.0.0
+Requires-Dist: snowflake-client >=3.2.3, <4.0.0
+Requires-Dist: fluidattacks-utils-logger >=1.0.0, <2.0.0

fluidattacks_target_warehouse-1.0.0/build/default.nix

@@ -0,0 +1,18 @@
+{ nixpkgs, builders, scripts, src, }:
+let
+  build_bin = bundle:
+    nixpkgs.writeShellApplication {
+      name = "target-warehouse";
+      runtimeInputs = [ bundle.env.runtime ];
+      text = ''
+        target-warehouse "''${@}"
+      '';
+    };
+in {
+  inherit src;
+  root_path = "observes/singer/target-warehouse";
+  module_name = "fluidattacks_target_warehouse";
+  pypi_token_var = "TARGET_WAREHOUSE_TOKEN";
+  override = bundle: bundle // { bin = build_bin bundle; };
+}
+

fluidattacks_target_warehouse-1.0.0/build/filter.nix

@@ -0,0 +1,12 @@
+path_filter: src:
+path_filter {
+  root = src;
+  include = [
+    "fluidattacks_target_warehouse"
+    "tests"
+    "pyproject.toml"
+    "mypy.ini"
+    "ruff.toml"
+    "uv.lock"
+  ];
+}

fluidattacks_target_warehouse-1.0.0/flake.lock

@@ -0,0 +1,351 @@
+{
+  "nodes": {
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": "nixpkgs-lib"
+      },
+      "locked": {
+        "lastModified": 1754487366,
+        "narHash": "sha256-pHYj8gUBapuUzKV/kN/tR3Zvqc7o6gdFB9XKXIp1SQ8=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "af66ad14b28a127c5c0f3bbb298218fc63528a18",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "nix_filter": {
+      "locked": {
+        "lastModified": 1731533336,
+        "narHash": "sha256-oRam5PS1vcrr5UPgALW0eo1m/5/pls27Z/pabHNy2Ms=",
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "rev": "f7653272fd234696ae94229839a99b73c9ab7de0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "type": "github"
+      }
+    },
+    "nix_filter_2": {
+      "locked": {
+        "lastModified": 1731533336,
+        "narHash": "sha256-oRam5PS1vcrr5UPgALW0eo1m/5/pls27Z/pabHNy2Ms=",
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "rev": "f7653272fd234696ae94229839a99b73c9ab7de0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1736441877,
+        "narHash": "sha256-m3+PhBFkDwqo9lBplG4AyMW8P4/KcioJRS1UG8N7okM=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "ce3899414dab3297cf025bfa356dc2da426feefd",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-lib": {
+      "locked": {
+        "lastModified": 1753579242,
+        "narHash": "sha256-zvaMGVn14/Zz8hnp4VWT9xVnhc8vuL3TStRqwk22biA=",
+        "owner": "nix-community",
+        "repo": "nixpkgs.lib",
+        "rev": "0f36c44e01a6129be94e3ade315a5883f0228a6e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nixpkgs.lib",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1758690382,
+        "narHash": "sha256-NY3kSorgqE5LMm1LqNwGne3ZLMF2/ILgLpFr1fS4X3o=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "e643668fd71b949c53f8626614b21ff71a07379d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_3": {
+      "locked": {
+        "lastModified": 1758690382,
+        "narHash": "sha256-NY3kSorgqE5LMm1LqNwGne3ZLMF2/ILgLpFr1fS4X3o=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "e643668fd71b949c53f8626614b21ff71a07379d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_4": {
+      "locked": {
+        "lastModified": 1761373498,
+        "narHash": "sha256-Q/uhWNvd7V7k1H1ZPMy/vkx3F8C13ZcdrKjO7Jv7v0c=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "6a08e6bb4e46ff7fcbb53d409b253f6bad8a28ce",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_flake": {
+      "locked": {
+        "lastModified": 1756313514,
+        "narHash": "sha256-3Xbak0pXR8ziNv1ghHyJ5a5Ti2kt/LWq6hKZVJTvjBs=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "181464235b2daff8725773fef43ffc9d6b02e1c2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "observes_flake_builder": {
+      "inputs": {
+        "nix_filter": "nix_filter",
+        "nixpkgs_flake": "nixpkgs_flake",
+        "pynix_flake": "pynix_flake",
+        "pyproject-build-systems": "pyproject-build-systems",
+        "pyproject-nix": "pyproject-nix_2",
+        "shell-helpers": "shell-helpers",
+        "uv2nix": "uv2nix_2"
+      },
+      "locked": {
+        "dir": "observes/common/std_flake_2",
+        "lastModified": 1761684858,
+        "narHash": "sha256-gybMRO/MSOPXQjI//u2UKfvs8pD0sACqL2Kjp+xR4rI=",
+        "owner": "fluidattacks",
+        "repo": "universe",
+        "rev": "c5053864bb838edbe226d74a650d901c0270843b",
+        "type": "github"
+      },
+      "original": {
+        "dir": "observes/common/std_flake_2",
+        "owner": "fluidattacks",
+        "repo": "universe",
+        "rev": "c5053864bb838edbe226d74a650d901c0270843b",
+        "type": "github"
+      }
+    },
+    "pynix_flake": {
+      "inputs": {
+        "nix_filter": "nix_filter_2",
+        "nixpkgs": "nixpkgs"
+      },
+      "locked": {
+        "lastModified": 1758642502,
+        "narHash": "sha256-PD/bQMz2dqZSkydHyvRh+jS/0qoV8SdcIVs6sassteQ=",
+        "owner": "dmurciaatfluid",
+        "repo": "python_nix_builder",
+        "rev": "809aafe2e1995e72c15378d099bf251b78f04a20",
+        "type": "gitlab"
+      },
+      "original": {
+        "owner": "dmurciaatfluid",
+        "repo": "python_nix_builder",
+        "type": "gitlab"
+      }
+    },
+    "pyproject-build-systems": {
+      "inputs": {
+        "nixpkgs": "nixpkgs_2",
+        "pyproject-nix": "pyproject-nix",
+        "uv2nix": "uv2nix"
+      },
+      "locked": {
+        "lastModified": 1759113590,
+        "narHash": "sha256-fgxP2RCN4cg0jYiMYoETYc7TZ2JjgyvJa2y9l8oSUFE=",
+        "owner": "pyproject-nix",
+        "repo": "build-system-pkgs",
+        "rev": "dbfc0483b5952c6b86e36f8b3afeb9dde30ea4b5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "pyproject-nix",
+        "repo": "build-system-pkgs",
+        "type": "github"
+      }
+    },
+    "pyproject-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "observes_flake_builder",
+          "pyproject-build-systems",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1758265079,
+        "narHash": "sha256-amLaLNwKSZPShQHzfgmc/9o76dU8xzN0743dWgvYlr8=",
+        "owner": "nix-community",
+        "repo": "pyproject.nix",
+        "rev": "02e9418fd4af638447dca4b17b1280da95527fc9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "pyproject.nix",
+        "type": "github"
+      }
+    },
+    "pyproject-nix_2": {
+      "inputs": {
+        "nixpkgs": "nixpkgs_3"
+      },
+      "locked": {
+        "lastModified": 1760402624,
+        "narHash": "sha256-jF6UKLs2uGc2rtved8Vrt58oTWjTQoAssuYs/0578Z4=",
+        "owner": "pyproject-nix",
+        "repo": "pyproject.nix",
+        "rev": "84c4ea102127c77058ea1ed7be7300261fafc7d2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "pyproject-nix",
+        "repo": "pyproject.nix",
+        "type": "github"
+      }
+    },
+    "pyproject-nix_3": {
+      "inputs": {
+        "nixpkgs": [
+          "observes_flake_builder",
+          "uv2nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1760402624,
+        "narHash": "sha256-jF6UKLs2uGc2rtved8Vrt58oTWjTQoAssuYs/0578Z4=",
+        "owner": "pyproject-nix",
+        "repo": "pyproject.nix",
+        "rev": "84c4ea102127c77058ea1ed7be7300261fafc7d2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "pyproject-nix",
+        "repo": "pyproject.nix",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "observes_flake_builder": "observes_flake_builder"
+      }
+    },
+    "shell-helpers": {
+      "inputs": {
+        "flake-parts": "flake-parts",
+        "nixpkgs": [
+          "observes_flake_builder",
+          "nixpkgs_flake"
+        ]
+      },
+      "locked": {
+        "dir": "common/utils/shell-helpers",
+        "lastModified": 1752896460,
+        "narHash": "sha256-AsyTatXMx839cGsF6knwTrDZHk3Eue2QD3eSP7bkmpg=",
+        "owner": "fluidattacks",
+        "repo": "universe",
+        "rev": "27749d2c3a2b018eb010a322e5e1352f993c9e86",
+        "type": "github"
+      },
+      "original": {
+        "dir": "common/utils/shell-helpers",
+        "owner": "fluidattacks",
+        "repo": "universe",
+        "rev": "27749d2c3a2b018eb010a322e5e1352f993c9e86",
+        "type": "github"
+      }
+    },
+    "uv2nix": {
+      "inputs": {
+        "nixpkgs": [
+          "observes_flake_builder",
+          "pyproject-build-systems",
+          "nixpkgs"
+        ],
+        "pyproject-nix": [
+          "observes_flake_builder",
+          "pyproject-build-systems",
+          "pyproject-nix"
+        ]
+      },
+      "locked": {
+        "lastModified": 1758933732,
+        "narHash": "sha256-HAmm1GBS1myZCFuog0DC2ZLaynvZtiUI2Crmo+cdQI0=",
+        "owner": "pyproject-nix",
+        "repo": "uv2nix",
+        "rev": "273ce18f913d8559e0d04f820d724308966d7c4d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "pyproject-nix",
+        "repo": "uv2nix",
+        "type": "github"
+      }
+    },
+    "uv2nix_2": {
+      "inputs": {
+        "nixpkgs": "nixpkgs_4",
+        "pyproject-nix": "pyproject-nix_3"
+      },
+      "locked": {
+        "lastModified": 1761527626,
+        "narHash": "sha256-neDfvbpFlzUQfH9C+hVRUX0/RXUbJBidw4pFcdMYhZA=",
+        "owner": "pyproject-nix",
+        "repo": "uv2nix",
+        "rev": "46a8e8bbb2d9e34b686329ac16e4a8861394f03a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "pyproject-nix",
+        "repo": "uv2nix",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

fluidattacks_target_warehouse-1.0.0/flake.nix

@@ -0,0 +1,19 @@
+{
+  description = "Singer target for fluidattacks warehouse";
+
+  inputs = {
+    observes_flake_builder = {
+      url =
+        "github:fluidattacks/universe/c5053864bb838edbe226d74a650d901c0270843b?shallow=1&dir=observes/common/std_flake_2";
+    };
+  };
+
+  outputs = { self, ... }@inputs:
+    let
+      build_args = { system, python_version, nixpkgs, builders, scripts }:
+        import ./build {
+          inherit nixpkgs builders scripts;
+          src = import ./build/filter.nix nixpkgs.nix-filter self;
+        };
+    in { packages = inputs.observes_flake_builder.outputs.build build_args; };
+}

fluidattacks_target_warehouse-1.0.0/fluidattacks_target_warehouse/__init__.py

@@ -0,0 +1,11 @@
+from fa_purity import (
+    Unsafe,
+)
+
+from ._logger import (
+    set_logger,
+)
+
+__version__ = "1.0.0"
+
+Unsafe.compute(set_logger(__name__, __version__))

fluidattacks_target_warehouse-1.0.0/fluidattacks_target_warehouse/_cli/__init__.py

@@ -0,0 +1,16 @@
+import click
+
+from . import (
+    _append,
+    _recreate,
+)
+
+
+@click.group()
+def main() -> None:
+    # main cli entrypoint
+    pass
+
+
+main.add_command(_append.only_append)
+main.add_command(_recreate.destroy_and_upload)

fluidattacks_target_warehouse-1.0.0/fluidattacks_target_warehouse/_cli/_append.py

@@ -0,0 +1,132 @@
+import inspect
+from typing import (
+    IO,
+    NoReturn,
+)
+
+import click
+from fa_purity import (
+    Cmd,
+    Maybe,
+)
+from fluidattacks_etl_utils.bug import (
+    Bug,
+)
+from fluidattacks_etl_utils.parallel import (
+    ThreadPool,
+)
+from redshift_client.core.id_objs import (
+    Identifier,
+    SchemaId,
+)
+
+from fluidattacks_target_warehouse._s3 import (
+    S3URI,
+)
+from fluidattacks_target_warehouse.executor import (
+    GenericExecutor,
+)
+from fluidattacks_target_warehouse.loader import (
+    SingerHandlerOptions,
+)
+
+from ._decode import (
+    decode_columns_map,
+)
+
+
+@click.command()
+@click.option(
+    "-s",
+    "--schema-name",
+    type=str,
+    required=True,
+    help="Schema name in your warehouse",
+)
+# -- Optional --
+@click.option(
+    "--records-per-query",
+    type=int,
+    required=False,
+    default=1000,
+    help="Max # of records per sql query",
+)
+@click.option(
+    "--s3-state",
+    type=str,
+    required=False,
+    default=None,
+    help="S3 file obj URI to upload the state; e.g. s3://mybucket/folder/state.json",
+)
+@click.option(
+    "--threads",
+    type=int,
+    required=False,
+    default=1000,
+    help="max number of threads",
+)
+# -- Flags --
+@click.option(
+    "--ignore-failed",
+    type=bool,
+    is_flag=True,
+    help="ignore json items that does not decode to a singer message",
+)
+@click.option(
+    "--truncate",
+    type=bool,
+    is_flag=True,
+    help="Truncate records that exceed column size?",
+)
+@click.option(
+    "--columns-map",
+    type=click.File("r"),
+    help="Custom map from singer properties into ColumnId",
+)
+@click.option(
+    "--use-snowflake",
+    type=bool,
+    is_flag=True,
+    help="Use snowflake implementation",
+)
+def only_append(
+    schema_name: str,
+    records_per_query: int,
+    s3_state: str | None,
+    threads: int,
+    ignore_failed: bool,
+    truncate: bool,
+    columns_map: IO[str] | None,
+    use_snowflake: bool,  # noqa: ARG001
+    # for legacy cli calls
+) -> NoReturn:
+    target = SchemaId(Identifier.new(schema_name))
+    options = SingerHandlerOptions(
+        truncate,
+        records_per_query,
+    )
+    state = (
+        Maybe.from_optional(s3_state)
+        .map(S3URI.from_raw)
+        .map(lambda r: Bug.assume_success("S3URI", inspect.currentframe(), (str(s3_state),), r))
+    )
+    pool = ThreadPool.new(threads)
+    _columns_map = Bug.assume_success(
+        "decode_columns_map",
+        inspect.currentframe(),
+        (),
+        decode_columns_map(columns_map),
+    )
+    executor = pool.map(
+        lambda p: GenericExecutor(
+            target,
+            options,
+            state,
+            ignore_failed,
+            lambda s, t: s.only_append(t, True),
+            p,
+            _columns_map,
+        ),
+    )
+    cmd: Cmd[None] = executor.bind(lambda e: e.execute())
+    cmd.compute()

fluidattacks_target_warehouse-1.0.0/fluidattacks_target_warehouse/_cli/_decode.py

@@ -0,0 +1,43 @@
+from typing import (
+    IO,
+)
+
+from fa_purity import (
+    FrozenDict,
+    Result,
+    ResultE,
+)
+from fa_purity.json import (
+    JsonPrimitiveUnfolder,
+    JsonUnfolder,
+    JsonValueFactory,
+    Unfolder,
+)
+from redshift_client.core.id_objs import (
+    ColumnId,
+    Identifier,
+)
+
+from fluidattacks_target_warehouse.data_schema.duplicates import (
+    SingerToColumnMap,
+)
+
+
+def _decode_columns_map(raw: FrozenDict[str, str]) -> SingerToColumnMap:
+    return SingerToColumnMap(FrozenDict({k: ColumnId(Identifier.new(v)) for k, v in raw.items()}))
+
+
+def decode_columns_map(raw: IO[str] | None) -> ResultE[SingerToColumnMap]:
+    if raw is not None:
+        return (
+            JsonValueFactory.load(raw)
+            .bind(Unfolder.to_json)
+            .bind(
+                lambda j: JsonUnfolder.map_values(
+                    j,
+                    lambda v: Unfolder.to_primitive(v).bind(JsonPrimitiveUnfolder.to_str),
+                ),
+            )
+            .map(_decode_columns_map)
+        )
+    return Result.success(SingerToColumnMap(FrozenDict({})))