fluidattacks_connection_manager 1.0.0__tar.gz

fluidattacks_connection_manager-1.0.0/.envrc

@@ -0,0 +1,2 @@
+ln -f ../mypy.ini ./mypy.ini
+use_flake ".#python311.devShell"

fluidattacks_connection_manager-1.0.0/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.4
+Name: fluidattacks_connection_manager
+Version: 1.0.0
+Summary: Common connection utils
+Author-email: Product Team <development@fluidattacks.com>
+Requires-Python: >=3.11
+Classifier: License :: OSI Approved :: MIT License
+Requires-Dist: fa-purity >=2.1.0, <3.0.0
+Requires-Dist: fluidattacks-etl-utils >=1.0.0, <2.0.0
+Requires-Dist: redshift-client >=8.0.1, <9.0.0
+Requires-Dist: snowflake-client >=3.0.2, <4.0.0

fluidattacks_connection_manager-1.0.0/build/default.nix

@@ -0,0 +1,16 @@
+{ nixpkgs, pynix, src, local_lib }:
+let
+  deps = import ./deps { inherit nixpkgs pynix local_lib; };
+  requirements = python_pkgs: {
+    runtime_deps = with python_pkgs; [ etl-utils fa-purity snowflake-client ];
+    build_deps = with python_pkgs; [ flit-core ];
+    test_deps = with python_pkgs; [ arch-lint mypy pytest pytest-cov ruff ];
+  };
+in {
+  inherit src requirements;
+  root_path = "observes/common/connection-manager";
+  module_name = "fluidattacks_connection_manager";
+  pypi_token_var = "CONNECTION_MANAGER_TOKEN";
+  defaultDeps = deps.python_pkgs;
+  override = b: b;
+}

fluidattacks_connection_manager-1.0.0/build/deps/arch_lint.nix

@@ -0,0 +1,17 @@
+{ nixpkgs, pynix, python_pkgs }:
+let
+  commit = "43aa33ef4cfa12c37148f4653dd89233c585c368"; # v4.0.4
+  sha256 = "06r2xfj50rn9m87i3vwa9bilhnrz3njhmfd992vzp4a5x937rfq2";
+  bundle = let
+    src = builtins.fetchTarball {
+      inherit sha256;
+      url =
+        "https://gitlab.com/dmurciaatfluid/arch_lint/-/archive/${commit}/arch_lint-${commit}.tar";
+    };
+  in import "${src}/build" {
+    inherit src;
+    inherit nixpkgs pynix;
+  };
+  extended_python_pkgs = python_pkgs // { inherit (bundle.deps) grimp; };
+in bundle.builders.pkgBuilder
+(bundle.builders.requirements extended_python_pkgs)

fluidattacks_connection_manager-1.0.0/build/deps/default.nix

@@ -0,0 +1,32 @@
+{ nixpkgs, pynix, local_lib }:
+let
+  python_overlay = final: prev: {
+    arch-lint = import ./arch_lint.nix {
+      inherit nixpkgs pynix;
+      python_pkgs = prev;
+    };
+    fa-purity = import ./fa_purity.nix {
+      inherit nixpkgs pynix;
+      python_pkgs = final;
+    };
+    etl-utils = local_lib.etl_utils_bundle.builders.pkgBuilder
+      (local_lib.etl_utils_bundle.builders.requirements {
+        inherit nixpkgs;
+        python_pkgs = final;
+      });
+    s3transfer = prev.s3transfer.overridePythonAttrs (oldAttrs: {
+      preCheck = nixpkgs.lib.optionalString nixpkgs.stdenv.isDarwin ''
+        export TMPDIR="/tmp"
+      '';
+    });
+    snowflake-client = let
+      bundle =
+        import ./snowflake_client.nix { inherit nixpkgs pynix python_pkgs; };
+    in bundle.pkg;
+  };
+  python = pynix.lib.python.override {
+    packageOverrides = python_overlay;
+    self = python;
+  };
+  python_pkgs = python.pkgs;
+in { inherit python_pkgs; }

fluidattacks_connection_manager-1.0.0/build/deps/fa_purity.nix

@@ -0,0 +1,19 @@
+{ nixpkgs, pynix, python_pkgs }:
+let
+  commit = "e669114103b334e532aec34dc5b2ec06dfd24247"; # v2.5.0+1
+  sha256 = "1x0ipsklpsxyrzy8j6agkqppj4n2x67kaz7zsi3mkycan9z9z3f8";
+  bundle = let
+    src = builtins.fetchTarball {
+      inherit sha256;
+      url =
+        "https://gitlab.com/dmurciaatfluid/purity/-/archive/${commit}/purity-${commit}.tar";
+    };
+  in import "${src}/build" {
+    inherit src;
+    inherit nixpkgs pynix;
+  };
+  extended_python_pkgs = python_pkgs // {
+    inherit (bundle.deps) types-simplejson;
+  };
+in bundle.builders.pkgBuilder
+(bundle.builders.requirements extended_python_pkgs)

fluidattacks_connection_manager-1.0.0/build/deps/snowflake_client.nix

@@ -0,0 +1,40 @@
+{ nixpkgs, pynix, python_pkgs }:
+let
+  commit = "b516409c11a25af8acf933dee055f9784c994b2f"; # v3.2.3
+  sha256 = "0jv5h7wbcn437l3rf41qc7sdg31q900lhn2pcjix00s181psmihq";
+  bundle = let
+    src = builtins.fetchTarball {
+      inherit sha256;
+      url =
+        "https://gitlab.com/dmurciaatfluid/snowflake_client/-/archive/${commit}/snowflake_client-${commit}.tar";
+    };
+  in import "${src}/build" {
+    inherit src;
+    inherit nixpkgs pynix;
+  };
+  extended_python_pkgs = python_pkgs // {
+    inherit (bundle.deps) redshift-client s3transfer boto3;
+    snowflake-connector-python = let
+      pkg = bundle.deps.snowflake-connector-python.overridePythonAttrs (old: {
+        version = "3.14.0";
+        src = nixpkgs.fetchFromGitHub {
+          owner = "snowflakedb";
+          repo = "snowflake-connector-python";
+          tag = "v3.14.0";
+          hash = "sha256-r3g+eVVyK9t5qpAGvimapuWilAh3eHJEFUw8VBwtKw8=";
+        };
+        disabledTestPaths = old.disabledTestPaths ++ [
+          "test/unit/test_wiremock_client.py"
+          "test/unit/test_put_get.py"
+        ];
+      });
+    in pynix.utils.compose [
+      pynix.patch.homelessPatch
+      pynix.patch.disableChecks
+    ] pkg;
+  };
+in {
+  inherit extended_python_pkgs;
+  pkg = bundle.builders.pkgBuilder
+    (bundle.builders.requirements extended_python_pkgs);
+}

fluidattacks_connection_manager-1.0.0/build/filter.nix

@@ -0,0 +1,11 @@
+path_filter: src:
+path_filter {
+  root = src;
+  include = [
+    "fluidattacks_connection_manager"
+    "tests"
+    "pyproject.toml"
+    "mypy.ini"
+    "ruff.toml"
+  ];
+}

fluidattacks_connection_manager-1.0.0/flake.lock

@@ -0,0 +1,196 @@
+{
+  "nodes": {
+    "etl_utils": {
+      "inputs": {
+        "observes_flake_builder": [
+          "observes_flake_builder"
+        ]
+      },
+      "locked": {
+        "dir": "observes/common/etl-utils",
+        "lastModified": 1760615285,
+        "narHash": "sha256-Es+4RZ/ZvNz/xEbhMlWoUrebj7YvXysTVfwY20p0nKs=",
+        "owner": "fluidattacks",
+        "repo": "universe",
+        "rev": "d5147093dbd95ca7740d40a542650b9e2573f941",
+        "type": "github"
+      },
+      "original": {
+        "dir": "observes/common/etl-utils",
+        "owner": "fluidattacks",
+        "repo": "universe",
+        "rev": "d5147093dbd95ca7740d40a542650b9e2573f941",
+        "type": "github"
+      }
+    },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": "nixpkgs-lib"
+      },
+      "locked": {
+        "lastModified": 1754487366,
+        "narHash": "sha256-pHYj8gUBapuUzKV/kN/tR3Zvqc7o6gdFB9XKXIp1SQ8=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "af66ad14b28a127c5c0f3bbb298218fc63528a18",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "nix_filter": {
+      "locked": {
+        "lastModified": 1731533336,
+        "narHash": "sha256-oRam5PS1vcrr5UPgALW0eo1m/5/pls27Z/pabHNy2Ms=",
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "rev": "f7653272fd234696ae94229839a99b73c9ab7de0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "type": "github"
+      }
+    },
+    "nix_filter_2": {
+      "locked": {
+        "lastModified": 1731533336,
+        "narHash": "sha256-oRam5PS1vcrr5UPgALW0eo1m/5/pls27Z/pabHNy2Ms=",
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "rev": "f7653272fd234696ae94229839a99b73c9ab7de0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1736441877,
+        "narHash": "sha256-m3+PhBFkDwqo9lBplG4AyMW8P4/KcioJRS1UG8N7okM=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "ce3899414dab3297cf025bfa356dc2da426feefd",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-lib": {
+      "locked": {
+        "lastModified": 1753579242,
+        "narHash": "sha256-zvaMGVn14/Zz8hnp4VWT9xVnhc8vuL3TStRqwk22biA=",
+        "owner": "nix-community",
+        "repo": "nixpkgs.lib",
+        "rev": "0f36c44e01a6129be94e3ade315a5883f0228a6e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nixpkgs.lib",
+        "type": "github"
+      }
+    },
+    "nixpkgs_flake": {
+      "locked": {
+        "lastModified": 1756313514,
+        "narHash": "sha256-3Xbak0pXR8ziNv1ghHyJ5a5Ti2kt/LWq6hKZVJTvjBs=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "181464235b2daff8725773fef43ffc9d6b02e1c2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "observes_flake_builder": {
+      "inputs": {
+        "nix_filter": "nix_filter",
+        "nixpkgs_flake": "nixpkgs_flake",
+        "pynix_flake": "pynix_flake",
+        "shell-helpers": "shell-helpers"
+      },
+      "locked": {
+        "dir": "observes/common/std_flake",
+        "lastModified": 1760615285,
+        "narHash": "sha256-Es+4RZ/ZvNz/xEbhMlWoUrebj7YvXysTVfwY20p0nKs=",
+        "owner": "fluidattacks",
+        "repo": "universe",
+        "rev": "d5147093dbd95ca7740d40a542650b9e2573f941",
+        "type": "github"
+      },
+      "original": {
+        "dir": "observes/common/std_flake",
+        "owner": "fluidattacks",
+        "repo": "universe",
+        "rev": "d5147093dbd95ca7740d40a542650b9e2573f941",
+        "type": "github"
+      }
+    },
+    "pynix_flake": {
+      "inputs": {
+        "nix_filter": "nix_filter_2",
+        "nixpkgs": "nixpkgs"
+      },
+      "locked": {
+        "lastModified": 1758642502,
+        "narHash": "sha256-PD/bQMz2dqZSkydHyvRh+jS/0qoV8SdcIVs6sassteQ=",
+        "owner": "dmurciaatfluid",
+        "repo": "python_nix_builder",
+        "rev": "809aafe2e1995e72c15378d099bf251b78f04a20",
+        "type": "gitlab"
+      },
+      "original": {
+        "owner": "dmurciaatfluid",
+        "repo": "python_nix_builder",
+        "type": "gitlab"
+      }
+    },
+    "root": {
+      "inputs": {
+        "etl_utils": "etl_utils",
+        "observes_flake_builder": "observes_flake_builder"
+      }
+    },
+    "shell-helpers": {
+      "inputs": {
+        "flake-parts": "flake-parts",
+        "nixpkgs": [
+          "observes_flake_builder",
+          "nixpkgs_flake"
+        ]
+      },
+      "locked": {
+        "dir": "common/utils/shell-helpers",
+        "lastModified": 1752896460,
+        "narHash": "sha256-AsyTatXMx839cGsF6knwTrDZHk3Eue2QD3eSP7bkmpg=",
+        "owner": "fluidattacks",
+        "repo": "universe",
+        "rev": "27749d2c3a2b018eb010a322e5e1352f993c9e86",
+        "type": "github"
+      },
+      "original": {
+        "dir": "common/utils/shell-helpers",
+        "owner": "fluidattacks",
+        "repo": "universe",
+        "rev": "27749d2c3a2b018eb010a322e5e1352f993c9e86",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

fluidattacks_connection_manager-1.0.0/flake.nix

@@ -0,0 +1,28 @@
+{
+  description = "Utils for snowflake connection";
+
+  inputs = {
+    observes_flake_builder = {
+      url =
+        "github:fluidattacks/universe/d5147093dbd95ca7740d40a542650b9e2573f941?shallow=1&dir=observes/common/std_flake";
+    };
+    etl_utils = {
+      url =
+        "github:fluidattacks/universe/d5147093dbd95ca7740d40a542650b9e2573f941?shallow=1&dir=observes/common/etl-utils";
+      inputs.observes_flake_builder.follows = "observes_flake_builder";
+    };
+  };
+
+  outputs = { self, ... }@inputs:
+    let
+      build_args = { system, python_version, nixpkgs, pynix }:
+        let
+          with_python = bundle: bundle.packages."${system}"."${python_version}";
+        in import ./build {
+          inherit nixpkgs pynix;
+          src = import ./build/filter.nix nixpkgs.nix-filter self;
+          local_lib = { etl_utils_bundle = with_python inputs.etl_utils; };
+        };
+    in { packages = inputs.observes_flake_builder.outputs.build build_args; };
+}
+

fluidattacks_connection_manager-1.0.0/fluidattacks_connection_manager/__init__.py

@@ -0,0 +1,31 @@
+from ._common import (
+    ClientAdapter,
+    CommonSchemaClient,
+    CommonTableClient,
+)
+from ._connection import (
+    ConnectionConf,
+    ConnectionManager,
+    ConnectionManagerFactory,
+    Databases,
+    DbClients,
+    Roles,
+    SetupException,
+    Warehouses,
+)
+
+__version__ = "1.0.0"
+
+__all__ = [
+    "ClientAdapter",
+    "CommonSchemaClient",
+    "CommonTableClient",
+    "ConnectionConf",
+    "ConnectionManager",
+    "ConnectionManagerFactory",
+    "Databases",
+    "DbClients",
+    "Roles",
+    "SetupException",
+    "Warehouses",
+]

fluidattacks_connection_manager-1.0.0/fluidattacks_connection_manager/_common.py

@@ -0,0 +1,178 @@
+from dataclasses import (
+    dataclass,
+)
+
+from fa_purity import (
+    Cmd,
+    FrozenDict,
+    Maybe,
+    PureIter,
+    ResultE,
+)
+from fluidattacks_etl_utils.typing import (
+    Callable,
+    FrozenSet,
+)
+from redshift_client.client import (
+    GroupedRows,
+)
+from redshift_client.client import (
+    SchemaClient as RedshiftSchemaClient,
+)
+from redshift_client.client import (
+    TableClient as RedshiftTableClient,
+)
+from redshift_client.core.column import (
+    Column,
+    ColumnObj,
+)
+from redshift_client.core.id_objs import (
+    ColumnId,
+    DbTableId,
+    SchemaId,
+    TableId,
+)
+from redshift_client.core.table import (
+    DistStyle,
+    SortKeys,
+    Table,
+    TableAttrs,
+)
+from redshift_client.sql_client import (
+    Limit,
+    RowData,
+)
+from snowflake_client import (
+    SchemaClient,
+    TableClient,
+)
+
+BluePrint = DbTableId
+NewTable = DbTableId
+Source = DbTableId
+Target = DbTableId
+
+
+@dataclass(frozen=True, kw_only=True)
+class CommonTableClient:
+    """Table client interface. See factory method documentation for further details."""
+
+    get: Callable[[DbTableId], Cmd[ResultE[Table]]]
+    exist: Callable[[DbTableId], Cmd[ResultE[bool]]]
+    insert: Callable[[DbTableId, Table, PureIter[RowData], Limit], Cmd[ResultE[None]]]
+    named_insert: Callable[[DbTableId, GroupedRows], Cmd[ResultE[None]]]
+    rename: Callable[[DbTableId, str], Cmd[ResultE[TableId]]]
+    delete: Callable[[DbTableId], Cmd[ResultE[None]]]
+    delete_cascade: Callable[[DbTableId], Cmd[ResultE[None]]]
+    add_column: Callable[[DbTableId, ColumnObj], Cmd[ResultE[None]]]
+    add_columns: Callable[[DbTableId, FrozenDict[ColumnId, Column]], Cmd[ResultE[None]]]
+    new: Callable[[DbTableId, Table], Cmd[ResultE[None]]]
+    new_if_not_exist: Callable[[DbTableId, Table], Cmd[ResultE[None]]]
+    create_like: Callable[[BluePrint, NewTable], Cmd[ResultE[None]]]
+    move_data: Callable[[Source, Target], Cmd[ResultE[None]]]
+    move: Callable[[Source, Target], Cmd[ResultE[None]]]
+    migrate: Callable[[Source, Target], Cmd[ResultE[None]]]
+
+
+@dataclass(frozen=True, kw_only=True)
+class CommonSchemaClient:
+    all_schemas: Cmd[ResultE[FrozenSet[SchemaId]]]
+    table_ids: Callable[[SchemaId], Cmd[ResultE[FrozenSet[DbTableId]]]]
+    exist: Callable[[SchemaId], Cmd[ResultE[bool]]]
+    delete: Callable[[SchemaId], Cmd[ResultE[None]]]
+    delete_cascade: Callable[[SchemaId], Cmd[ResultE[None]]]
+    rename: Callable[[SchemaId, SchemaId], Cmd[ResultE[None]]]
+    create: Callable[[SchemaId], Cmd[ResultE[None]]]
+    create_if_not_exist: Callable[[SchemaId], Cmd[ResultE[None]]]
+    recreate: Callable[[SchemaId], Cmd[ResultE[None]]]
+    recreate_cascade: Callable[[SchemaId], Cmd[ResultE[None]]]
+    migrate: Callable[[SchemaId, SchemaId], Cmd[ResultE[None]]]
+    move: Callable[[SchemaId, SchemaId], Cmd[ResultE[None]]]
+
+
+@dataclass(frozen=True)
+class ClientAdapter:
+    @staticmethod
+    def snowflake_table_client_adapter(
+        client: TableClient,
+    ) -> CommonTableClient:
+        return CommonTableClient(
+            get=client.get,
+            exist=client.exist,
+            insert=client.insert,
+            named_insert=client.named_insert,
+            rename=client.rename,
+            delete=client.delete,
+            delete_cascade=client.delete_cascade,
+            add_column=client.add_column,
+            add_columns=client.add_columns,
+            new=client.new,
+            new_if_not_exist=client.new_if_not_exist,
+            create_like=client.create_like,
+            move_data=client.move_data,
+            move=client.move,
+            migrate=client.migrate,
+        )
+
+    @staticmethod
+    def redshift_table_client_adapter(
+        client: RedshiftTableClient,
+    ) -> CommonTableClient:
+        return CommonTableClient(
+            get=client.get,
+            exist=client.exist,
+            insert=client.insert,
+            named_insert=client.named_insert,
+            rename=client.rename,
+            delete=client.delete,
+            delete_cascade=client.delete_cascade,
+            add_column=client.add_column,
+            add_columns=client.add_columns,
+            new=lambda i, t: client.new(
+                i,
+                t,
+                TableAttrs(DistStyle.AUTO, Maybe.empty(), True, SortKeys.auto()),
+            ),
+            new_if_not_exist=lambda i, t: client.new_if_not_exist(
+                i,
+                t,
+                TableAttrs(DistStyle.AUTO, Maybe.empty(), True, SortKeys.auto()),
+            ),
+            create_like=client.create_like,
+            move_data=client.move_data,
+            move=client.move,
+            migrate=client.migrate,
+        )
+
+    @staticmethod
+    def _schema_client_adapter(
+        client: SchemaClient | RedshiftSchemaClient,
+    ) -> CommonSchemaClient:
+        return CommonSchemaClient(
+            all_schemas=client.all_schemas,
+            table_ids=client.table_ids,
+            exist=client.exist,
+            delete=client.delete,
+            delete_cascade=client.delete_cascade,
+            rename=client.rename,
+            create=client.create,
+            create_if_not_exist=client.create_if_not_exist,
+            recreate=client.recreate,
+            recreate_cascade=client.recreate_cascade,
+            migrate=client.migrate,
+            move=client.move,
+        )
+
+    @classmethod
+    def snowflake_schema_client_adapter(
+        cls,
+        client: SchemaClient,
+    ) -> CommonSchemaClient:
+        return cls._schema_client_adapter(client)
+
+    @classmethod
+    def redshift_schema_client_adapter(
+        cls,
+        client: RedshiftSchemaClient,
+    ) -> CommonSchemaClient:
+        return cls._schema_client_adapter(client)