recurvedata-lib 0.1.487__py2.py3-none-any.whl

recurvedata/pigeon/connector/hdfs.py

@@ -0,0 +1,204 @@
+import concurrent.futures
+import logging
+import os
+import shutil
+
+from pywebhdfs.webhdfs import PyWebHdfsClient
+
+from recurvedata.pigeon.connector._registry import register_connector_class
+from recurvedata.pigeon.utils import extract_dict, mp
+
+
+@register_connector_class('hdfs')
+class HDFSConnector(object):
+    def __init__(self, host, port, username=None, user_name=None, **kwargs):
+        self.host = host
+        self.port = port
+        self.user_name = username or user_name
+        extra_opts = extract_dict(kwargs, ['path_to_hosts', 'timeout', 'base_uri_pattern', 'request_extra_opts'])
+        self.hdfs = PyWebHdfsClient(host=self.host, port=self.port, user_name=self.user_name, **extra_opts)
+
+    def list_dir(self, path):
+        return self.hdfs.list_dir(path)
+
+    def make_dir(self, path, **kwargs):
+        return self.hdfs.make_dir(path, **kwargs)
+
+    def delete_file(self, path, recursive=False):
+        return self.hdfs.delete_file_dir(path, recursive=recursive)
+
+    def upload_file(self, local_path, hdfs_path=None, overwrite=True):
+        if not hdfs_path:
+            hdfs_path = os.path.basename(local_path)
+
+        if not os.path.dirname(hdfs_path):
+            hdfs_path = os.path.join('/tmp', hdfs_path)
+
+        self.delete_file(hdfs_path)
+
+        with open(local_path, 'rb') as data:
+            self.hdfs.create_file(hdfs_path, data, overwrite=overwrite)
+        return hdfs_path
+
+    def upload_files(self, local_paths, hdfs_folder, num_threads=2):
+        """num_threads is currently not used"""
+        for lf in local_paths:
+            hdfs_filename = os.path.join(hdfs_folder, os.path.basename(lf))
+            self.upload_file(lf, hdfs_filename, overwrite=True)
+            logging.info(f'uploaded {lf} to {hdfs_filename}')
+
+
+class HDFSCliConnector(HDFSConnector):
+    def __init__(self, hdfs_cli=None, **kwargs):
+        if not hdfs_cli:
+            hdfs_cli = shutil.which('hdfs')
+        if not hdfs_cli:
+            raise ValueError('could not locate hdfs command line')
+        self.hdfs_cli = hdfs_cli
+
+    def list_dir(self, path):
+        raise NotImplementedError
+
+    def make_dir(self, path, **kwargs):
+        self._run_cmd(f'-mkdir {path}')
+
+    def delete_file(self, path, recursive=False):
+        self._run_cmd(f'-rm {"-r" if recursive else ""} -f {path}')
+
+    def upload_file(self, local_path, hdfs_path=None, overwrite=True):
+        if not hdfs_path:
+            hdfs_path = os.path.basename(local_path)
+
+        if not os.path.dirname(hdfs_path):
+            hdfs_path = os.path.join('/tmp', hdfs_path)
+        self._run_cmd(f'-put {"-f" if overwrite else ""} {local_path} {hdfs_path}')
+        return hdfs_path
+
+    def upload_files(self, local_paths, hdfs_folder, num_threads=2):
+        local_path_groups = partition_files_equally(local_paths, num_threads)
+        sub_cmds = [f'-put {" ".join(x)} {hdfs_folder}' for x in local_path_groups]
+        pool_size = min(num_threads, len(local_path_groups))
+        with concurrent.futures.ThreadPoolExecutor(max_workers=pool_size) as executor:
+            for _ in executor.map(self._run_cmd, sub_cmds):
+                # exhaust the iterator returned by executor.map
+                # if any task raises an exception, other tasks will be canceled by executor
+                pass
+
+    def _run_cmd(self, sub_cmd):
+        cmd = f'{self.hdfs_cli} dfs {sub_cmd}'
+        logging.info(cmd)
+        output = mp.run_subprocess(cmd, return_output=True, shell=True)
+        if 'NotReplicatedYetException' in output:
+            raise IOError('Incomplete copying from /data/oneflow to /tmp/oneflow/ !')
+        return output
+
+
+def partition_files_equally(local_paths, num_groups: int):
+    groups = _do_partition_files_equally([(f, os.stat(f).st_size) for f in local_paths], num_groups)
+    return [[x[0] for x in g] for g in groups if g]
+
+
+def _do_partition_files_equally(filename_size_pairs, num_groups: int):
+    """把文件划分为若干个总大小相当的分组
+    抄了这个算法: https://cloud.tencent.com/developer/article/1659134，以下文字来自该文章
+
+    这个问题是典型的动态规划的问题，理论上是无法找到最优解的，但是本次只是为了解决实际生产中的问题，而不是要AC，所以我们只需要找到一个相对合理的算法，使得partition的分配相对均衡就好了。
+
+    输入：int数组，分组数divisionNum
+    1. 对数组倒序排序
+    2. 计算数组的平均值 avg
+    3. 遍历数组。
+        * 如果第一个数大于等于avg，将这个数单独作为一组，因为再加下一个数也不会使得求和更接近avg；
+          然后将剩下的数重新求平均，表示需要让剩下的数分配得更加平均，这样可以避免极值的影响，然后重新开始下一轮计算
+        * 如果第一个数num小于avg，我们将这个数加入到数组中，然后我们需要找到一（或若干）个数，使得其和更接近delta = avg-num，
+            - 继续遍历数组，若发现某个数k==delta，将k加入到数组，结束本轮寻找
+            - 若发现a > delta > b；此时要继续判断，如果(delta - b) > (a - delta)，将b加入到数组，delta = delta - b，然后继续遍历；
+              如果(delta - b) < (a - delta)，保存distance = delta - b，然后将a将入到数组中，继续往下遍历，
+              判断能否找到距离 < distance的，如果有则选择距离更小的这组，否则选择将b加入数组。
+
+    :param filename_size_pairs: 文件路径和大小的组合，格式 [(name1, size1), (name2, size2)...]
+    :param num_groups: 分组数量
+    """
+    filename_size_pairs = sorted(filename_size_pairs, key=lambda x: x[1], reverse=True)
+    total_size = sum(x[1] for x in filename_size_pairs)
+    avg = total_size / num_groups
+    groups = []
+    for idx in range(num_groups):
+        if idx == num_groups - 1:
+            # 最后一个分组，把剩余的全部放一起
+            groups.append(filename_size_pairs)
+            break
+
+        if filename_size_pairs and filename_size_pairs[0][1] >= avg:
+            sub_group = [filename_size_pairs[0]]
+            total_size -= filename_size_pairs[0][1]
+            avg = total_size / (num_groups - len(groups))
+        else:
+            sub_group, _ = __get_list(filename_size_pairs, avg, abs(avg))
+        groups.append(sub_group)
+        for item in sub_group:
+            filename_size_pairs.remove(item)
+    return groups
+
+
+def __get_list(filename_size_pairs, delta: float, distance: float):
+    result = []
+    if not filename_size_pairs:
+        return result, -1
+
+    for idx, (filename, size) in enumerate(filename_size_pairs):
+        if delta < size:
+            continue
+        if delta == size:
+            result.append((filename, size))
+            return result, 0
+        else:
+            if idx == 0:
+                result.append((filename, size))
+                delta -= size
+                distance = abs(delta)
+                tmp, d = __get_list(filename_size_pairs[idx + 1:], delta, distance)
+                result.extend(tmp)
+                return result, d
+            else:
+                dis1 = abs(filename_size_pairs[idx - 1][1] - delta)
+                dis2 = abs(delta - size)
+                if dis1 > dis2:
+                    result.append((filename, size))
+                    delta -= size
+                    tmp, d = __get_list(filename_size_pairs[idx + 1:], delta, dis2)
+                    result.extend(tmp)
+                    return result, d
+                else:
+                    tmp, d = __get_list(filename_size_pairs[idx:], delta, dis2)
+                    if dis1 > d:
+                        result.extend(tmp)
+                        return result, d
+                    result.append(filename_size_pairs[idx - 1])
+                    return result, dis1
+
+    dis = abs(delta - filename_size_pairs[-1][1])
+    if dis < distance:
+        return filename_size_pairs[-1:], dis
+    return [], -1
+
+
+if __name__ == '__main__':
+    data = [('233dafd9b1d0b03e6e784987fe748be5.5', 400275118),
+            ('233dafd9b1d0b03e6e784987fe748be5.2', 1147688439),
+            ('233dafd9b1d0b03e6e784987fe748be5.4', 1232810556),
+            ('233dafd9b1d0b03e6e784987fe748be5.3', 1318304652),
+            ('233dafd9b1d0b03e6e784987fe748be5.0', 1392554705),
+            ('233dafd9b1d0b03e6e784987fe748be5.8', 1440314997),
+            ('233dafd9b1d0b03e6e784987fe748be5.7', 1453587946),
+            ('233dafd9b1d0b03e6e784987fe748be5.6', 1470806585),
+            ('233dafd9b1d0b03e6e784987fe748be5.1', 1509157699),
+            ('233dafd9b1d0b03e6e784987fe748be5.9', 1546082238)]
+    groups = _do_partition_files_equally(data, 5)
+    for g in groups:
+        print(g, sum(x[1] for x in g))
+    # [('233dafd9b1d0b03e6e784987fe748be5.9', 1546082238), ('233dafd9b1d0b03e6e784987fe748be5.5', 400275118)] 1946357356
+    # [('233dafd9b1d0b03e6e784987fe748be5.1', 1509157699), ('233dafd9b1d0b03e6e784987fe748be5.2', 1147688439)] 2656846138
+    # [('233dafd9b1d0b03e6e784987fe748be5.6', 1470806585), ('233dafd9b1d0b03e6e784987fe748be5.4', 1232810556)] 2703617141
+    # [('233dafd9b1d0b03e6e784987fe748be5.7', 1453587946), ('233dafd9b1d0b03e6e784987fe748be5.3', 1318304652)] 2771892598
+    # [('233dafd9b1d0b03e6e784987fe748be5.8', 1440314997), ('233dafd9b1d0b03e6e784987fe748be5.0', 1392554705)] 2832869702

recurvedata/pigeon/connector/hive_impala.py

@@ -0,0 +1,383 @@
+# flake8: noqa: E402
+
+# pylint: disable=wrong-import-position
+
+import os
+import re
+import shutil
+from typing import List, Optional, Union
+
+import pyhive.hive
+
+_ = 0  # prevent PyCharm to auto-format
+import cytoolz as toolz
+
+# impyla breaks TCLIService, which leads to ImportError while importing pyhive
+# see https://github.com/cloudera/impyla/issues/277
+import impala.dbapi
+import sqlalchemy
+from impala.error import HiveServer2Error
+from pyhive.exc import OperationalError
+
+from recurvedata.pigeon.connector._registry import register_connector_class
+from recurvedata.pigeon.connector.dbapi import DBAPIConnector, _ShowTableLikeMixin
+from recurvedata.pigeon.connector.hdfs import HDFSCliConnector, HDFSConnector
+from recurvedata.pigeon.const import HIVE_FILE_FORMATS
+from recurvedata.pigeon.schema import types
+from recurvedata.pigeon.utils import ensure_list, trim_suffix
+from recurvedata.pigeon.utils.sql import sqlformat
+
+_hive_type_to_canonical_type = {
+    "BOOLEAN": types.BOOLEAN,
+    "TINYINT": types.INT8,
+    "SMALLINT": types.INT16,
+    "INT": types.INT32,
+    "BIGINT": types.INT64,
+    "FLOAT": types.FLOAT32,
+    "DOUBLE": types.FLOAT64,
+    "DECIMAL": types.FLOAT64,
+    "REAL": types.FLOAT64,
+    "TIMESTAMP": types.DATETIME,
+    "DATE": types.DATE,
+    "CHAR": types.STRING,
+    "VARCHAR": types.STRING,
+    "STRING": types.STRING,
+}
+
+_canonical_type_to_hive_type = {
+    types.BOOLEAN: "BOOLEAN",
+    types.INT8: "TINYINT",
+    types.INT16: "SMALLINT",
+    types.INT32: "INT",
+    types.INT64: "BIGINT",
+    types.FLOAT32: "DOUBLE",
+    types.FLOAT64: "DOUBLE",
+    # treat date, datetime as string
+    types.DATE: "STRING",
+    types.DATETIME: "STRING",
+    types.STRING: "STRING",
+    types.JSON: "STRING",
+}
+
+
+class _HiveSQLMixin:
+    def create_partition_table_like(self, table, like_table, partitions):
+        """建一个分区表 Like 已有的一个未分区的表，并添加分区键"""
+        if not self.has_table(like_table):
+            raise ValueError(f"like table {like_table!r} not exists")
+        partitions = [f"`{pname}` {ptype}" for pname, ptype in partitions.items()]
+        partitions = ", ".join(partitions)
+        with self.cursor() as cursor:
+            cursor.execute(f"SELECT * FROM {like_table} LIMIT 0")
+            columns = [(x[0], x[1]) for x in cursor.description]
+        columns = ",\n".join("{} {}".format(*x) for x in columns)
+        sql = f"""
+            CREATE TABLE {table} (
+                {columns}
+            ) PARTITIONED BY ({partitions})
+        """
+        self.execute(sqlformat(sql))
+
+    def is_table_partitioned(self, database, table):
+        with self.cursor() as cursor:
+            try:
+                # 查看表的分区情况，如果没有报错就返回True
+                cursor.execute(f"SHOW PARTITIONS {database}.{table}")
+                return True
+            except (OperationalError, HiveServer2Error) as e:
+                msg = str(e).lower()
+                if "table not found" in msg or "table does not exist:" in msg:
+                    return False
+                elif "is not a partitioned table" in msg or "table is not partitioned" in msg:
+                    return False
+                else:
+                    raise e
+
+    @staticmethod
+    def to_canonical_type(type_code, size):
+        type_code = trim_suffix(type_code, "_TYPE")
+        return _hive_type_to_canonical_type.get(type_code, types.STRING)
+
+    @staticmethod
+    def from_canonical_type(canonical_type, size):
+        return _canonical_type_to_hive_type.get(canonical_type, "STRING")
+
+    def generate_create_table_ddl(self, name, schema, **kwargs):
+        cols = []
+        for f in schema:
+            col_name = self.quote_identifier(f.name)
+            if f.comment:
+                cols.append(f"{col_name} {self.from_canonical_type(f.type, f.size)} COMMENT {f.comment!r}")
+            else:
+                cols.append(f"{col_name} {self.from_canonical_type(f.type, f.size)}")
+
+        file_format = kwargs.get("file_format", "PARQUET")
+        col_types = ", \n".join(cols)
+        name = self.quote_identifier(name)
+        ddl = f"CREATE TABLE {name} (\n{col_types}\n) STORED AS {file_format}"
+        return ddl
+
+
+@register_connector_class("hive")
+class HiveConnector(_ShowTableLikeMixin, _HiveSQLMixin, DBAPIConnector):
+    _sqla_driver = "hive"
+    _log_query = False
+    _default_port = 10000
+
+    _complex_types = ("array", "map", "struct")
+
+    def connect_impl(self, autocommit=False, *args, **kwargs):
+        params = {
+            "host": self.host,
+            "port": self.port,
+            "username": self.user,
+            "database": self.database or "default",
+        }
+        if self.password:
+            params.update({"password": self.password, "auth": self.kwargs["auth"]})
+        hive_conf = self.hive_conf
+        hive_conf.update(kwargs.get("hive_conf", {}))
+        if hive_conf:
+            params["configuration"] = hive_conf
+        return pyhive.hive.connect(**params)
+
+    def create_engine(self, engine_kwargs=None, url_queries=None):
+        return sqlalchemy.create_engine("hive://", creator=self.connect)
+
+    def is_hive(self):
+        return True
+
+    @toolz.memoize
+    def create_hdfs_connector(self) -> Optional[HDFSConnector]:
+        hdfs_options = self.kwargs.get("hdfs_options")
+        if not hdfs_options:
+            return None
+        return HDFSConnector(**hdfs_options)
+
+    def has_complex_type_fields(self, table):
+        table = self.quote_identifier(table)
+        with self.cursor() as cursor:
+            cursor.execute("DESCRIBE {}".format(table))
+            for r in cursor.fetchall():
+                if r[0] == "":
+                    break
+                has_complex = any(x in r[1].lower() for x in self._complex_types)
+                if has_complex:
+                    return True
+        return False
+
+    def get_columns(self, table, database=None, exclude=None):
+        if database is None:
+            database = self.database
+        with self.cursor() as cursor:
+            if not self.has_table(table, database, cursor=cursor):
+                raise ValueError("Table {!r} not exists in {!r}".format(table, database))
+            # Hive bug https://issues.apache.org/jira/browse/HIVE-12184
+            cursor.execute("USE {}".format(self.quote_identifier(database)))
+            cursor.execute("DESCRIBE {}".format(self.quote_identifier(table)))
+            cols = []
+            for r in cursor.fetchall():
+                # the following is partition information
+                if r[0] == "":
+                    break
+                cols.append(r[0])
+        if exclude:
+            cols = [x for x in cols if x not in exclude]
+        return cols
+
+    def load_local_file(self, table, filepath, overwrite=True):
+        hdfs_clients = []
+        hdfs_cli = shutil.which("hdfs")
+        if hdfs_cli:
+            hdfs = HDFSCliConnector(hdfs_cli)
+            hdfs_clients.append(hdfs)
+        webhdfs = self.create_hdfs_connector()
+        if webhdfs:
+            hdfs_clients.append(webhdfs)
+
+        exc = None
+        for hdfs in hdfs_clients:
+            self.logger.info(f"try to load file using {hdfs}")
+            try:
+                self._load_local_file_to_hive_impl(table, filepath, hdfs, overwrite)
+                self.logger.info("finished load files")
+            except Exception as e:
+                exc = e
+                self.logger.exception(f"failed to load file using {hdfs}")
+            else:
+                exc = None
+                break
+
+        if exc:
+            raise exc
+
+    def _load_local_file_to_hive_impl(
+        self, table: str, filepath: Union[str, List[str]], hdfs: HDFSConnector, overwrite=True
+    ):
+        staging_folder = self.kwargs.get("hdfs_options", {}).get("staging_folder", "/tmp")
+        hdfs_folder = os.path.join(staging_folder, f"{self.database}_{table}_")
+        hdfs.delete_file(hdfs_folder, recursive=True)
+        hdfs.make_dir(hdfs_folder)
+        hdfs.upload_files(ensure_list(filepath), hdfs_folder)
+        query = f"LOAD DATA INPATH '{hdfs_folder}/*' {'OVERWRITE' if overwrite else ''} INTO TABLE {table}"
+        self.execute(query)
+        hdfs.delete_file(hdfs_folder, recursive=True)
+
+    def generate_ddl(self, table, database=None, if_exists=True, file_format="text"):
+        file_format = file_format.lower()
+        if file_format not in HIVE_FILE_FORMATS:
+            raise ValueError(f"Format {file_format!r} is not supported")
+        if database is None:
+            database = self.database
+        if not self.has_table(table, database):
+            raise ValueError(f"Table {table!r} not exists in {database!r}")
+
+        with self.cursor() as cursor:
+            cursor.execute(f"USE {self.quote_identifier(database)}")
+            cursor.execute(f"SHOW CREATE TABLE {self.quote_identifier(table)}")
+            result = cursor.fetchall()
+
+        body = ""
+        for r in result[1:]:
+            if "ROW FORMAT" in r[0]:
+                break
+            body += r[0]
+        if_exists_stmt = " IF NOT EXISTS " if if_exists else " "
+        file_format_stmt = f" STORED AS {HIVE_FILE_FORMATS[file_format]}"
+        return f"CREATE TABLE{if_exists_stmt}{self.quote_identifier(table)} ({body}{file_format_stmt}"
+
+    def _add_leading_comment_impl(self, query, comment):
+        comment = self._safe_comment(comment)
+        return "-- {}\n{}".format(comment, query.strip("\n"))
+
+    @property
+    def hive_conf(self):
+        """
+        用于设置 hive query 的参数，与在 hive 里执行 set xxx=xxx 基本一致（数字需要用字符串形式）；
+        字典类型，例如 {
+                    'spark.yarn.queue': 'etl',
+                    'spark.app.name': 'pigeon',
+                    'spark.executor.instances': '3'
+                  }
+        注意字典里数字要写成字符串的形式
+        """
+        if "hive_conf" in self.kwargs:
+            # 考虑到 hive_conf 都是单层 k,v ，不使用 deepcopy
+            return self.kwargs["hive_conf"].copy()
+        return {}
+
+    def generate_load_staging_table_ddl(self, staging_table, table, database=None, exclude_columns=None):
+        if database is None:
+            database = self.database
+        if exclude_columns:
+            exclude_columns = [col.lower().replace("`", "") for col in exclude_columns]
+
+        with self.cursor() as cursor:
+            cursor.execute(f"USE {self.quote_identifier(database)}")
+            cursor.execute(f"SHOW CREATE TABLE {self.quote_identifier(table)}")
+            result = cursor.fetchall()
+
+        body = pre_row = ""
+        for r in result[1:]:
+            row = r[0].lower().strip()
+            if row.startswith("partitioned by ("):
+                continue
+            if row.startswith("comment"):
+                continue
+            if exclude_columns:
+                col_name = row.split(" ")[0].strip("`")
+                if col_name in exclude_columns:
+                    continue
+            if row.endswith(")"):
+                row = ",".join(row.rsplit(")", 1))
+            if row.startswith("row format"):
+                pre_row = ")".join(pre_row.rsplit(",", 1))
+                body += pre_row
+                break
+            body += pre_row
+            pre_row = row
+
+        return f"CREATE TABLE {self.quote_identifier(staging_table)} ({body}"
+
+
+@register_connector_class("impala")
+class ImpalaConnector(_ShowTableLikeMixin, _HiveSQLMixin, DBAPIConnector):
+    _sqla_driver = "impala"
+    _default_port = 21050
+
+    def connect_impl(self, autocommit=False, *args, **kwargs):
+        params = {
+            "host": self.host,
+            "port": self.port,
+            "database": self.database or "default",
+            "user": self.user,
+            "password": self.password,
+        }
+        if "auth_mechanism" in self.kwargs:
+            params["auth_mechanism"] = self.kwargs["auth_mechanism"]
+        return impala.dbapi.connect(**params)
+
+    def create_engine(self, engine_kwargs=None, url_queries=None):
+        return sqlalchemy.create_engine("impala://", creator=self.connect)
+
+    def is_impala(self):
+        return True
+
+    def get_columns(self, table, database=None, exclude=None):
+        if database is None:
+            database = self.database
+        with self.cursor() as cursor:
+            if not self.has_table(table, database, cursor=cursor):
+                raise ValueError("Table {!r} not exists in {!r}".format(table, database))
+            cursor.execute("DESCRIBE {}.{}".format(self.quote_identifier(database), self.quote_identifier(table)))
+            cols = [x[0] for x in cursor.fetchall()]
+        if exclude:
+            cols = [x for x in cols if x not in exclude]
+        return cols
+
+    def invalidate_metadata(self, table=None):
+        if table:
+            table = self.quote_identifier(table)
+        else:
+            table = ""
+        query = f"INVALIDATE METADATA {table}"
+        self.execute(query)
+
+    def refresh(self, table, compute_stats=True):
+        table = self.quote_identifier(table)
+        queries = "REFRESH {}".format(table)
+        try:
+            self.execute(queries)
+        except Exception as e:
+            self.logger.error(f"failed to refresh, err: {e}, use INVALIDATE")
+            queries = "INVALIDATE METADATA {}".format(table)
+            self.execute(queries)
+        if compute_stats:
+            queries = "COMPUTE INCREMENTAL STATS {}".format(table)
+            self.execute(queries)
+
+    def generate_ddl(self, table, database=None, if_exists=True, file_format="text"):
+        file_format = file_format.lower()
+        # ORC is not supported in Impala
+        # https://www.cloudera.com/documentation/enterprise/5-12-x/topics/impala_file_formats.html
+        if file_format == "orc" or file_format not in HIVE_FILE_FORMATS:
+            raise ValueError(f"Format {file_format!r} is not supported")
+        if database is None:
+            database = self.database
+        if not self.has_table(table, database):
+            raise ValueError(f"Table {table!r} not exists in {database!r}")
+
+        with self.cursor() as cursor:
+            cursor.execute(f"USE {self.quote_identifier(database)}")
+            cursor.execute(f"SHOW CREATE TABLE {self.quote_identifier(table)}")
+            body = re.search(r"\.(.*)\nSTORED", cursor.fetchall()[0][0], flags=re.S).group(1)
+        if_exists_stmt = " IF NOT EXISTS " if if_exists else " "
+        file_format_stmt = f" STORED AS {HIVE_FILE_FORMATS[file_format]}"
+        return f"CREATE TABLE{if_exists_stmt}{body}{file_format_stmt}"
+
+    @toolz.memoize
+    def create_hdfs_connector(self):
+        hdfs_options = self.kwargs.get("hdfs_options")
+        if not hdfs_options:
+            return None
+        return HDFSConnector(**hdfs_options)

recurvedata/pigeon/connector/microsoft_fabric.py

@@ -0,0 +1,95 @@
+from collections import OrderedDict
+from typing import Any, Optional
+
+import pyodbc
+
+from recurvedata.pigeon.connector._registry import register_connector_class
+from recurvedata.pigeon.connector.mssql import BaseAzureSQLConnector
+from recurvedata.pigeon.schema import types
+from recurvedata.pigeon.utils import safe_int
+
+
+@register_connector_class("microsoft_fabric")  # type: ignore
+class MsFabricConnector(BaseAzureSQLConnector):
+    """Connector for Microsoft Fabric.
+
+    This connector extends BaseAzureSQLConnector to support Microsoft Fabric specific features:
+    - Azure AD authentication
+    - Workspace-level access control
+    - Special ODBC driver configuration
+    """
+
+    def __init__(
+        self,
+        host: str | None = None,
+        port: int | None = None,
+        database: str | None = None,
+        schema: str | None = None,
+        tenant_id: str | None = None,
+        client_id: str | None = None,
+        client_secret: str | None = None,
+        authentication: str = "ServicePrincipal",
+        odbc_driver: str = "ODBC Driver 18 for SQL Server",
+        encrypt: bool = True,
+        trust_server_certificate: bool = False,
+        *args: Any,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(host, port, database, schema=schema, *args, **kwargs)
+        self.tenant_id = tenant_id
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.authentication = authentication
+        self.odbc_driver = odbc_driver
+        self.driver = "mssql+pyodbc"
+        self.encrypt = encrypt
+        self.trust_server_certificate = trust_server_certificate
+
+    def _get_sqlalchemy_uri(self) -> str:
+        """Generate SQLAlchemy URI for Microsoft Fabric."""
+        return (
+            f"{self.driver}://{self.client_id}:{self.client_secret}@{self.host}:{self.port}/"
+            f"{self.database}?driver={self.odbc_driver}&encrypt={self.encrypt}&trust_server_certificate={self.trust_server_certificate}"
+        )
+
+    def is_fabric(self) -> bool:
+        """Check if this is a Microsoft Fabric connector."""
+        return True
+
+    @staticmethod
+    def to_canonical_type(type_code: Any, size: Optional[int] = None) -> str:
+        """Convert Microsoft Fabric type to canonical type."""
+        return BaseAzureSQLConnector.to_canonical_type(type_code, size)
+
+    @staticmethod
+    def from_canonical_type(canonical_type: str, size: Optional[int] = None) -> str:
+        """Convert canonical type to Microsoft Fabric type."""
+        if canonical_type == types.STRING:
+            if size is None or size == 0:
+                return "VARCHAR(max)"
+            safe_size = safe_int(size * 4)
+            if safe_size > 4000:
+                return "VARCHAR(max)"
+            return f"VARCHAR({safe_size})"
+        return BaseAzureSQLConnector.from_canonical_type(canonical_type, size)
+
+    @property
+    def conn_string(self) -> str:
+        """Generate connection string for Microsoft Fabric with Azure AD authentication."""
+        options = OrderedDict(
+            {
+                "Driver": f"{{{self.odbc_driver}}}",
+                "Server": f"{self.host}",
+                "Database": str(self.database),
+                "Authentication": "ActiveDirectoryServicePrincipal",
+                "Encrypt": "yes" if self.encrypt else "no",
+                "TrustServerCertificate": "yes" if self.trust_server_certificate else "no",
+                "Uid": self.client_id,
+                "Pwd": self.client_secret,
+                "Connection Timeout": 30,
+            }
+        )
+        return ";".join([f"{k}={v}" for k, v in options.items()])
+
+    def connect_impl(self, autocommit=False, *args, **kwargs):
+        return pyodbc.connect(self.conn_string, autocommit=autocommit)