lsst-felis 28.2024.4500__py3-none-any.whl → 30.0.0rc3__py3-none-any.whl

felis/__init__.py

@@ -19,4 +19,12 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from .version import *
+from .datamodel import *
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("lsst-felis")
+except PackageNotFoundError:
+    # Package not installed or scons not run.
+    __version__ = "0.0.0"

felis/cli.py

@@ -29,15 +29,13 @@ from typing import IO
 
 import click
 from pydantic import ValidationError
-from sqlalchemy.engine import Engine, create_engine, make_url
-from sqlalchemy.engine.mock import MockConnection, create_mock_engine
 
 from . import __version__
 from .datamodel import Schema
-from .db.utils import DatabaseContext, is_mock_url
-from .metadata import MetaDataBuilder
-from .tap import Tap11Base, TapLoadingVisitor, init_tables
-from .tap_schema import DataLoader, TableManager
+from .db.database_context import create_database_context
+from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
+from .metadata import create_metadata
+from .tap_schema import DataLoader, MetadataInserter, TableManager
 
 __all__ = ["cli"]
 
@@ -62,7 +60,10 @@ loglevel_choices = ["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"]
     help="Felis log file path",
 )
 @click.option(
-    "--id-generation", is_flag=True, help="Generate IDs for all objects that do not have them", default=False
+    "--id-generation/--no-id-generation",
+    is_flag=True,
+    help="Generate IDs for all objects that do not have them",
+    default=True,
 )
 @click.pass_context
 def cli(ctx: click.Context, log_level: str, log_file: str | None, id_generation: bool) -> None:
@@ -71,6 +72,8 @@ def cli(ctx: click.Context, log_level: str, log_file: str | None, id_generation:
     ctx.obj["id_generation"] = id_generation
     if ctx.obj["id_generation"]:
         logger.info("ID generation is enabled")
+    else:
+        logger.info("ID generation is disabled")
     if log_file:
         logging.basicConfig(filename=log_file, level=log_level)
     else:
@@ -94,6 +97,7 @@ def cli(ctx: click.Context, log_level: str, log_file: str | None, id_generation:
     "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
 )
 @click.option("--ignore-constraints", is_flag=True, help="Ignore constraints when creating tables")
+@click.option("--skip-indexes", is_flag=True, help="Skip creating indexes when building metadata")
 @click.argument("file", type=click.File())
 @click.pass_context
 def create(
@@ -106,6 +110,7 @@ def create(
     dry_run: bool,
     output_file: IO[str] | None,
     ignore_constraints: bool,
+    skip_indexes: bool,
     file: IO[str],
 ) -> None:
     """Create database objects from the Felis file.
@@ -128,230 +133,141 @@ def create(
         Write SQL commands to a file instead of executing.
     ignore_constraints
         Ignore constraints when creating tables.
+    skip_indexes
+        Skip creating indexes when building metadata.
     file
         Felis file to read.
     """
     try:
-        schema = Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]})
-        url = make_url(engine_url)
-        if schema_name:
-            logger.info(f"Overriding schema name with: {schema_name}")
-            schema.name = schema_name
-        elif url.drivername == "sqlite":
-            logger.info("Overriding schema name for sqlite with: main")
-            schema.name = "main"
-        if not url.host and not url.drivername == "sqlite":
-            dry_run = True
-            logger.info("Forcing dry run for non-sqlite engine URL with no host")
-
-        metadata = MetaDataBuilder(schema, ignore_constraints=ignore_constraints).build()
-        logger.debug(f"Created metadata with schema name: {metadata.schema}")
-
-        engine: Engine | MockConnection
-        if not dry_run and not output_file:
-            engine = create_engine(url, echo=echo)
-        else:
-            if dry_run:
-                logger.info("Dry run will be executed")
-            engine = DatabaseContext.create_mock_engine(url, output_file)
-            if output_file:
-                logger.info("Writing SQL output to: " + output_file.name)
+        metadata = create_metadata(
+            file,
+            id_generation=ctx.obj["id_generation"],
+            schema_name=schema_name,
+            ignore_constraints=ignore_constraints,
+            skip_indexes=skip_indexes,
+            engine_url=engine_url,
+        )
 
-        context = DatabaseContext(metadata, engine)
+        with create_database_context(
+            engine_url,
+            metadata,
+            echo=echo,
+            dry_run=dry_run,
+            output_file=output_file,
+        ) as db_ctx:
+            if drop and initialize:
+                raise ValueError("Cannot drop and initialize schema at the same time")
 
-        if drop and initialize:
-            raise ValueError("Cannot drop and initialize schema at the same time")
+            if drop:
+                logger.debug("Dropping schema if it exists")
+                db_ctx.drop()
+                initialize = True  # If schema is dropped, it needs to be recreated.
 
-        if drop:
-            logger.debug("Dropping schema if it exists")
-            context.drop()
-            initialize = True  # If schema is dropped, it needs to be recreated.
+            if initialize:
+                logger.debug("Creating schema if not exists")
+                db_ctx.initialize()
 
-        if initialize:
-            logger.debug("Creating schema if not exists")
-            context.initialize()
+            db_ctx.create_all()
 
-        context.create_all()
     except Exception as e:
         logger.exception(e)
         raise click.ClickException(str(e))
 
 
-@cli.command("init-tap", help="Initialize TAP_SCHEMA objects in the database")
-@click.option("--tap-schema-name", help="Alternate database schema name for 'TAP_SCHEMA'")
-@click.option("--tap-schemas-table", help="Alternate table name for 'schemas'")
-@click.option("--tap-tables-table", help="Alternate table name for 'tables'")
-@click.option("--tap-columns-table", help="Alternate table name for 'columns'")
-@click.option("--tap-keys-table", help="Alternate table name for 'keys'")
-@click.option("--tap-key-columns-table", help="Alternate table name for 'key_columns'")
-@click.argument("engine-url")
-def init_tap(
+@cli.command("create-indexes", help="Create database indexes defined in the Felis file")
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
+@click.option("--schema-name", help="Alternate schema name to override Felis file")
+@click.argument("file", type=click.File())
+@click.pass_context
+def create_indexes(
+    ctx: click.Context,
     engine_url: str,
-    tap_schema_name: str,
-    tap_schemas_table: str,
-    tap_tables_table: str,
-    tap_columns_table: str,
-    tap_keys_table: str,
-    tap_key_columns_table: str,
+    schema_name: str | None,
+    file: IO[str],
 ) -> None:
-    """Initialize TAP_SCHEMA objects in the database.
+    """Create indexes from a Felis YAML file in a target database.
 
     Parameters
     ----------
     engine_url
-        SQLAlchemy Engine URL. The target PostgreSQL schema or MySQL database
-        must already exist and be referenced in the URL.
-    tap_schema_name
-        Alterate name for the database schema ``TAP_SCHEMA``.
-    tap_schemas_table
-        Alterate table name for ``schemas``.
-    tap_tables_table
-        Alterate table name for ``tables``.
-    tap_columns_table
-        Alterate table name for ``columns``.
-    tap_keys_table
-        Alterate table name for ``keys``.
-    tap_key_columns_table
-        Alterate table name for ``key_columns``.
-
-    Notes
-    -----
-    The supported version of TAP_SCHEMA in the SQLAlchemy metadata is 1.1. The
-    tables are created in the database schema specified by the engine URL,
-    which must be a PostgreSQL schema or MySQL database that already exists.
+        SQLAlchemy Engine URL.
+    file
+        Felis file to read.
     """
-    engine = create_engine(engine_url)
-    init_tables(
-        tap_schema_name,
-        tap_schemas_table,
-        tap_tables_table,
-        tap_columns_table,
-        tap_keys_table,
-        tap_key_columns_table,
-    )
-    Tap11Base.metadata.create_all(engine)
+    try:
+        metadata = create_metadata(
+            file, id_generation=ctx.obj["id_generation"], schema_name=schema_name, engine_url=engine_url
+        )
+        with create_database_context(engine_url, metadata) as db_ctx:
+            db_ctx.create_indexes()
+    except Exception as e:
+        logger.exception(e)
+        raise click.ClickException("Error creating indexes: " + str(e))
 
 
-@cli.command("load-tap", help="Load metadata from a Felis file into a TAP_SCHEMA database")
-@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
-@click.option("--schema-name", help="Alternate Schema Name for Felis file")
-@click.option("--catalog-name", help="Catalog Name for Schema")
-@click.option("--dry-run", is_flag=True, help="Dry Run Only. Prints out the DDL that would be executed")
-@click.option("--tap-schema-name", help="Alternate schema name for 'TAP_SCHEMA'")
-@click.option("--tap-tables-postfix", help="Postfix for TAP_SCHEMA table names")
-@click.option("--tap-schemas-table", help="Alternate table name for 'schemas'")
-@click.option("--tap-tables-table", help="Alternate table name for 'tables'")
-@click.option("--tap-columns-table", help="Alternate table name for 'columns'")
-@click.option("--tap-keys-table", help="Alternate table name for 'keys'")
-@click.option("--tap-key-columns-table", help="Alternate table name for 'key_columns'")
-@click.option("--tap-schema-index", type=int, help="TAP_SCHEMA index of the schema in this environment")
+@cli.command("drop-indexes", help="Drop database indexes defined in the Felis file")
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
+@click.option("--schema-name", help="Alternate schema name to override Felis file")
 @click.argument("file", type=click.File())
-def load_tap(
+@click.pass_context
+def drop_indexes(
+    ctx: click.Context,
     engine_url: str,
-    schema_name: str,
-    catalog_name: str,
-    dry_run: bool,
-    tap_schema_name: str,
-    tap_tables_postfix: str,
-    tap_schemas_table: str,
-    tap_tables_table: str,
-    tap_columns_table: str,
-    tap_keys_table: str,
-    tap_key_columns_table: str,
-    tap_schema_index: int,
+    schema_name: str | None,
     file: IO[str],
 ) -> None:
-    """Load TAP metadata from a Felis file.
-
-    This command loads the associated TAP metadata from a Felis YAML file
-    into the TAP_SCHEMA tables.
+    """Drop indexes from a Felis YAML file in a target database.
 
     Parameters
     ----------
     engine_url
-        SQLAlchemy Engine URL to catalog.
-    schema_name
-        Alternate schema name. This overrides the schema name in the
-        ``catalog`` field of the Felis file.
-    catalog_name
-        Catalog name for the schema. This possibly duplicates the
-        ``tap_schema_name`` argument (DM-44870).
-    dry_run
-        Dry run only to print out commands instead of executing.
-    tap_schema_name
-        Alternate name for the schema of TAP_SCHEMA in the database.
-    tap_tables_postfix
-        Postfix for TAP table names that will be automatically appended.
-    tap_schemas_table
-        Alternate table name for ``schemas``.
-    tap_tables_table
-        Alternate table name for ``tables``.
-    tap_columns_table
-        Alternate table name for ``columns``.
-    tap_keys_table
-        Alternate table name for ``keys``.
-    tap_key_columns_table
-        Alternate table name for ``key_columns``.
-    tap_schema_index
-        TAP_SCHEMA index of the schema in this TAP environment.
+        SQLAlchemy Engine URL.
+    schema-name
+        Alternate schema name to override Felis file.
     file
         Felis file to read.
-
-    Notes
-    -----
-    The data will be loaded into the TAP_SCHEMA from the engine URL. The
-    tables must have already been initialized or an error will occur.
     """
-    schema = Schema.from_stream(file)
-
-    tap_tables = init_tables(
-        tap_schema_name,
-        tap_tables_postfix,
-        tap_schemas_table,
-        tap_tables_table,
-        tap_columns_table,
-        tap_keys_table,
-        tap_key_columns_table,
-    )
-
-    if not dry_run:
-        engine = create_engine(engine_url)
-
-        if engine_url == "sqlite://" and not dry_run:
-            # In Memory SQLite - Mostly used to test
-            Tap11Base.metadata.create_all(engine)
-
-        tap_visitor = TapLoadingVisitor(
-            engine,
-            catalog_name=catalog_name,
-            schema_name=schema_name,
-            tap_tables=tap_tables,
-            tap_schema_index=tap_schema_index,
-        )
-        tap_visitor.visit_schema(schema)
-    else:
-        conn = DatabaseContext.create_mock_engine(engine_url)
-
-        tap_visitor = TapLoadingVisitor.from_mock_connection(
-            conn,
-            catalog_name=catalog_name,
-            schema_name=schema_name,
-            tap_tables=tap_tables,
-            tap_schema_index=tap_schema_index,
+    try:
+        metadata = create_metadata(
+            file, id_generation=ctx.obj["id_generation"], schema_name=schema_name, engine_url=engine_url
         )
-        tap_visitor.visit_schema(schema)
+        with create_database_context(engine_url, metadata) as db:
+            db.drop_indexes()
+    except Exception as e:
+        logger.exception(e)
+        raise click.ClickException("Error dropping indexes: " + str(e))
 
 
 @cli.command("load-tap-schema", help="Load metadata from a Felis file into a TAP_SCHEMA database")
 @click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
-@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
 @click.option(
-    "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
+    "--tap-schema-name", "-n", help="Name of the TAP_SCHEMA schema in the database (default: TAP_SCHEMA)"
+)
+@click.option(
+    "--tap-tables-postfix",
+    "-p",
+    help="Postfix which is applied to standard TAP_SCHEMA table names",
+    default="",
+)
+@click.option("--tap-schema-index", "-i", type=int, help="TAP_SCHEMA index of the schema in this environment")
+@click.option("--dry-run", "-D", is_flag=True, help="Execute dry run only. Does not insert any data.")
+@click.option("--echo", "-e", is_flag=True, help="Print out the generated insert statements to stdout")
+@click.option(
+    "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
+)
+@click.option(
+    "--force-unbounded-arraysize",
+    is_flag=True,
+    help="Use unbounded arraysize by default for all variable length string columns"
+    ", e.g., ``votable:arraysize: *`` (workaround for astropy bug #18099)",
+)  # DM-50899: Variable-length bounded strings are not handled correctly in astropy
+@click.option(
+    "--unique-keys",
+    "-u",
+    is_flag=True,
+    help="Generate unique key_id values for keys and key_columns tables by prepending the schema name",
+    default=False,
 )
-@click.option("--tap-schema-index", type=int, help="TAP_SCHEMA index of the schema in this environment")
-@click.option("--dry-run", is_flag=True, help="Execute dry run only. Does not insert any data.")
-@click.option("--echo", is_flag=True, help="Print out the generated insert statements to stdout")
-@click.option("--output-file", type=click.Path(), help="Write SQL commands to a file")
 @click.argument("file", type=click.File())
 @click.pass_context
 def load_tap_schema(
@@ -362,7 +278,9 @@ def load_tap_schema(
     tap_schema_index: int,
     dry_run: bool,
     echo: bool,
-    output_file: str | None,
+    output_file: IO[str] | None,
+    force_unbounded_arraysize: bool,
+    unique_keys: bool,
     file: IO[str],
 ) -> None:
     """Load TAP metadata from a Felis file.
@@ -389,30 +307,99 @@ def load_tap_schema(
     The TAP_SCHEMA database must already exist or the command will fail. This
     command will not initialize the TAP_SCHEMA tables.
     """
-    url = make_url(engine_url)
-    engine: Engine | MockConnection
-    if dry_run or is_mock_url(url):
-        engine = create_mock_engine(url, executor=None)
-    else:
-        engine = create_engine(engine_url)
+    # Create TableManager with automatic dialect detection
     mgr = TableManager(
-        engine=engine,
-        apply_schema_to_metadata=False if engine.dialect.name == "sqlite" else True,
+        engine_url=engine_url,
         schema_name=tap_schema_name,
         table_name_postfix=tap_tables_postfix,
     )
 
-    schema = Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]})
+    # Create DatabaseContext using TableManager's metadata
+    with create_database_context(
+        engine_url, mgr.metadata, echo=echo, dry_run=dry_run, output_file=output_file
+    ) as db_ctx:
+        schema = Schema.from_stream(
+            file,
+            context={
+                "id_generation": ctx.obj["id_generation"],
+                "force_unbounded_arraysize": force_unbounded_arraysize,
+            },
+        )
 
-    DataLoader(
-        schema,
-        mgr,
-        engine,
-        tap_schema_index=tap_schema_index,
-        dry_run=dry_run,
-        print_sql=echo,
-        output_path=output_file,
-    ).load()
+        DataLoader(
+            schema,
+            mgr,
+            db_context=db_ctx,
+            tap_schema_index=tap_schema_index,
+            dry_run=dry_run,
+            print_sql=echo,
+            output_file=output_file,
+            unique_keys=unique_keys,
+        ).load()
+
+
+@cli.command("init-tap-schema", help="Initialize a standard TAP_SCHEMA database")
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", required=True)
+@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
+@click.option(
+    "--extensions",
+    type=str,
+    default=None,
+    help=(
+        "Optional path to extensions YAML file (system path or resource:// URI). "
+        "If not provided, no extensions will be applied. "
+        "Example (default packaged extensions): "
+        "--extensions resource://felis/config/tap_schema/tap_schema_extensions.yaml"
+    ),
+)
+@click.option(
+    "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
+)
+@click.option(
+    "--insert-metadata/--no-insert-metadata",
+    is_flag=True,
+    help="Insert metadata describing TAP_SCHEMA itself",
+    default=True,
+)
+@click.pass_context
+def init_tap_schema(
+    ctx: click.Context,
+    engine_url: str,
+    tap_schema_name: str,
+    extensions: str | None,
+    tap_tables_postfix: str,
+    insert_metadata: bool,
+) -> None:
+    """Initialize a standard TAP_SCHEMA database.
+
+    Parameters
+    ----------
+    engine_url
+        SQLAlchemy Engine URL.
+    tap_schema_name
+        Name of the TAP_SCHEMA schema in the database.
+    extensions
+        Extensions YAML file.
+    tap_tables_postfix
+        Postfix which is applied to standard TAP_SCHEMA table names.
+    insert_metadata
+        Insert metadata describing TAP_SCHEMA itself.
+        If set to False, only the TAP_SCHEMA tables will be created, but no
+        metadata will be inserted.
+    """
+    # Create TableManager with automatic dialect detection
+    mgr = TableManager(
+        engine_url=engine_url,
+        schema_name=tap_schema_name,
+        table_name_postfix=tap_tables_postfix,
+        extensions_path=extensions,
+    )
+
+    # Create DatabaseContext using TableManager's metadata
+    with create_database_context(engine_url, mgr.metadata) as db_ctx:
+        mgr.initialize_database(db_context=db_ctx)
+        if insert_metadata:
+            MetadataInserter(mgr, db_context=db_ctx).insert_metadata()
 
 
 @cli.command("validate", help="Validate one or more Felis YAML files")
@@ -493,5 +480,117 @@ def validate(
         raise click.exceptions.Exit(rc)
 
 
+@cli.command(
+    "diff",
+    help="""
+    Compare two schemas or a schema and a database for changes
+
+    Examples:
+
+      felis diff schema1.yaml schema2.yaml
+
+      felis diff -c alembic schema1.yaml schema2.yaml
+
+      felis diff --engine-url sqlite:///test.db schema.yaml
+    """,
+)
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
+@click.option(
+    "-c",
+    "--comparator",
+    type=click.Choice(["alembic", "deepdiff"], case_sensitive=False),
+    help="Comparator to use for schema comparison",
+    default="deepdiff",
+)
+@click.option("-E", "--error-on-change", is_flag=True, help="Exit with error code if schemas are different")
+@click.argument("files", nargs=-1, type=click.File())
+@click.pass_context
+def diff(
+    ctx: click.Context,
+    engine_url: str | None,
+    comparator: str,
+    error_on_change: bool,
+    files: Iterable[IO[str]],
+) -> None:
+    files_list = list(files)
+    schemas = [
+        Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]}) for file in files_list
+    ]
+    diff: SchemaDiff
+    if len(schemas) == 2:
+        if comparator == "alembic":
+            # Reset file stream to beginning before re-reading
+            files_list[0].seek(0)
+            metadata = create_metadata(
+                files_list[0], id_generation=ctx.obj["id_generation"], engine_url=engine_url
+            )
+            with create_database_context(
+                engine_url if engine_url else "sqlite:///:memory:", metadata
+            ) as db_ctx:
+                db_ctx.initialize()
+                db_ctx.create_all()
+                diff = DatabaseDiff(schemas[1], db_ctx.engine)
+        else:
+            diff = FormattedSchemaDiff(schemas[0], schemas[1])
+    elif len(schemas) == 1 and engine_url is not None:
+        # Create minimal metadata for the context manager
+        from sqlalchemy import MetaData
+
+        metadata = MetaData()
+
+        with create_database_context(engine_url, metadata) as db_ctx:
+            diff = DatabaseDiff(schemas[0], db_ctx.engine)
+    else:
+        raise click.ClickException(
+            "Invalid arguments - provide two schemas or a single schema and a database engine URL"
+        )
+
+    diff.print()
+
+    if diff.has_changes and error_on_change:
+        raise click.ClickException("Schema was changed")
+
+
+@cli.command(
+    "dump",
+    help="""
+    Dump a schema file to YAML or JSON format
+
+    Example:
+
+      felis dump schema.yaml schema.json
+
+      felis dump schema.yaml schema_dump.yaml
+    """,
+)
+@click.option(
+    "--strip-ids/--no-strip-ids",
+    is_flag=True,
+    help="Strip IDs from the output schema",
+    default=False,
+)
+@click.argument("files", nargs=2, type=click.Path())
+@click.pass_context
+def dump(
+    ctx: click.Context,
+    strip_ids: bool,
+    files: list[str],
+) -> None:
+    if strip_ids:
+        logger.info("Stripping IDs from the output schema")
+    if files[1].endswith(".json"):
+        format = "json"
+    elif files[1].endswith(".yaml"):
+        format = "yaml"
+    else:
+        raise click.ClickException("Output file must have a .json or .yaml extension")
+    schema = Schema.from_uri(files[0], context={"id_generation": ctx.obj["id_generation"]})
+    with open(files[1], "w") as f:
+        if format == "yaml":
+            schema.dump_yaml(f, strip_ids=strip_ids)
+        elif format == "json":
+            schema.dump_json(f, strip_ids=strip_ids)
+
+
 if __name__ == "__main__":
     cli()

felis/config/tap_schema/columns.csv

@@ -0,0 +1,33 @@
+table_name,column_name,utype,ucd,unit,description,datatype,arraysize,xtype,size,principal,indexed,std,column_index
+tap_schema.columns,"""size""",\N,\N,\N,deprecated: use arraysize,int,\N,\N,\N,1,0,1,9
+tap_schema.columns,arraysize,\N,\N,\N,lists the size of variable-length columns in the tableset,char,16*,\N,16,1,0,1,8
+tap_schema.columns,column_index,\N,\N,\N,recommended sort order when listing columns of a table,int,\N,\N,\N,1,0,1,13
+tap_schema.columns,column_name,\N,\N,\N,the column name,char,64*,\N,64,1,0,1,2
+tap_schema.columns,datatype,\N,\N,\N,lists the ADQL datatype of columns in the tableset,char,64*,\N,64,1,0,1,7
+tap_schema.columns,description,\N,\N,\N,describes the columns in the tableset,char,512*,\N,512,1,0,1,6
+tap_schema.columns,indexed,\N,\N,\N,"an indexed column; 1 means 1, 0 means 0",int,\N,\N,\N,1,0,1,11
+tap_schema.columns,principal,\N,\N,\N,"a principal column; 1 means 1, 0 means 0",int,\N,\N,\N,1,0,1,10
+tap_schema.columns,std,\N,\N,\N,"a standard column; 1 means 1, 0 means 0",int,\N,\N,\N,1,0,1,12
+tap_schema.columns,table_name,\N,\N,\N,the table this column belongs to,char,64*,\N,64,1,0,1,1
+tap_schema.columns,ucd,\N,\N,\N,lists the UCDs of columns in the tableset,char,64*,\N,64,1,0,1,4
+tap_schema.columns,unit,\N,\N,\N,lists the unit used for column values in the tableset,char,64*,\N,64,1,0,1,5
+tap_schema.columns,utype,\N,\N,\N,lists the utypes of columns in the tableset,char,512*,\N,512,1,0,1,3
+tap_schema.columns,xtype,\N,\N,\N,a DALI or custom extended type annotation,char,64*,\N,64,1,0,1,7
+tap_schema.key_columns,from_column,\N,\N,\N,column in the from_table,char,64*,\N,64,1,0,1,2
+tap_schema.key_columns,key_id,\N,\N,\N,key to join to tap_schema.keys,char,64*,\N,64,1,0,1,1
+tap_schema.key_columns,target_column,\N,\N,\N,column in the target_table,char,64*,\N,64,1,0,1,3
+tap_schema.keys,description,\N,\N,\N,describes keys in the tableset,char,512*,\N,512,1,0,1,5
+tap_schema.keys,from_table,\N,\N,\N,the table with the foreign key,char,64*,\N,64,1,0,1,2
+tap_schema.keys,key_id,\N,\N,\N,unique key to join to tap_schema.key_columns,char,64*,\N,64,1,0,1,1
+tap_schema.keys,target_table,\N,\N,\N,the table with the primary key,char,64*,\N,64,1,0,1,3
+tap_schema.keys,utype,\N,\N,\N,lists the utype of keys in the tableset,char,512*,\N,512,1,0,1,4
+tap_schema.schemas,description,\N,\N,\N,describes schemas in the tableset,char,512*,\N,512,1,0,1,3
+tap_schema.schemas,schema_index,\N,\N,\N,recommended sort order when listing schemas,int,\N,\N,\N,1,0,1,4
+tap_schema.schemas,schema_name,\N,\N,\N,schema name for reference to tap_schema.schemas,char,64*,\N,64,1,0,1,1
+tap_schema.schemas,utype,\N,\N,\N,lists the utypes of schemas in the tableset,char,512*,\N,512,1,0,1,2
+tap_schema.tables,description,\N,\N,\N,describes tables in the tableset,char,512*,\N,512,1,0,1,5
+tap_schema.tables,schema_name,\N,\N,\N,the schema this table belongs to,char,512*,\N,512,1,0,1,1
+tap_schema.tables,table_index,\N,\N,\N,recommended sort order when listing tables,int,\N,\N,\N,1,0,1,6
+tap_schema.tables,table_name,\N,\N,\N,the fully qualified table name,char,64*,\N,64,1,0,1,2
+tap_schema.tables,table_type,\N,\N,\N,one of: table view,char,8*,\N,8,1,0,1,3
+tap_schema.tables,utype,\N,\N,\N,lists the utype of tables in the tableset,char,512*,\N,512,1,0,1,4