ingestr 0.0.1__tar.gz → 0.0.3__tar.gz

{ingestr-0.0.1 → ingestr-0.0.3}/Makefile

@@ -1,15 +1,15 @@
 .ONESHELL:
-.PHONY: test lint format ftl test-ci lint-ci build
+.PHONY: test lint format ftl test-ci lint-ci build upload-release
 
 venv: venv/touchfile
 
-venv/touchfile: requirements.txt
+venv/touchfile: requirements-dev.txt
 	test -d venv || python3 -m venv venv
-	. venv/bin/activate; pip install -r requirements.txt
+	. venv/bin/activate; pip install -r requirements-dev.txt
 	touch venv/touchfile
 
-install-deps:
-	pip install -r requirements.txt
+deps:
+	pip install -r requirements-dev.txt
 
 test-ci:
 	pytest -rP -vv --tb=short --cov=ingestr --no-cov-on-fail
@@ -32,3 +32,5 @@ tl: test lint
 build:
 	python3 -m build
 
+upload-release:
+	twine upload dist/*

{ingestr-0.0.1 → ingestr-0.0.3}/PKG-INFO

@@ -1,15 +1,32 @@
 Metadata-Version: 2.1
 Name: ingestr
-Version: 0.0.1
+Version: 0.0.3
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
 Author-email: Burak Karakan <burak.karakan@getbruin.com>
 License-File: LICENSE.md
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
 Requires-Python: >=3.9
+Requires-Dist: databricks-sql-connector==2.9.3
+Requires-Dist: dlt==0.4.3
+Requires-Dist: duckdb==0.9.2
+Requires-Dist: pendulum==3.0.0
+Requires-Dist: psycopg2==2.9.9
+Requires-Dist: pyodbc==5.1.0
+Requires-Dist: rich==13.7.0
+Requires-Dist: snowflake-sqlalchemy==1.5.1
+Requires-Dist: sqlalchemy-bigquery==1.9.0
+Requires-Dist: sqlalchemy2-stubs==0.0.2a38
+Requires-Dist: sqlalchemy==1.4.51
+Requires-Dist: tqdm==4.66.2
+Requires-Dist: typer==0.9.0
 Description-Content-Type: text/markdown
 
 <div align="center">

{ingestr-0.0.1 → ingestr-0.0.3}/ingestr/main.py

@@ -4,38 +4,37 @@ import dlt
 import typer
 
 from ingestr.src.factory import SourceDestinationFactory
-from rich import print
+from rich.console import Console
 from dlt.common.pipeline import LoadInfo
 import humanize
+from typing_extensions import Annotated
 
-app = typer.Typer(name="ingestr")
+app = typer.Typer(
+    name="ingestr",
+    help="ingestr is the CLI tool to ingest data from one source to another",
+    rich_markup_mode="rich",
+)
+
+
+console = Console()
+print = console.print
 
 
 @app.command()
 def ingest(
-    source_uri: str = None,  # type: ignore
-    dest_uri: str = None,  # type: ignore
-    source_table: str = None,  # type: ignore
-    dest_table: str = None,  # type: ignore
-    incremental_key: str = None,  # type: ignore
-    incremental_strategy: str = "replace",  # type: ignore
+    source_uri: Annotated[str, typer.Option(help="The URI of the [green]source[/green]")],  # type: ignore
+    dest_uri: Annotated[str, typer.Option(help="The URI of the [cyan]destination[/cyan]")],  # type: ignore
+    source_table: Annotated[str, typer.Option(help="The table name in the [green]source[/green] to fetch")],  # type: ignore
+    dest_table: Annotated[str, typer.Option(help="The table in the [cyan]destination[/cyan] to save the data into")] = None,  # type: ignore
+    incremental_key: Annotated[str, typer.Option(help="The incremental key from the table to be used for incremental strategies")] = None,  # type: ignore
+    incremental_strategy: Annotated[str, typer.Option(help="The incremental strategy to use, must be one of 'replace', 'append' or 'merge'")] = "replace",  # type: ignore
 ):
-    if not source_uri:
-        typer.echo("Please provide a source URI")
-        raise typer.Abort()
-
-    if not dest_uri:
-        typer.echo("Please provide a destination URI")
-        raise typer.Abort()
-
-    if not source_table:
-        print("[bold red]Please provide a source table [\red bold]")
-        raise typer.Abort()
-
     if not dest_table:
-        typer.echo("Please provide a destination table")
-        raise typer.Abort()
-
+        print()
+        print(
+            "[yellow]Destination table is not given, defaulting to the source table.[/yellow]"
+        )
+        dest_table = source_table
 
     factory = SourceDestinationFactory(source_uri, dest_uri)
     source = factory.get_source()
@@ -43,9 +42,11 @@ def ingest(
 
     m = hashlib.sha256()
     m.update(dest_table.encode("utf-8"))
+    pipeline_name = m.hexdigest()
+    short_pipeline_name = pipeline_name[:8]
 
     pipeline = dlt.pipeline(
-        pipeline_name=m.hexdigest(),
+        pipeline_name=pipeline_name,
         destination=destination.dlt_dest(
             uri=dest_uri,
         ),
@@ -54,7 +55,26 @@ def ingest(
     )
 
     print()
-    print(f"[bold green]Initiated pipeline, starting...[/bold green]")
+    print(f"[bold green]Initiated the pipeline with the following:[/bold green]")
+    print(f"[bold yellow]  Pipeline ID:[/bold yellow] {short_pipeline_name}")
+    print(
+        f"[bold yellow]  Source:[/bold yellow] {factory.source_scheme} / {source_table}"
+    )
+    print(
+        f"[bold yellow]  Destination:[/bold yellow] {factory.destination_scheme} / {dest_table}"
+    )
+    print(f"[bold yellow]  Incremental Strategy:[/bold yellow] {incremental_strategy}")
+    print(
+        f"[bold yellow]  Incremental Key:[/bold yellow] {incremental_key if incremental_key else 'None'}"
+    )
+    print()
+
+    continuePipeline = typer.confirm("Are you sure you would like to continue?")
+    if not continuePipeline:
+        raise typer.Abort()
+
+    print()
+    print(f"[bold green]Starting the ingestion...[/bold green]")
     print()
 
     incremental = []
@@ -76,34 +96,37 @@ def ingest(
         primary_key=incremental,
     )
 
-    print()
-    print(f"[bold green]Successfully finished loading data from '{factory.source_scheme}' to '{factory.destination_scheme}'. [/bold green]")
-    # typer.echo(printLoadInfo(run_info))
+    elapsedHuman = ""
+    if run_info.started_at:
+        elapsed = run_info.finished_at - run_info.started_at
+        elapsedHuman = f"in {humanize.precisedelta(elapsed)}"
+
+    print(
+        f"[bold green]Successfully finished loading data from '{factory.source_scheme}' to '{factory.destination_scheme}' {elapsedHuman} [/bold green]"
+    )
 
+    # printLoadInfo(short_pipeline_name, run_info)
 
-def printLoadInfo(info: LoadInfo):
-    msg = f"Pipeline {info.pipeline.pipeline_name} load step completed in "
+
+def printLoadInfo(short_pipeline_name: str, info: LoadInfo):
     if info.started_at:
         elapsed = info.finished_at - info.started_at
-        msg += humanize.precisedelta(elapsed)
-    else:
-        msg += "---"
-    msg += (
-        f"\n{len(info.loads_ids)} load package(s) were loaded to destination"
-        f" {info.destination_name} and into dataset {info.dataset_name}\n"
-    )
-    if info.staging_name:
-        msg += (
-            f"The {info.staging_name} staging destination used"
-            f" {info.staging_displayable_credentials} location to stage data\n"
+        print(
+            f"  ├── Pipeline {short_pipeline_name} load step completed in [bold green]{humanize.precisedelta(elapsed)}[/bold green]"
         )
 
-    msg += (
-        f"The {info.destination_name} destination used"
-        f" {info.destination_displayable_credentials} location to store data"
+    connector = "└──"
+    if info.staging_name:
+        connector = "├──"
+
+    print(
+        f"  {connector} {len(info.loads_ids)} load package{'s were' if len(info.loads_ids) > 1 else ' was'} loaded to destination [bold cyan]{info.destination_name}[/bold cyan] and into dataset [bold cyan]{info.dataset_name}[/bold cyan]",
+        highlight=False,
     )
-    msg += info._load_packages_asstr(info.load_packages, 0)
-    return msg
+    if info.staging_name:
+        print(
+            f"  └── The [bold cyan]{info.staging_name}[/bold cyan] staging destination used [bold cyan]{info.staging_displayable_credentials}[/bold cyan] location to stage data"
+        )
 
 
 @app.command()

{ingestr-0.0.1 → ingestr-0.0.3}/pyproject.toml

@@ -65,22 +65,27 @@ exclude = ['venv']
 
 
 [build-system]
-requires = ["hatchling"]
+requires = ["hatchling", "hatch-requirements-txt"]
 build-backend = "hatchling.build"
 
 [project]
 name = "ingestr"
-version = "0.0.1"
+version = "0.0.3"
 authors = [
   { name="Burak Karakan", email="burak.karakan@getbruin.com" },
 ]
 description = "ingestr is a command-line application that ingests data from various sources and stores them in any database."
 readme = "README.md"
 requires-python = ">=3.9"
+dynamic = ["dependencies"]
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Topic :: Database"
 ]
 
 [project.urls]
@@ -92,3 +97,6 @@ packages = ["ingestr"]
 
 [project.scripts]
 ingestr = "ingestr.main:main"
+
+[tool.hatch.metadata.hooks.requirements_txt]
+files = ["requirements.txt"]

ingestr-0.0.3/requirements-dev.txt

@@ -0,0 +1,7 @@
+-r requirements.txt
+
+isort==5.13.2
+mypy==1.8.0
+pytest-cov==4.1.0
+pytest==8.0.0
+ruff==0.2.1

{ingestr-0.0.1 → ingestr-0.0.3}/requirements.txt

@@ -1,16 +1,10 @@
 databricks-sql-connector==2.9.3
-deepdiff==6.7.1
 dlt==0.4.3
 duckdb==0.9.2
-isort==5.13.2
-mypy==1.8.0
 pendulum==3.0.0
 psycopg2==2.9.9
 pyodbc==5.1.0
-pytest-cov==4.1.0
-pytest==8.0.0
 rich==13.7.0
-ruff==0.2.1
 snowflake-sqlalchemy==1.5.1
 sqlalchemy-bigquery==1.9.0
 SQLAlchemy==1.4.51

ingestr-0.0.3/resources/demo.tape

@@ -0,0 +1,32 @@
+# Where should we write the GIF?
+Output demo.gif
+
+# Set up a 1200x600 terminal with 46px font.
+Set FontSize 20
+Set Width 1200
+Set Height 900
+Set LineHeight 1.8
+Set TypingSpeed 0.01
+Set Framerate 60
+
+Type "ingestr ingest \"
+Ctrl+Enter
+
+Type "   --source-uri $POSTGRES_URI \"
+Ctrl+Enter
+
+Type "   --source-table 'testschema.table_name' \"
+Ctrl+Enter
+
+Type "   --dest-uri $MSSQL_URI\"
+Ctrl+Enter
+
+Type "   --dest-table 'testschema.table_name'"
+Enter
+
+Sleep 2s
+
+Type "y"
+Ctrl+Enter
+
+Sleep 1s