hadoop-handler 1.2.0__tar.gz → 2.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. {hadoop_handler-1.2.0/hadoop_handler.egg-info → hadoop_handler-2.0.0}/PKG-INFO +50 -17
  2. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/README.md +47 -14
  3. hadoop_handler-2.0.0/hadoop-handler +25 -0
  4. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler/__init__.py +2 -2
  5. hadoop_handler-2.0.0/hadoop_handler/addons/__init__.py +5 -0
  6. hadoop_handler-2.0.0/hadoop_handler/addons/base.py +56 -0
  7. hadoop_handler-2.0.0/hadoop_handler/addons/hive.py +243 -0
  8. hadoop_handler-2.0.0/hadoop_handler/addons/kafka.py +173 -0
  9. hadoop_handler-2.0.0/hadoop_handler/addons/manager.py +173 -0
  10. hadoop_handler-2.0.0/hadoop_handler/addons/spark.py +200 -0
  11. hadoop_handler-2.0.0/hadoop_handler/addons/trino.py +212 -0
  12. hadoop_handler-2.0.0/hadoop_handler/cli.py +312 -0
  13. hadoop_handler-2.0.0/hadoop_handler/core/__init__.py +1 -0
  14. hadoop_handler-2.0.0/hadoop_handler/core/cluster.py +610 -0
  15. hadoop_handler-2.0.0/hadoop_handler/core/config.py +99 -0
  16. hadoop_handler-2.0.0/hadoop_handler/core/downloader.py +107 -0
  17. hadoop_handler-2.0.0/hadoop_handler/core/java.py +346 -0
  18. hadoop_handler-2.0.0/hadoop_handler/core/system.py +423 -0
  19. hadoop_handler-2.0.0/hadoop_handler/core/tui.py +190 -0
  20. hadoop_handler-2.0.0/hadoop_handler/scripts/hadoop-handler +25 -0
  21. hadoop_handler-2.0.0/hadoop_handler/scripts/install.ps1 +18 -0
  22. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0/hadoop_handler.egg-info}/PKG-INFO +50 -17
  23. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler.egg-info/SOURCES.txt +14 -0
  24. hadoop_handler-2.0.0/install.ps1 +18 -0
  25. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/pyproject.toml +7 -2
  26. hadoop_handler-1.2.0/hadoop-handler +0 -1780
  27. hadoop_handler-1.2.0/hadoop_handler/cli.py +0 -138
  28. hadoop_handler-1.2.0/hadoop_handler/scripts/hadoop-handler +0 -1780
  29. hadoop_handler-1.2.0/hadoop_handler/scripts/install.ps1 +0 -495
  30. hadoop_handler-1.2.0/install.ps1 +0 -495
  31. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/LICENSE +0 -0
  32. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/MANIFEST.in +0 -0
  33. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/configs/core-site.xml +0 -0
  34. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/configs/hdfs-site.xml +0 -0
  35. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/configs/mapred-site.xml +0 -0
  36. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/configs/workers +0 -0
  37. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/configs/yarn-site.xml +0 -0
  38. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler/configs/core-site.xml +0 -0
  39. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler/configs/hdfs-site.xml +0 -0
  40. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler/configs/mapred-site.xml +0 -0
  41. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler/configs/workers +0 -0
  42. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler/configs/yarn-site.xml +0 -0
  43. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler.egg-info/dependency_links.txt +0 -0
  44. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler.egg-info/entry_points.txt +0 -0
  45. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler.egg-info/top_level.txt +0 -0
  46. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/install.sh +0 -0
  47. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/setup.cfg +0 -0
  48. {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/setup.sh +0 -0
@@ -1,7 +1,7 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hadoop-handler
3
- Version: 1.2.0
4
- Summary: Interactive cross-platform control panel for Apache Hadoop Single-Node clusters with version switching, Java management, and compatibility matrix checker
3
+ Version: 2.0.0
4
+ Summary: Pure Python cluster control center and ecosystem addon manager for Apache Hadoop, Spark, Hive, Kafka, and Trino
5
5
  Author-email: Aditya Pidurkar <adityapidurkar01@gmail.com>
6
6
  Maintainer-email: Aditya Pidurkar <adityapidurkar01@gmail.com>
7
7
  License-Expression: Apache-2.0
@@ -9,7 +9,7 @@ Project-URL: Homepage, https://github.com/itsadityapidurkar/hadoop-handler
9
9
  Project-URL: Documentation, https://github.com/itsadityapidurkar/hadoop-handler#readme
10
10
  Project-URL: Repository, https://github.com/itsadityapidurkar/hadoop-handler.git
11
11
  Project-URL: Issues, https://github.com/itsadityapidurkar/hadoop-handler/issues
12
- Keywords: hadoop,hdfs,yarn,cluster,automation,cli,big-data,single-node
12
+ Keywords: hadoop,hdfs,yarn,spark,hive,kafka,trino,cluster,automation,cli,big-data,single-node
13
13
  Classifier: Development Status :: 5 - Production/Stable
14
14
  Classifier: Environment :: Console
15
15
  Classifier: Intended Audience :: Developers
@@ -37,7 +37,7 @@ Dynamic: license-file
37
37
 
38
38
  # Hadoop Handler
39
39
 
40
- An interactive, cross-platform command-line tool suite designed to automate the installation, configuration, verification, and management of **Apache Hadoop 3.4.2** Single-Node clusters on **Linux**, **macOS**, and **Windows**.
40
+ An interactive, pure Python cross-platform control center and ecosystem addon manager for **Apache Hadoop**, **Apache Spark**, **Apache Hive**, **Apache Kafka**, and **Trino (PrestoSQL)** on **Linux**, **macOS**, and **Windows**.
41
41
 
42
42
  Developed by **Aditya Pidurkar ([@itsadityap](https://github.com/itsadityapidurkar))**.
43
43
 
@@ -64,7 +64,7 @@ You can also run subcommands and flags directly:
64
64
  hadoop-handler --version
65
65
  hadoop-handler --help
66
66
 
67
- # Direct subcommands
67
+ # Hadoop Cluster Commands
68
68
  hadoop-handler verify # Verify Hadoop, Java, and running daemons
69
69
  hadoop-handler install # Install Apache Hadoop (defaults to 3.4.2)
70
70
  hadoop-handler install --version 3.3.6 # Install specific Hadoop version
@@ -78,6 +78,19 @@ hadoop-handler stop # Stop services
78
78
  hadoop-handler restart # Restart cluster daemons
79
79
  hadoop-handler diagnose # Check and repair dependencies
80
80
  hadoop-handler delete # Wipe and uninstall cluster
81
+
82
+ # Ecosystem Addon Commands (Spark, Hive, Kafka, Trino)
83
+ hadoop-handler addon list # List installed and available ecosystem addons
84
+ hadoop-handler addon install spark # Install Apache Spark with HDFS & YARN integration
85
+ hadoop-handler addon install hive # Install Apache Hive data warehouse with Derby metastore
86
+ hadoop-handler addon install kafka # Install Apache Kafka in lightweight KRaft mode
87
+ hadoop-handler addon install trino # Install Trino SQL query engine with Hive connector
88
+ hadoop-handler addon status spark # Check status and open ports for addon
89
+ hadoop-handler addon start spark # Start addon services
90
+ hadoop-handler addon stop spark # Stop addon services
91
+ hadoop-handler addon remove spark # Remove addon and clean environment
92
+
93
+ # System & Shell Integration
81
94
  hadoop-handler service enable # Register cluster as systemd user service
82
95
  hadoop-handler service status # Check systemd service status
83
96
  hadoop-handler update # In-place self update via git or pip
@@ -118,13 +131,19 @@ chmod +x setup.sh
118
131
 
119
132
  ## Features & Menu Overview
120
133
 
121
- * **Arrow-Key Navigation (TUI Menu):** Navigate menu items effortlessly with `Up` / `Down` arrow keys, Vim keys (`j` / `k`), direct numbers (`0-13`), or `Enter` to select. Zero-flicker in-place terminal redraw.
134
+ * **Pure Python Native Core:** Built completely on the Python 3 standard library with zero third-party pip dependencies. Eliminates shell script divergence across Linux, macOS, and Windows.
135
+ * **Ecosystem Addon Manager:** Automated installation, configuration, daemon supervision, and HDFS/YARN integration for major big data frameworks:
136
+ * **Apache Spark:** Fast in-memory analytics engine with native HDFS classpath and YARN execution.
137
+ * **Apache Hive:** Distributed data warehouse with Derby/HDFS metastore and automatic Guava conflict resolution.
138
+ * **Apache Kafka:** Distributed streaming platform initialized in modern, lightweight KRaft mode (no ZooKeeper needed).
139
+ * **Trino (PrestoSQL):** Interactive distributed SQL engine with built-in Hive connector querying HDFS datasets.
140
+ * **Arrow-Key Navigation (TUI Menu):** Navigate menu items effortlessly with `Up` / `Down` arrow keys, Vim keys (`j` / `k`), direct numbers (`0-14`), or `Enter` to select. Zero-flicker in-place terminal redraw.
122
141
  * **Live Visual Status Badges:** Instant status display at the top of the menu:
123
142
  * Hadoop: `[Installed (3.4.2)]` or `[Not Installed]`
124
143
  * Java: `[OpenJDK 21]` or `[Missing]`
125
144
  * Services: `[5/5 Daemons Running]` or `[Stopped]`
126
145
  * **Dynamic Hadoop Version Selector:** Choose from modern versions (3.4.2, 3.4.1, 3.3.6, 3.3.5, 3.2.4) or provide a custom version during installation with automatic pre-flight compatibility checks.
127
- * **Java Runtime Detection & Switcher:** Detects all installed JDKs across standard system directories (`/usr/lib/jvm/`, macOS JVM paths) and enables instant active JVM switching for Hadoop (`hadoop-handler java list`, `hadoop-handler java switch`, `hadoop-handler java use <ver>`).
146
+ * **Java Runtime Detection & Switcher:** Detects all installed JDKs across standard system directories (`/usr/lib/jvm/`, macOS JVM paths, Windows paths) and enables instant active JVM switching for Hadoop (`hadoop-handler java list`, `hadoop-handler java switch`, `hadoop-handler java use <ver>`).
128
147
  * **Cross-Compatibility Matrix Checker:** Comprehensive matrix checker preventing common JVM reflection and security manager crashes across Hadoop 3.4+, 3.3.x, 3.2.x, and 2.x on Java 8, 11, 17, and 21 (`hadoop-handler check-compat`).
129
148
  * **In-Place Upgrades & Downgrades:** Upgrade or downgrade Hadoop versions in-place (`hadoop-handler upgrade [version]`) while preserving existing XML configurations and protecting HDFS data blocks from reformatting.
130
149
  * **Smooth Download Progress Bar:** Live, single-line progress indicator with byte counter, download rate (MB/s), and ETA calculation.
@@ -140,29 +159,43 @@ chmod +x setup.sh
140
159
 
141
160
  ## Windows Daemon Management
142
161
 
143
- On Windows, open a Command Prompt (CMD) as Administrator and run:
144
- - Start HDFS: `start-dfs.cmd`
145
- - Start YARN: `start-yarn.cmd`
146
- - Stop services: `stop-all.cmd` or close the daemon windows.
162
+ On Windows, open a Command Prompt (CMD) or PowerShell and run:
163
+ - Start HDFS: `hadoop-handler start`
164
+ - Stop services: `hadoop-handler stop`
165
+ - Check status: `hadoop-handler verify`
147
166
 
148
167
  ---
149
168
 
150
169
  ## Repository Structure
151
170
 
152
- - `hadoop_handler/`: The core Python package with cross-platform CLI runner and templates.
153
- - `cli.py`: Platform-aware launcher for Linux, macOS, and Windows.
154
- - `scripts/`: Embedded shell scripts (`hadoop-handler`, `install.ps1`).
171
+ - `hadoop_handler/`: The core Python package.
172
+ - `cli.py`: Platform-agnostic CLI dispatcher and entry point.
173
+ - `core/`: Core cluster engine and system utilities.
174
+ - `config.py`: Cluster paths, platform constants, and default directories.
175
+ - `system.py`: OS operations, service unit generators, notifications, completions.
176
+ - `downloader.py`: Streaming downloader with live progress bar and mirror fallback.
177
+ - `java.py`: Multi-JDK detection, switcher, and compatibility matrix.
178
+ - `cluster.py`: Hadoop installation, in-place upgrade, daemons, and diagnostics.
179
+ - `tui.py`: Zero-flicker arrow-key menu selector and live badges.
180
+ - `addons/`: Big data ecosystem addon suite.
181
+ - `base.py`: Base class defining addon lifecycle.
182
+ - `manager.py`: Registry and interactive management submenu.
183
+ - `spark.py`: Apache Spark integration with HDFS/YARN.
184
+ - `hive.py`: Apache Hive data warehouse and Derby metastore.
185
+ - `kafka.py`: Apache Kafka in KRaft mode.
186
+ - `trino.py`: Trino (PrestoSQL) distributed query engine.
155
187
  - `configs/`: Pre-configured XML templates (`core-site.xml`, `hdfs-site.xml`, `mapred-site.xml`, `yarn-site.xml`, `workers`).
156
188
  - `pyproject.toml`: Modern PEP 517/621 build configuration for PyPI.
157
189
  - `setup.sh`: Global setup script for Unix systems.
158
190
  - `install.sh`: One-liner bootstrap script for Unix systems.
159
- - `install.ps1`: Native PowerShell installer for Windows.
160
- - `configs/`: Fallback cluster templates.
191
+ - `install.ps1`: Native PowerShell launcher for Windows.
161
192
 
162
193
  ---
163
194
 
164
- ## Roadmap / Upcoming Features (TODO)
195
+ ## Roadmap / Completed Features
165
196
 
197
+ - [x] **Pure Python Native Core**: Unified standard-library engine deprecating shell script divergence across Linux, macOS, and Windows.
198
+ - [x] **Ecosystem Addon Manager**: Full lifecycle management (install, start, stop, status, remove) for Apache Spark, Apache Hive, Apache Kafka, and Trino (PrestoSQL).
166
199
  - [x] **Arrow-Key Navigation (TUI Menu)**: Interactive navigation with live cluster status badges.
167
200
  - [x] **Smooth Download Progress Bar**: Single-line progress bar with transfer rate and ETA.
168
201
  - [x] **Desktop Notifications**: Native desktop alerts for task completion.
@@ -1,6 +1,6 @@
1
1
  # Hadoop Handler
2
2
 
3
- An interactive, cross-platform command-line tool suite designed to automate the installation, configuration, verification, and management of **Apache Hadoop 3.4.2** Single-Node clusters on **Linux**, **macOS**, and **Windows**.
3
+ An interactive, pure Python cross-platform control center and ecosystem addon manager for **Apache Hadoop**, **Apache Spark**, **Apache Hive**, **Apache Kafka**, and **Trino (PrestoSQL)** on **Linux**, **macOS**, and **Windows**.
4
4
 
5
5
  Developed by **Aditya Pidurkar ([@itsadityap](https://github.com/itsadityapidurkar))**.
6
6
 
@@ -27,7 +27,7 @@ You can also run subcommands and flags directly:
27
27
  hadoop-handler --version
28
28
  hadoop-handler --help
29
29
 
30
- # Direct subcommands
30
+ # Hadoop Cluster Commands
31
31
  hadoop-handler verify # Verify Hadoop, Java, and running daemons
32
32
  hadoop-handler install # Install Apache Hadoop (defaults to 3.4.2)
33
33
  hadoop-handler install --version 3.3.6 # Install specific Hadoop version
@@ -41,6 +41,19 @@ hadoop-handler stop # Stop services
41
41
  hadoop-handler restart # Restart cluster daemons
42
42
  hadoop-handler diagnose # Check and repair dependencies
43
43
  hadoop-handler delete # Wipe and uninstall cluster
44
+
45
+ # Ecosystem Addon Commands (Spark, Hive, Kafka, Trino)
46
+ hadoop-handler addon list # List installed and available ecosystem addons
47
+ hadoop-handler addon install spark # Install Apache Spark with HDFS & YARN integration
48
+ hadoop-handler addon install hive # Install Apache Hive data warehouse with Derby metastore
49
+ hadoop-handler addon install kafka # Install Apache Kafka in lightweight KRaft mode
50
+ hadoop-handler addon install trino # Install Trino SQL query engine with Hive connector
51
+ hadoop-handler addon status spark # Check status and open ports for addon
52
+ hadoop-handler addon start spark # Start addon services
53
+ hadoop-handler addon stop spark # Stop addon services
54
+ hadoop-handler addon remove spark # Remove addon and clean environment
55
+
56
+ # System & Shell Integration
44
57
  hadoop-handler service enable # Register cluster as systemd user service
45
58
  hadoop-handler service status # Check systemd service status
46
59
  hadoop-handler update # In-place self update via git or pip
@@ -81,13 +94,19 @@ chmod +x setup.sh
81
94
 
82
95
  ## Features & Menu Overview
83
96
 
84
- * **Arrow-Key Navigation (TUI Menu):** Navigate menu items effortlessly with `Up` / `Down` arrow keys, Vim keys (`j` / `k`), direct numbers (`0-13`), or `Enter` to select. Zero-flicker in-place terminal redraw.
97
+ * **Pure Python Native Core:** Built completely on the Python 3 standard library with zero third-party pip dependencies. Eliminates shell script divergence across Linux, macOS, and Windows.
98
+ * **Ecosystem Addon Manager:** Automated installation, configuration, daemon supervision, and HDFS/YARN integration for major big data frameworks:
99
+ * **Apache Spark:** Fast in-memory analytics engine with native HDFS classpath and YARN execution.
100
+ * **Apache Hive:** Distributed data warehouse with Derby/HDFS metastore and automatic Guava conflict resolution.
101
+ * **Apache Kafka:** Distributed streaming platform initialized in modern, lightweight KRaft mode (no ZooKeeper needed).
102
+ * **Trino (PrestoSQL):** Interactive distributed SQL engine with built-in Hive connector querying HDFS datasets.
103
+ * **Arrow-Key Navigation (TUI Menu):** Navigate menu items effortlessly with `Up` / `Down` arrow keys, Vim keys (`j` / `k`), direct numbers (`0-14`), or `Enter` to select. Zero-flicker in-place terminal redraw.
85
104
  * **Live Visual Status Badges:** Instant status display at the top of the menu:
86
105
  * Hadoop: `[Installed (3.4.2)]` or `[Not Installed]`
87
106
  * Java: `[OpenJDK 21]` or `[Missing]`
88
107
  * Services: `[5/5 Daemons Running]` or `[Stopped]`
89
108
  * **Dynamic Hadoop Version Selector:** Choose from modern versions (3.4.2, 3.4.1, 3.3.6, 3.3.5, 3.2.4) or provide a custom version during installation with automatic pre-flight compatibility checks.
90
- * **Java Runtime Detection & Switcher:** Detects all installed JDKs across standard system directories (`/usr/lib/jvm/`, macOS JVM paths) and enables instant active JVM switching for Hadoop (`hadoop-handler java list`, `hadoop-handler java switch`, `hadoop-handler java use <ver>`).
109
+ * **Java Runtime Detection & Switcher:** Detects all installed JDKs across standard system directories (`/usr/lib/jvm/`, macOS JVM paths, Windows paths) and enables instant active JVM switching for Hadoop (`hadoop-handler java list`, `hadoop-handler java switch`, `hadoop-handler java use <ver>`).
91
110
  * **Cross-Compatibility Matrix Checker:** Comprehensive matrix checker preventing common JVM reflection and security manager crashes across Hadoop 3.4+, 3.3.x, 3.2.x, and 2.x on Java 8, 11, 17, and 21 (`hadoop-handler check-compat`).
92
111
  * **In-Place Upgrades & Downgrades:** Upgrade or downgrade Hadoop versions in-place (`hadoop-handler upgrade [version]`) while preserving existing XML configurations and protecting HDFS data blocks from reformatting.
93
112
  * **Smooth Download Progress Bar:** Live, single-line progress indicator with byte counter, download rate (MB/s), and ETA calculation.
@@ -103,29 +122,43 @@ chmod +x setup.sh
103
122
 
104
123
  ## Windows Daemon Management
105
124
 
106
- On Windows, open a Command Prompt (CMD) as Administrator and run:
107
- - Start HDFS: `start-dfs.cmd`
108
- - Start YARN: `start-yarn.cmd`
109
- - Stop services: `stop-all.cmd` or close the daemon windows.
125
+ On Windows, open a Command Prompt (CMD) or PowerShell and run:
126
+ - Start HDFS: `hadoop-handler start`
127
+ - Stop services: `hadoop-handler stop`
128
+ - Check status: `hadoop-handler verify`
110
129
 
111
130
  ---
112
131
 
113
132
  ## Repository Structure
114
133
 
115
- - `hadoop_handler/`: The core Python package with cross-platform CLI runner and templates.
116
- - `cli.py`: Platform-aware launcher for Linux, macOS, and Windows.
117
- - `scripts/`: Embedded shell scripts (`hadoop-handler`, `install.ps1`).
134
+ - `hadoop_handler/`: The core Python package.
135
+ - `cli.py`: Platform-agnostic CLI dispatcher and entry point.
136
+ - `core/`: Core cluster engine and system utilities.
137
+ - `config.py`: Cluster paths, platform constants, and default directories.
138
+ - `system.py`: OS operations, service unit generators, notifications, completions.
139
+ - `downloader.py`: Streaming downloader with live progress bar and mirror fallback.
140
+ - `java.py`: Multi-JDK detection, switcher, and compatibility matrix.
141
+ - `cluster.py`: Hadoop installation, in-place upgrade, daemons, and diagnostics.
142
+ - `tui.py`: Zero-flicker arrow-key menu selector and live badges.
143
+ - `addons/`: Big data ecosystem addon suite.
144
+ - `base.py`: Base class defining addon lifecycle.
145
+ - `manager.py`: Registry and interactive management submenu.
146
+ - `spark.py`: Apache Spark integration with HDFS/YARN.
147
+ - `hive.py`: Apache Hive data warehouse and Derby metastore.
148
+ - `kafka.py`: Apache Kafka in KRaft mode.
149
+ - `trino.py`: Trino (PrestoSQL) distributed query engine.
118
150
  - `configs/`: Pre-configured XML templates (`core-site.xml`, `hdfs-site.xml`, `mapred-site.xml`, `yarn-site.xml`, `workers`).
119
151
  - `pyproject.toml`: Modern PEP 517/621 build configuration for PyPI.
120
152
  - `setup.sh`: Global setup script for Unix systems.
121
153
  - `install.sh`: One-liner bootstrap script for Unix systems.
122
- - `install.ps1`: Native PowerShell installer for Windows.
123
- - `configs/`: Fallback cluster templates.
154
+ - `install.ps1`: Native PowerShell launcher for Windows.
124
155
 
125
156
  ---
126
157
 
127
- ## Roadmap / Upcoming Features (TODO)
158
+ ## Roadmap / Completed Features
128
159
 
160
+ - [x] **Pure Python Native Core**: Unified standard-library engine deprecating shell script divergence across Linux, macOS, and Windows.
161
+ - [x] **Ecosystem Addon Manager**: Full lifecycle management (install, start, stop, status, remove) for Apache Spark, Apache Hive, Apache Kafka, and Trino (PrestoSQL).
129
162
  - [x] **Arrow-Key Navigation (TUI Menu)**: Interactive navigation with live cluster status badges.
130
163
  - [x] **Smooth Download Progress Bar**: Single-line progress bar with transfer rate and ETA.
131
164
  - [x] **Desktop Notifications**: Native desktop alerts for task completion.
@@ -0,0 +1,25 @@
1
+ #!/usr/bin/env bash
2
+ # ==============================================================================
3
+ # Hadoop Handler v2.0.0 — Pure Python Engine Launcher
4
+ # Developer: Aditya Pidurkar (github.com/itsadityapidurkar)
5
+ # ==============================================================================
6
+
7
+ # Portable symlink resolution to find original install directory
8
+ TARGET_FILE="${BASH_SOURCE[0]}"
9
+ while [ -h "$TARGET_FILE" ]; do
10
+ DIR="$(dirname "$TARGET_FILE")"
11
+ TARGET_FILE="$(readlink "$TARGET_FILE")"
12
+ [[ "$TARGET_FILE" != /* ]] && TARGET_FILE="$DIR/$TARGET_FILE"
13
+ done
14
+ SCRIPT_DIR="$(cd -P "$(dirname "$TARGET_FILE")" && pwd)"
15
+
16
+ export PYTHONPATH="${SCRIPT_DIR}:${PYTHONPATH}"
17
+
18
+ if command -v python3 >/dev/null 2>&1; then
19
+ exec python3 -m hadoop_handler.cli "$@"
20
+ elif command -v python >/dev/null 2>&1; then
21
+ exec python -m hadoop_handler.cli "$@"
22
+ else
23
+ echo "[!] Error: Python 3 is required to run hadoop-handler v2.0." >&2
24
+ exit 1
25
+ fi
@@ -1,8 +1,8 @@
1
- """Hadoop Handler — Apache Hadoop Single-Node Control Center.
1
+ """Hadoop Handler — Pure Python Cluster Control Center & Ecosystem Manager.
2
2
 
3
3
  Developer: Aditya Pidurkar (https://github.com/itsadityapidurkar)
4
4
  Licensed under Apache License 2.0.
5
5
  """
6
6
 
7
- __version__ = "1.2.0"
7
+ __version__ = "2.0.0"
8
8
  __author__ = "Aditya Pidurkar"
@@ -0,0 +1,5 @@
1
+ """Ecosystem Addon Manager for Apache Spark, Apache Hive, Apache Kafka, and Trino."""
2
+
3
+ from hadoop_handler.addons.manager import AddonManager
4
+
5
+ __all__ = ["AddonManager"]
@@ -0,0 +1,56 @@
1
+ """Base class defining the standard lifecycle for all ecosystem addons."""
2
+
3
+ import abc
4
+ from pathlib import Path
5
+ from typing import Optional
6
+
7
+
8
+ class BaseAddon(abc.ABC):
9
+ """Abstract base class for all big-data ecosystem addons."""
10
+
11
+ id: str
12
+ name: str
13
+ description: str
14
+ default_version: str
15
+ default_ports: dict[str, int]
16
+ install_dir: Path
17
+
18
+ @abc.abstractmethod
19
+ def is_installed(self) -> bool:
20
+ """Check if addon is installed on the filesystem."""
21
+ pass
22
+
23
+ @abc.abstractmethod
24
+ def get_installed_version(self) -> Optional[str]:
25
+ """Return the version string of the installed addon, or None."""
26
+ pass
27
+
28
+ @abc.abstractmethod
29
+ def is_running(self) -> bool:
30
+ """Check if addon services/daemons are actively running."""
31
+ pass
32
+
33
+ @abc.abstractmethod
34
+ def install(self, version: Optional[str] = None) -> bool:
35
+ """Download, extract, and configure the addon."""
36
+ pass
37
+
38
+ @abc.abstractmethod
39
+ def start(self) -> bool:
40
+ """Start addon services/daemons."""
41
+ pass
42
+
43
+ @abc.abstractmethod
44
+ def stop(self) -> bool:
45
+ """Stop addon services/daemons."""
46
+ pass
47
+
48
+ @abc.abstractmethod
49
+ def status(self) -> dict:
50
+ """Return a structured dictionary with status details."""
51
+ pass
52
+
53
+ @abc.abstractmethod
54
+ def remove(self) -> bool:
55
+ """Stop, remove installation directory, and clean environment variables."""
56
+ pass
@@ -0,0 +1,243 @@
1
+ """Apache Hive addon installer, metastore schema init, and HDFS warehouse integration."""
2
+
3
+ import os
4
+ import shutil
5
+ import subprocess
6
+ import tarfile
7
+ import time
8
+ from pathlib import Path
9
+ from typing import Optional
10
+
11
+ from hadoop_handler.addons.base import BaseAddon
12
+ from hadoop_handler.core.cluster import get_active_daemons, is_port_open
13
+ from hadoop_handler.core.config import (
14
+ BCYAN,
15
+ DIVIDER,
16
+ DOWNLOADS_DIR,
17
+ GRAY,
18
+ GREEN,
19
+ NC,
20
+ RED,
21
+ YELLOW,
22
+ get_default_addon_root,
23
+ get_default_hadoop_home,
24
+ )
25
+ from hadoop_handler.core.downloader import download_file, resolve_apache_url
26
+ from hadoop_handler.core.system import (
27
+ get_active_shell_config,
28
+ register_shell_env,
29
+ send_notification,
30
+ )
31
+
32
+
33
+ class HiveAddon(BaseAddon):
34
+ """Apache Hive data warehouse with Derby/HDFS metastore integration."""
35
+
36
+ id = "hive"
37
+ name = "Apache Hive"
38
+ description = "Distributed SQL data warehouse engine built on top of Apache Hadoop HDFS"
39
+ default_version = "3.1.3"
40
+ default_ports = {"Metastore Thrift": 9083, "HiveServer2 JDBC": 10000, "Web UI": 10002}
41
+
42
+ def __init__(self) -> None:
43
+ self.install_dir = get_default_addon_root() / "hive"
44
+
45
+ def is_installed(self) -> bool:
46
+ return self.install_dir.exists() and (self.install_dir / "bin/hive").exists()
47
+
48
+ def get_installed_version(self) -> Optional[str]:
49
+ if not self.is_installed():
50
+ return None
51
+ hive_bin = self.install_dir / "bin/hive"
52
+ try:
53
+ res = subprocess.run([str(hive_bin), "--version"], capture_output=True, text=True, timeout=5.0)
54
+ out = res.stderr or res.stdout
55
+ for line in out.splitlines():
56
+ if "hive" in line.lower() and "version" in line.lower():
57
+ parts = line.strip().split()
58
+ for p in parts:
59
+ if p.replace(".", "").isdigit():
60
+ return p
61
+ except Exception:
62
+ pass
63
+ return "Installed"
64
+
65
+ def is_running(self) -> bool:
66
+ daemons = get_active_daemons()
67
+ return (
68
+ "RunJar" in daemons
69
+ or is_port_open(9083)
70
+ or is_port_open(10000)
71
+ )
72
+
73
+ def install(self, version: Optional[str] = None) -> bool:
74
+ target_ver = version or self.default_version
75
+ print(DIVIDER)
76
+ print(f" {BCYAN}ACTION: Installing Apache Hive {target_ver}{NC}")
77
+ print(f"{DIVIDER}\n")
78
+
79
+ tar_name = f"apache-hive-{target_ver}-bin.tar.gz"
80
+ tar_path = DOWNLOADS_DIR / tar_name
81
+ download_url = resolve_apache_url("hive", tar_name, f"hive-{target_ver}")
82
+
83
+ print(f" {GRAY}[1/5]{NC} Retrieving Apache Hive {target_ver} archive...")
84
+ if tar_path.exists() and tar_path.stat().st_size > 10000000:
85
+ print(f" {tar_name} already cached in ~/Downloads.")
86
+ else:
87
+ success = download_file(download_url, tar_path, f"Downloading Apache Hive {target_ver}")
88
+ if not success:
89
+ if tar_path.exists():
90
+ tar_path.unlink()
91
+ return False
92
+
93
+ print(f"\n {GRAY}[2/5]{NC} Extracting Hive binaries to {self.install_dir}...")
94
+ if self.install_dir.exists():
95
+ shutil.rmtree(self.install_dir, ignore_errors=True)
96
+
97
+ self.install_dir.parent.mkdir(parents=True, exist_ok=True)
98
+ with tarfile.open(tar_path, "r:gz") as tar:
99
+ tar.extractall(path=self.install_dir.parent)
100
+
101
+ extracted = self.install_dir.parent / f"apache-hive-{target_ver}-bin"
102
+ if extracted.exists() and extracted != self.install_dir:
103
+ extracted.rename(self.install_dir)
104
+
105
+ print(f"\n {GRAY}[3/5]{NC} Resolving Hadoop classpath and Guava dependency conflicts...")
106
+ hadoop_home = get_default_hadoop_home()
107
+ # Fix Guava conflict: Hive 3 ships old Guava 19, Hadoop 3 uses newer Guava 27+
108
+ hive_lib = self.install_dir / "lib"
109
+ for old_guava in hive_lib.glob("guava-19*.jar"):
110
+ old_guava.unlink()
111
+ print(" Removed outdated guava-19 jar from Hive lib.")
112
+
113
+ hadoop_guavas = list((hadoop_home / "share/hadoop/common/lib").glob("guava-*.jar"))
114
+ if hadoop_guavas:
115
+ shutil.copy2(hadoop_guavas[0], hive_lib / hadoop_guavas[0].name)
116
+ print(f" Copied compatible {hadoop_guavas[0].name} from Hadoop.")
117
+
118
+ print(f"\n {GRAY}[4/5]{NC} Configuring hive-site.xml and Derby metastore warehouse...")
119
+ conf_dir = self.install_dir / "conf"
120
+ conf_dir.mkdir(parents=True, exist_ok=True)
121
+
122
+ hive_site = """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
123
+ <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
124
+ <configuration>
125
+ <property>
126
+ <name>javax.jdo.option.ConnectionURL</name>
127
+ <value>jdbc:derby:;databaseName=metastore_db;create=true</value>
128
+ <description>JDBC connect string for a JDBC metastore</description>
129
+ </property>
130
+ <property>
131
+ <name>javax.jdo.option.ConnectionDriverName</name>
132
+ <value>org.apache.derby.jdbc.EmbeddedDriver</value>
133
+ <description>Driver class name for a JDBC metastore</description>
134
+ </property>
135
+ <property>
136
+ <name>hive.metastore.warehouse.dir</name>
137
+ <value>/user/hive/warehouse</value>
138
+ <description>location of default database for the warehouse</description>
139
+ </property>
140
+ <property>
141
+ <name>hive.metastore.uris</name>
142
+ <value>thrift://localhost:9083</value>
143
+ </property>
144
+ <property>
145
+ <name>hive.metastore.schema.verification</name>
146
+ <value>false</value>
147
+ </property>
148
+ </configuration>
149
+ """
150
+ (conf_dir / "hive-site.xml").write_text(hive_site, encoding="utf-8")
151
+
152
+ # Initialize schema if schematool exists
153
+ schematool = self.install_dir / "bin/schematool"
154
+ if schematool.exists():
155
+ print(" Initializing Derby metastore schema...")
156
+ env = os.environ.copy()
157
+ env["HADOOP_HOME"] = str(hadoop_home)
158
+ env["HIVE_HOME"] = str(self.install_dir)
159
+ subprocess.run([str(schematool), "-dbType", "derby", "-initSchema"], env=env, capture_output=True, check=False)
160
+
161
+ print(f"\n {GRAY}[5/5]{NC} Registering HIVE_HOME in shell configuration...")
162
+ register_shell_env({"HIVE_HOME": str(self.install_dir)})
163
+
164
+ print(f"\n {GREEN}[✓] Apache Hive {target_ver} installed and configured successfully!{NC}")
165
+ shell_conf = get_active_shell_config()
166
+ if shell_conf:
167
+ print(f" Run: {YELLOW}source {shell_conf}{NC} to activate environment.")
168
+ send_notification("Hadoop Handler", f"Apache Hive {target_ver} addon installed.")
169
+ return True
170
+
171
+ def start(self) -> bool:
172
+ if not self.is_installed():
173
+ print(f" {RED}[!] Apache Hive is not installed. Run 'hadoop-handler addon install hive' first.{NC}")
174
+ return False
175
+
176
+ print(f" [+] Starting Apache Hive Metastore Service (Thrift on port 9083)...")
177
+ hive_bin = self.install_dir / "bin/hive"
178
+ logs_dir = self.install_dir / "logs"
179
+ logs_dir.mkdir(parents=True, exist_ok=True)
180
+
181
+ env = os.environ.copy()
182
+ env["HADOOP_HOME"] = str(get_default_hadoop_home())
183
+ env["HIVE_HOME"] = str(self.install_dir)
184
+
185
+ log_file = logs_dir / "metastore.log"
186
+ with open(log_file, "a") as f:
187
+ subprocess.Popen([str(hive_bin), "--service", "metastore"], stdout=f, stderr=f, env=env)
188
+
189
+ time.sleep(3)
190
+ if self.is_running():
191
+ print(f" {GREEN}[✓] Apache Hive Metastore started successfully!{NC}")
192
+ print(f" Thrift Endpoint: {CYAN}thrift://localhost:9083{NC}")
193
+ send_notification("Hadoop Handler", "Apache Hive Metastore service started.")
194
+ return True
195
+ else:
196
+ print(f" {YELLOW}[*] Start signal sent. Check {log_file} for startup details.{NC}")
197
+ return False
198
+
199
+ def stop(self) -> bool:
200
+ if not self.is_installed():
201
+ return True
202
+
203
+ print(f" [+] Stopping Apache Hive services...")
204
+ # Kill running metastore or hiveserver2 Java processes
205
+ try:
206
+ res = subprocess.run(["jps", "-l"], capture_output=True, text=True)
207
+ for line in res.stdout.splitlines():
208
+ parts = line.strip().split()
209
+ if len(parts) >= 2 and ("HiveMetaStore" in parts[1] or "HiveServer2" in parts[1]):
210
+ pid = parts[0]
211
+ subprocess.run(["kill", "-9", pid], check=False)
212
+ print(f" Stopped Hive process (PID {pid}).")
213
+ except Exception:
214
+ pass
215
+
216
+ print(f" {GREEN}[✓] Apache Hive stopped.{NC}")
217
+ send_notification("Hadoop Handler", "Apache Hive services stopped.")
218
+ return True
219
+
220
+ def status(self) -> dict:
221
+ installed = self.is_installed()
222
+ version = self.get_installed_version() if installed else None
223
+ running = self.is_running() if installed else False
224
+ return {
225
+ "id": self.id,
226
+ "name": self.name,
227
+ "installed": installed,
228
+ "version": version,
229
+ "running": running,
230
+ "ports": {
231
+ "Metastore Thrift (9083)": is_port_open(9083),
232
+ "HiveServer2 (10000)": is_port_open(10000),
233
+ },
234
+ }
235
+
236
+ def remove(self) -> bool:
237
+ print(f" [+] Removing Apache Hive addon...")
238
+ self.stop()
239
+ if self.install_dir.exists():
240
+ shutil.rmtree(self.install_dir, ignore_errors=True)
241
+ print(f" {GREEN}[✓] Apache Hive removed.{NC}")
242
+ send_notification("Hadoop Handler", "Apache Hive addon removed.")
243
+ return True