hadoop-handler 1.2.0__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hadoop_handler-1.2.0/hadoop_handler.egg-info → hadoop_handler-2.0.0}/PKG-INFO +50 -17
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/README.md +47 -14
- hadoop_handler-2.0.0/hadoop-handler +25 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler/__init__.py +2 -2
- hadoop_handler-2.0.0/hadoop_handler/addons/__init__.py +5 -0
- hadoop_handler-2.0.0/hadoop_handler/addons/base.py +56 -0
- hadoop_handler-2.0.0/hadoop_handler/addons/hive.py +243 -0
- hadoop_handler-2.0.0/hadoop_handler/addons/kafka.py +173 -0
- hadoop_handler-2.0.0/hadoop_handler/addons/manager.py +173 -0
- hadoop_handler-2.0.0/hadoop_handler/addons/spark.py +200 -0
- hadoop_handler-2.0.0/hadoop_handler/addons/trino.py +212 -0
- hadoop_handler-2.0.0/hadoop_handler/cli.py +312 -0
- hadoop_handler-2.0.0/hadoop_handler/core/__init__.py +1 -0
- hadoop_handler-2.0.0/hadoop_handler/core/cluster.py +610 -0
- hadoop_handler-2.0.0/hadoop_handler/core/config.py +99 -0
- hadoop_handler-2.0.0/hadoop_handler/core/downloader.py +107 -0
- hadoop_handler-2.0.0/hadoop_handler/core/java.py +346 -0
- hadoop_handler-2.0.0/hadoop_handler/core/system.py +423 -0
- hadoop_handler-2.0.0/hadoop_handler/core/tui.py +190 -0
- hadoop_handler-2.0.0/hadoop_handler/scripts/hadoop-handler +25 -0
- hadoop_handler-2.0.0/hadoop_handler/scripts/install.ps1 +18 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0/hadoop_handler.egg-info}/PKG-INFO +50 -17
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler.egg-info/SOURCES.txt +14 -0
- hadoop_handler-2.0.0/install.ps1 +18 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/pyproject.toml +7 -2
- hadoop_handler-1.2.0/hadoop-handler +0 -1780
- hadoop_handler-1.2.0/hadoop_handler/cli.py +0 -138
- hadoop_handler-1.2.0/hadoop_handler/scripts/hadoop-handler +0 -1780
- hadoop_handler-1.2.0/hadoop_handler/scripts/install.ps1 +0 -495
- hadoop_handler-1.2.0/install.ps1 +0 -495
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/LICENSE +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/MANIFEST.in +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/configs/core-site.xml +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/configs/hdfs-site.xml +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/configs/mapred-site.xml +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/configs/workers +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/configs/yarn-site.xml +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler/configs/core-site.xml +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler/configs/hdfs-site.xml +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler/configs/mapred-site.xml +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler/configs/workers +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler/configs/yarn-site.xml +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler.egg-info/dependency_links.txt +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler.egg-info/entry_points.txt +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/hadoop_handler.egg-info/top_level.txt +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/install.sh +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/setup.cfg +0 -0
- {hadoop_handler-1.2.0 → hadoop_handler-2.0.0}/setup.sh +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hadoop-handler
|
|
3
|
-
Version:
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: Pure Python cluster control center and ecosystem addon manager for Apache Hadoop, Spark, Hive, Kafka, and Trino
|
|
5
5
|
Author-email: Aditya Pidurkar <adityapidurkar01@gmail.com>
|
|
6
6
|
Maintainer-email: Aditya Pidurkar <adityapidurkar01@gmail.com>
|
|
7
7
|
License-Expression: Apache-2.0
|
|
@@ -9,7 +9,7 @@ Project-URL: Homepage, https://github.com/itsadityapidurkar/hadoop-handler
|
|
|
9
9
|
Project-URL: Documentation, https://github.com/itsadityapidurkar/hadoop-handler#readme
|
|
10
10
|
Project-URL: Repository, https://github.com/itsadityapidurkar/hadoop-handler.git
|
|
11
11
|
Project-URL: Issues, https://github.com/itsadityapidurkar/hadoop-handler/issues
|
|
12
|
-
Keywords: hadoop,hdfs,yarn,cluster,automation,cli,big-data,single-node
|
|
12
|
+
Keywords: hadoop,hdfs,yarn,spark,hive,kafka,trino,cluster,automation,cli,big-data,single-node
|
|
13
13
|
Classifier: Development Status :: 5 - Production/Stable
|
|
14
14
|
Classifier: Environment :: Console
|
|
15
15
|
Classifier: Intended Audience :: Developers
|
|
@@ -37,7 +37,7 @@ Dynamic: license-file
|
|
|
37
37
|
|
|
38
38
|
# Hadoop Handler
|
|
39
39
|
|
|
40
|
-
An interactive, cross-platform
|
|
40
|
+
An interactive, pure Python cross-platform control center and ecosystem addon manager for **Apache Hadoop**, **Apache Spark**, **Apache Hive**, **Apache Kafka**, and **Trino (PrestoSQL)** on **Linux**, **macOS**, and **Windows**.
|
|
41
41
|
|
|
42
42
|
Developed by **Aditya Pidurkar ([@itsadityap](https://github.com/itsadityapidurkar))**.
|
|
43
43
|
|
|
@@ -64,7 +64,7 @@ You can also run subcommands and flags directly:
|
|
|
64
64
|
hadoop-handler --version
|
|
65
65
|
hadoop-handler --help
|
|
66
66
|
|
|
67
|
-
#
|
|
67
|
+
# Hadoop Cluster Commands
|
|
68
68
|
hadoop-handler verify # Verify Hadoop, Java, and running daemons
|
|
69
69
|
hadoop-handler install # Install Apache Hadoop (defaults to 3.4.2)
|
|
70
70
|
hadoop-handler install --version 3.3.6 # Install specific Hadoop version
|
|
@@ -78,6 +78,19 @@ hadoop-handler stop # Stop services
|
|
|
78
78
|
hadoop-handler restart # Restart cluster daemons
|
|
79
79
|
hadoop-handler diagnose # Check and repair dependencies
|
|
80
80
|
hadoop-handler delete # Wipe and uninstall cluster
|
|
81
|
+
|
|
82
|
+
# Ecosystem Addon Commands (Spark, Hive, Kafka, Trino)
|
|
83
|
+
hadoop-handler addon list # List installed and available ecosystem addons
|
|
84
|
+
hadoop-handler addon install spark # Install Apache Spark with HDFS & YARN integration
|
|
85
|
+
hadoop-handler addon install hive # Install Apache Hive data warehouse with Derby metastore
|
|
86
|
+
hadoop-handler addon install kafka # Install Apache Kafka in lightweight KRaft mode
|
|
87
|
+
hadoop-handler addon install trino # Install Trino SQL query engine with Hive connector
|
|
88
|
+
hadoop-handler addon status spark # Check status and open ports for addon
|
|
89
|
+
hadoop-handler addon start spark # Start addon services
|
|
90
|
+
hadoop-handler addon stop spark # Stop addon services
|
|
91
|
+
hadoop-handler addon remove spark # Remove addon and clean environment
|
|
92
|
+
|
|
93
|
+
# System & Shell Integration
|
|
81
94
|
hadoop-handler service enable # Register cluster as systemd user service
|
|
82
95
|
hadoop-handler service status # Check systemd service status
|
|
83
96
|
hadoop-handler update # In-place self update via git or pip
|
|
@@ -118,13 +131,19 @@ chmod +x setup.sh
|
|
|
118
131
|
|
|
119
132
|
## Features & Menu Overview
|
|
120
133
|
|
|
121
|
-
* **
|
|
134
|
+
* **Pure Python Native Core:** Built completely on the Python 3 standard library with zero third-party pip dependencies. Eliminates shell script divergence across Linux, macOS, and Windows.
|
|
135
|
+
* **Ecosystem Addon Manager:** Automated installation, configuration, daemon supervision, and HDFS/YARN integration for major big data frameworks:
|
|
136
|
+
* **Apache Spark:** Fast in-memory analytics engine with native HDFS classpath and YARN execution.
|
|
137
|
+
* **Apache Hive:** Distributed data warehouse with Derby/HDFS metastore and automatic Guava conflict resolution.
|
|
138
|
+
* **Apache Kafka:** Distributed streaming platform initialized in modern, lightweight KRaft mode (no ZooKeeper needed).
|
|
139
|
+
* **Trino (PrestoSQL):** Interactive distributed SQL engine with built-in Hive connector querying HDFS datasets.
|
|
140
|
+
* **Arrow-Key Navigation (TUI Menu):** Navigate menu items effortlessly with `Up` / `Down` arrow keys, Vim keys (`j` / `k`), direct numbers (`0-14`), or `Enter` to select. Zero-flicker in-place terminal redraw.
|
|
122
141
|
* **Live Visual Status Badges:** Instant status display at the top of the menu:
|
|
123
142
|
* Hadoop: `[Installed (3.4.2)]` or `[Not Installed]`
|
|
124
143
|
* Java: `[OpenJDK 21]` or `[Missing]`
|
|
125
144
|
* Services: `[5/5 Daemons Running]` or `[Stopped]`
|
|
126
145
|
* **Dynamic Hadoop Version Selector:** Choose from modern versions (3.4.2, 3.4.1, 3.3.6, 3.3.5, 3.2.4) or provide a custom version during installation with automatic pre-flight compatibility checks.
|
|
127
|
-
* **Java Runtime Detection & Switcher:** Detects all installed JDKs across standard system directories (`/usr/lib/jvm/`, macOS JVM paths) and enables instant active JVM switching for Hadoop (`hadoop-handler java list`, `hadoop-handler java switch`, `hadoop-handler java use <ver>`).
|
|
146
|
+
* **Java Runtime Detection & Switcher:** Detects all installed JDKs across standard system directories (`/usr/lib/jvm/`, macOS JVM paths, Windows paths) and enables instant active JVM switching for Hadoop (`hadoop-handler java list`, `hadoop-handler java switch`, `hadoop-handler java use <ver>`).
|
|
128
147
|
* **Cross-Compatibility Matrix Checker:** Comprehensive matrix checker preventing common JVM reflection and security manager crashes across Hadoop 3.4+, 3.3.x, 3.2.x, and 2.x on Java 8, 11, 17, and 21 (`hadoop-handler check-compat`).
|
|
129
148
|
* **In-Place Upgrades & Downgrades:** Upgrade or downgrade Hadoop versions in-place (`hadoop-handler upgrade [version]`) while preserving existing XML configurations and protecting HDFS data blocks from reformatting.
|
|
130
149
|
* **Smooth Download Progress Bar:** Live, single-line progress indicator with byte counter, download rate (MB/s), and ETA calculation.
|
|
@@ -140,29 +159,43 @@ chmod +x setup.sh
|
|
|
140
159
|
|
|
141
160
|
## Windows Daemon Management
|
|
142
161
|
|
|
143
|
-
On Windows, open a Command Prompt (CMD)
|
|
144
|
-
- Start HDFS: `start
|
|
145
|
-
-
|
|
146
|
-
-
|
|
162
|
+
On Windows, open a Command Prompt (CMD) or PowerShell and run:
|
|
163
|
+
- Start HDFS: `hadoop-handler start`
|
|
164
|
+
- Stop services: `hadoop-handler stop`
|
|
165
|
+
- Check status: `hadoop-handler verify`
|
|
147
166
|
|
|
148
167
|
---
|
|
149
168
|
|
|
150
169
|
## Repository Structure
|
|
151
170
|
|
|
152
|
-
- `hadoop_handler/`: The core Python package
|
|
153
|
-
- `cli.py`: Platform-
|
|
154
|
-
- `
|
|
171
|
+
- `hadoop_handler/`: The core Python package.
|
|
172
|
+
- `cli.py`: Platform-agnostic CLI dispatcher and entry point.
|
|
173
|
+
- `core/`: Core cluster engine and system utilities.
|
|
174
|
+
- `config.py`: Cluster paths, platform constants, and default directories.
|
|
175
|
+
- `system.py`: OS operations, service unit generators, notifications, completions.
|
|
176
|
+
- `downloader.py`: Streaming downloader with live progress bar and mirror fallback.
|
|
177
|
+
- `java.py`: Multi-JDK detection, switcher, and compatibility matrix.
|
|
178
|
+
- `cluster.py`: Hadoop installation, in-place upgrade, daemons, and diagnostics.
|
|
179
|
+
- `tui.py`: Zero-flicker arrow-key menu selector and live badges.
|
|
180
|
+
- `addons/`: Big data ecosystem addon suite.
|
|
181
|
+
- `base.py`: Base class defining addon lifecycle.
|
|
182
|
+
- `manager.py`: Registry and interactive management submenu.
|
|
183
|
+
- `spark.py`: Apache Spark integration with HDFS/YARN.
|
|
184
|
+
- `hive.py`: Apache Hive data warehouse and Derby metastore.
|
|
185
|
+
- `kafka.py`: Apache Kafka in KRaft mode.
|
|
186
|
+
- `trino.py`: Trino (PrestoSQL) distributed query engine.
|
|
155
187
|
- `configs/`: Pre-configured XML templates (`core-site.xml`, `hdfs-site.xml`, `mapred-site.xml`, `yarn-site.xml`, `workers`).
|
|
156
188
|
- `pyproject.toml`: Modern PEP 517/621 build configuration for PyPI.
|
|
157
189
|
- `setup.sh`: Global setup script for Unix systems.
|
|
158
190
|
- `install.sh`: One-liner bootstrap script for Unix systems.
|
|
159
|
-
- `install.ps1`: Native PowerShell
|
|
160
|
-
- `configs/`: Fallback cluster templates.
|
|
191
|
+
- `install.ps1`: Native PowerShell launcher for Windows.
|
|
161
192
|
|
|
162
193
|
---
|
|
163
194
|
|
|
164
|
-
## Roadmap /
|
|
195
|
+
## Roadmap / Completed Features
|
|
165
196
|
|
|
197
|
+
- [x] **Pure Python Native Core**: Unified standard-library engine deprecating shell script divergence across Linux, macOS, and Windows.
|
|
198
|
+
- [x] **Ecosystem Addon Manager**: Full lifecycle management (install, start, stop, status, remove) for Apache Spark, Apache Hive, Apache Kafka, and Trino (PrestoSQL).
|
|
166
199
|
- [x] **Arrow-Key Navigation (TUI Menu)**: Interactive navigation with live cluster status badges.
|
|
167
200
|
- [x] **Smooth Download Progress Bar**: Single-line progress bar with transfer rate and ETA.
|
|
168
201
|
- [x] **Desktop Notifications**: Native desktop alerts for task completion.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Hadoop Handler
|
|
2
2
|
|
|
3
|
-
An interactive, cross-platform
|
|
3
|
+
An interactive, pure Python cross-platform control center and ecosystem addon manager for **Apache Hadoop**, **Apache Spark**, **Apache Hive**, **Apache Kafka**, and **Trino (PrestoSQL)** on **Linux**, **macOS**, and **Windows**.
|
|
4
4
|
|
|
5
5
|
Developed by **Aditya Pidurkar ([@itsadityap](https://github.com/itsadityapidurkar))**.
|
|
6
6
|
|
|
@@ -27,7 +27,7 @@ You can also run subcommands and flags directly:
|
|
|
27
27
|
hadoop-handler --version
|
|
28
28
|
hadoop-handler --help
|
|
29
29
|
|
|
30
|
-
#
|
|
30
|
+
# Hadoop Cluster Commands
|
|
31
31
|
hadoop-handler verify # Verify Hadoop, Java, and running daemons
|
|
32
32
|
hadoop-handler install # Install Apache Hadoop (defaults to 3.4.2)
|
|
33
33
|
hadoop-handler install --version 3.3.6 # Install specific Hadoop version
|
|
@@ -41,6 +41,19 @@ hadoop-handler stop # Stop services
|
|
|
41
41
|
hadoop-handler restart # Restart cluster daemons
|
|
42
42
|
hadoop-handler diagnose # Check and repair dependencies
|
|
43
43
|
hadoop-handler delete # Wipe and uninstall cluster
|
|
44
|
+
|
|
45
|
+
# Ecosystem Addon Commands (Spark, Hive, Kafka, Trino)
|
|
46
|
+
hadoop-handler addon list # List installed and available ecosystem addons
|
|
47
|
+
hadoop-handler addon install spark # Install Apache Spark with HDFS & YARN integration
|
|
48
|
+
hadoop-handler addon install hive # Install Apache Hive data warehouse with Derby metastore
|
|
49
|
+
hadoop-handler addon install kafka # Install Apache Kafka in lightweight KRaft mode
|
|
50
|
+
hadoop-handler addon install trino # Install Trino SQL query engine with Hive connector
|
|
51
|
+
hadoop-handler addon status spark # Check status and open ports for addon
|
|
52
|
+
hadoop-handler addon start spark # Start addon services
|
|
53
|
+
hadoop-handler addon stop spark # Stop addon services
|
|
54
|
+
hadoop-handler addon remove spark # Remove addon and clean environment
|
|
55
|
+
|
|
56
|
+
# System & Shell Integration
|
|
44
57
|
hadoop-handler service enable # Register cluster as systemd user service
|
|
45
58
|
hadoop-handler service status # Check systemd service status
|
|
46
59
|
hadoop-handler update # In-place self update via git or pip
|
|
@@ -81,13 +94,19 @@ chmod +x setup.sh
|
|
|
81
94
|
|
|
82
95
|
## Features & Menu Overview
|
|
83
96
|
|
|
84
|
-
* **
|
|
97
|
+
* **Pure Python Native Core:** Built completely on the Python 3 standard library with zero third-party pip dependencies. Eliminates shell script divergence across Linux, macOS, and Windows.
|
|
98
|
+
* **Ecosystem Addon Manager:** Automated installation, configuration, daemon supervision, and HDFS/YARN integration for major big data frameworks:
|
|
99
|
+
* **Apache Spark:** Fast in-memory analytics engine with native HDFS classpath and YARN execution.
|
|
100
|
+
* **Apache Hive:** Distributed data warehouse with Derby/HDFS metastore and automatic Guava conflict resolution.
|
|
101
|
+
* **Apache Kafka:** Distributed streaming platform initialized in modern, lightweight KRaft mode (no ZooKeeper needed).
|
|
102
|
+
* **Trino (PrestoSQL):** Interactive distributed SQL engine with built-in Hive connector querying HDFS datasets.
|
|
103
|
+
* **Arrow-Key Navigation (TUI Menu):** Navigate menu items effortlessly with `Up` / `Down` arrow keys, Vim keys (`j` / `k`), direct numbers (`0-14`), or `Enter` to select. Zero-flicker in-place terminal redraw.
|
|
85
104
|
* **Live Visual Status Badges:** Instant status display at the top of the menu:
|
|
86
105
|
* Hadoop: `[Installed (3.4.2)]` or `[Not Installed]`
|
|
87
106
|
* Java: `[OpenJDK 21]` or `[Missing]`
|
|
88
107
|
* Services: `[5/5 Daemons Running]` or `[Stopped]`
|
|
89
108
|
* **Dynamic Hadoop Version Selector:** Choose from modern versions (3.4.2, 3.4.1, 3.3.6, 3.3.5, 3.2.4) or provide a custom version during installation with automatic pre-flight compatibility checks.
|
|
90
|
-
* **Java Runtime Detection & Switcher:** Detects all installed JDKs across standard system directories (`/usr/lib/jvm/`, macOS JVM paths) and enables instant active JVM switching for Hadoop (`hadoop-handler java list`, `hadoop-handler java switch`, `hadoop-handler java use <ver>`).
|
|
109
|
+
* **Java Runtime Detection & Switcher:** Detects all installed JDKs across standard system directories (`/usr/lib/jvm/`, macOS JVM paths, Windows paths) and enables instant active JVM switching for Hadoop (`hadoop-handler java list`, `hadoop-handler java switch`, `hadoop-handler java use <ver>`).
|
|
91
110
|
* **Cross-Compatibility Matrix Checker:** Comprehensive matrix checker preventing common JVM reflection and security manager crashes across Hadoop 3.4+, 3.3.x, 3.2.x, and 2.x on Java 8, 11, 17, and 21 (`hadoop-handler check-compat`).
|
|
92
111
|
* **In-Place Upgrades & Downgrades:** Upgrade or downgrade Hadoop versions in-place (`hadoop-handler upgrade [version]`) while preserving existing XML configurations and protecting HDFS data blocks from reformatting.
|
|
93
112
|
* **Smooth Download Progress Bar:** Live, single-line progress indicator with byte counter, download rate (MB/s), and ETA calculation.
|
|
@@ -103,29 +122,43 @@ chmod +x setup.sh
|
|
|
103
122
|
|
|
104
123
|
## Windows Daemon Management
|
|
105
124
|
|
|
106
|
-
On Windows, open a Command Prompt (CMD)
|
|
107
|
-
- Start HDFS: `start
|
|
108
|
-
-
|
|
109
|
-
-
|
|
125
|
+
On Windows, open a Command Prompt (CMD) or PowerShell and run:
|
|
126
|
+
- Start HDFS: `hadoop-handler start`
|
|
127
|
+
- Stop services: `hadoop-handler stop`
|
|
128
|
+
- Check status: `hadoop-handler verify`
|
|
110
129
|
|
|
111
130
|
---
|
|
112
131
|
|
|
113
132
|
## Repository Structure
|
|
114
133
|
|
|
115
|
-
- `hadoop_handler/`: The core Python package
|
|
116
|
-
- `cli.py`: Platform-
|
|
117
|
-
- `
|
|
134
|
+
- `hadoop_handler/`: The core Python package.
|
|
135
|
+
- `cli.py`: Platform-agnostic CLI dispatcher and entry point.
|
|
136
|
+
- `core/`: Core cluster engine and system utilities.
|
|
137
|
+
- `config.py`: Cluster paths, platform constants, and default directories.
|
|
138
|
+
- `system.py`: OS operations, service unit generators, notifications, completions.
|
|
139
|
+
- `downloader.py`: Streaming downloader with live progress bar and mirror fallback.
|
|
140
|
+
- `java.py`: Multi-JDK detection, switcher, and compatibility matrix.
|
|
141
|
+
- `cluster.py`: Hadoop installation, in-place upgrade, daemons, and diagnostics.
|
|
142
|
+
- `tui.py`: Zero-flicker arrow-key menu selector and live badges.
|
|
143
|
+
- `addons/`: Big data ecosystem addon suite.
|
|
144
|
+
- `base.py`: Base class defining addon lifecycle.
|
|
145
|
+
- `manager.py`: Registry and interactive management submenu.
|
|
146
|
+
- `spark.py`: Apache Spark integration with HDFS/YARN.
|
|
147
|
+
- `hive.py`: Apache Hive data warehouse and Derby metastore.
|
|
148
|
+
- `kafka.py`: Apache Kafka in KRaft mode.
|
|
149
|
+
- `trino.py`: Trino (PrestoSQL) distributed query engine.
|
|
118
150
|
- `configs/`: Pre-configured XML templates (`core-site.xml`, `hdfs-site.xml`, `mapred-site.xml`, `yarn-site.xml`, `workers`).
|
|
119
151
|
- `pyproject.toml`: Modern PEP 517/621 build configuration for PyPI.
|
|
120
152
|
- `setup.sh`: Global setup script for Unix systems.
|
|
121
153
|
- `install.sh`: One-liner bootstrap script for Unix systems.
|
|
122
|
-
- `install.ps1`: Native PowerShell
|
|
123
|
-
- `configs/`: Fallback cluster templates.
|
|
154
|
+
- `install.ps1`: Native PowerShell launcher for Windows.
|
|
124
155
|
|
|
125
156
|
---
|
|
126
157
|
|
|
127
|
-
## Roadmap /
|
|
158
|
+
## Roadmap / Completed Features
|
|
128
159
|
|
|
160
|
+
- [x] **Pure Python Native Core**: Unified standard-library engine deprecating shell script divergence across Linux, macOS, and Windows.
|
|
161
|
+
- [x] **Ecosystem Addon Manager**: Full lifecycle management (install, start, stop, status, remove) for Apache Spark, Apache Hive, Apache Kafka, and Trino (PrestoSQL).
|
|
129
162
|
- [x] **Arrow-Key Navigation (TUI Menu)**: Interactive navigation with live cluster status badges.
|
|
130
163
|
- [x] **Smooth Download Progress Bar**: Single-line progress bar with transfer rate and ETA.
|
|
131
164
|
- [x] **Desktop Notifications**: Native desktop alerts for task completion.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# ==============================================================================
|
|
3
|
+
# Hadoop Handler v2.0.0 — Pure Python Engine Launcher
|
|
4
|
+
# Developer: Aditya Pidurkar (github.com/itsadityapidurkar)
|
|
5
|
+
# ==============================================================================
|
|
6
|
+
|
|
7
|
+
# Portable symlink resolution to find original install directory
|
|
8
|
+
TARGET_FILE="${BASH_SOURCE[0]}"
|
|
9
|
+
while [ -h "$TARGET_FILE" ]; do
|
|
10
|
+
DIR="$(dirname "$TARGET_FILE")"
|
|
11
|
+
TARGET_FILE="$(readlink "$TARGET_FILE")"
|
|
12
|
+
[[ "$TARGET_FILE" != /* ]] && TARGET_FILE="$DIR/$TARGET_FILE"
|
|
13
|
+
done
|
|
14
|
+
SCRIPT_DIR="$(cd -P "$(dirname "$TARGET_FILE")" && pwd)"
|
|
15
|
+
|
|
16
|
+
export PYTHONPATH="${SCRIPT_DIR}:${PYTHONPATH}"
|
|
17
|
+
|
|
18
|
+
if command -v python3 >/dev/null 2>&1; then
|
|
19
|
+
exec python3 -m hadoop_handler.cli "$@"
|
|
20
|
+
elif command -v python >/dev/null 2>&1; then
|
|
21
|
+
exec python -m hadoop_handler.cli "$@"
|
|
22
|
+
else
|
|
23
|
+
echo "[!] Error: Python 3 is required to run hadoop-handler v2.0." >&2
|
|
24
|
+
exit 1
|
|
25
|
+
fi
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
"""Hadoop Handler —
|
|
1
|
+
"""Hadoop Handler — Pure Python Cluster Control Center & Ecosystem Manager.
|
|
2
2
|
|
|
3
3
|
Developer: Aditya Pidurkar (https://github.com/itsadityapidurkar)
|
|
4
4
|
Licensed under Apache License 2.0.
|
|
5
5
|
"""
|
|
6
6
|
|
|
7
|
-
__version__ = "
|
|
7
|
+
__version__ = "2.0.0"
|
|
8
8
|
__author__ = "Aditya Pidurkar"
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Base class defining the standard lifecycle for all ecosystem addons."""
|
|
2
|
+
|
|
3
|
+
import abc
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class BaseAddon(abc.ABC):
|
|
9
|
+
"""Abstract base class for all big-data ecosystem addons."""
|
|
10
|
+
|
|
11
|
+
id: str
|
|
12
|
+
name: str
|
|
13
|
+
description: str
|
|
14
|
+
default_version: str
|
|
15
|
+
default_ports: dict[str, int]
|
|
16
|
+
install_dir: Path
|
|
17
|
+
|
|
18
|
+
@abc.abstractmethod
|
|
19
|
+
def is_installed(self) -> bool:
|
|
20
|
+
"""Check if addon is installed on the filesystem."""
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
@abc.abstractmethod
|
|
24
|
+
def get_installed_version(self) -> Optional[str]:
|
|
25
|
+
"""Return the version string of the installed addon, or None."""
|
|
26
|
+
pass
|
|
27
|
+
|
|
28
|
+
@abc.abstractmethod
|
|
29
|
+
def is_running(self) -> bool:
|
|
30
|
+
"""Check if addon services/daemons are actively running."""
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
@abc.abstractmethod
|
|
34
|
+
def install(self, version: Optional[str] = None) -> bool:
|
|
35
|
+
"""Download, extract, and configure the addon."""
|
|
36
|
+
pass
|
|
37
|
+
|
|
38
|
+
@abc.abstractmethod
|
|
39
|
+
def start(self) -> bool:
|
|
40
|
+
"""Start addon services/daemons."""
|
|
41
|
+
pass
|
|
42
|
+
|
|
43
|
+
@abc.abstractmethod
|
|
44
|
+
def stop(self) -> bool:
|
|
45
|
+
"""Stop addon services/daemons."""
|
|
46
|
+
pass
|
|
47
|
+
|
|
48
|
+
@abc.abstractmethod
|
|
49
|
+
def status(self) -> dict:
|
|
50
|
+
"""Return a structured dictionary with status details."""
|
|
51
|
+
pass
|
|
52
|
+
|
|
53
|
+
@abc.abstractmethod
|
|
54
|
+
def remove(self) -> bool:
|
|
55
|
+
"""Stop, remove installation directory, and clean environment variables."""
|
|
56
|
+
pass
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
"""Apache Hive addon installer, metastore schema init, and HDFS warehouse integration."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import shutil
|
|
5
|
+
import subprocess
|
|
6
|
+
import tarfile
|
|
7
|
+
import time
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Optional
|
|
10
|
+
|
|
11
|
+
from hadoop_handler.addons.base import BaseAddon
|
|
12
|
+
from hadoop_handler.core.cluster import get_active_daemons, is_port_open
|
|
13
|
+
from hadoop_handler.core.config import (
|
|
14
|
+
BCYAN,
|
|
15
|
+
DIVIDER,
|
|
16
|
+
DOWNLOADS_DIR,
|
|
17
|
+
GRAY,
|
|
18
|
+
GREEN,
|
|
19
|
+
NC,
|
|
20
|
+
RED,
|
|
21
|
+
YELLOW,
|
|
22
|
+
get_default_addon_root,
|
|
23
|
+
get_default_hadoop_home,
|
|
24
|
+
)
|
|
25
|
+
from hadoop_handler.core.downloader import download_file, resolve_apache_url
|
|
26
|
+
from hadoop_handler.core.system import (
|
|
27
|
+
get_active_shell_config,
|
|
28
|
+
register_shell_env,
|
|
29
|
+
send_notification,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class HiveAddon(BaseAddon):
|
|
34
|
+
"""Apache Hive data warehouse with Derby/HDFS metastore integration."""
|
|
35
|
+
|
|
36
|
+
id = "hive"
|
|
37
|
+
name = "Apache Hive"
|
|
38
|
+
description = "Distributed SQL data warehouse engine built on top of Apache Hadoop HDFS"
|
|
39
|
+
default_version = "3.1.3"
|
|
40
|
+
default_ports = {"Metastore Thrift": 9083, "HiveServer2 JDBC": 10000, "Web UI": 10002}
|
|
41
|
+
|
|
42
|
+
def __init__(self) -> None:
|
|
43
|
+
self.install_dir = get_default_addon_root() / "hive"
|
|
44
|
+
|
|
45
|
+
def is_installed(self) -> bool:
|
|
46
|
+
return self.install_dir.exists() and (self.install_dir / "bin/hive").exists()
|
|
47
|
+
|
|
48
|
+
def get_installed_version(self) -> Optional[str]:
|
|
49
|
+
if not self.is_installed():
|
|
50
|
+
return None
|
|
51
|
+
hive_bin = self.install_dir / "bin/hive"
|
|
52
|
+
try:
|
|
53
|
+
res = subprocess.run([str(hive_bin), "--version"], capture_output=True, text=True, timeout=5.0)
|
|
54
|
+
out = res.stderr or res.stdout
|
|
55
|
+
for line in out.splitlines():
|
|
56
|
+
if "hive" in line.lower() and "version" in line.lower():
|
|
57
|
+
parts = line.strip().split()
|
|
58
|
+
for p in parts:
|
|
59
|
+
if p.replace(".", "").isdigit():
|
|
60
|
+
return p
|
|
61
|
+
except Exception:
|
|
62
|
+
pass
|
|
63
|
+
return "Installed"
|
|
64
|
+
|
|
65
|
+
def is_running(self) -> bool:
|
|
66
|
+
daemons = get_active_daemons()
|
|
67
|
+
return (
|
|
68
|
+
"RunJar" in daemons
|
|
69
|
+
or is_port_open(9083)
|
|
70
|
+
or is_port_open(10000)
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
def install(self, version: Optional[str] = None) -> bool:
|
|
74
|
+
target_ver = version or self.default_version
|
|
75
|
+
print(DIVIDER)
|
|
76
|
+
print(f" {BCYAN}ACTION: Installing Apache Hive {target_ver}{NC}")
|
|
77
|
+
print(f"{DIVIDER}\n")
|
|
78
|
+
|
|
79
|
+
tar_name = f"apache-hive-{target_ver}-bin.tar.gz"
|
|
80
|
+
tar_path = DOWNLOADS_DIR / tar_name
|
|
81
|
+
download_url = resolve_apache_url("hive", tar_name, f"hive-{target_ver}")
|
|
82
|
+
|
|
83
|
+
print(f" {GRAY}[1/5]{NC} Retrieving Apache Hive {target_ver} archive...")
|
|
84
|
+
if tar_path.exists() and tar_path.stat().st_size > 10000000:
|
|
85
|
+
print(f" {tar_name} already cached in ~/Downloads.")
|
|
86
|
+
else:
|
|
87
|
+
success = download_file(download_url, tar_path, f"Downloading Apache Hive {target_ver}")
|
|
88
|
+
if not success:
|
|
89
|
+
if tar_path.exists():
|
|
90
|
+
tar_path.unlink()
|
|
91
|
+
return False
|
|
92
|
+
|
|
93
|
+
print(f"\n {GRAY}[2/5]{NC} Extracting Hive binaries to {self.install_dir}...")
|
|
94
|
+
if self.install_dir.exists():
|
|
95
|
+
shutil.rmtree(self.install_dir, ignore_errors=True)
|
|
96
|
+
|
|
97
|
+
self.install_dir.parent.mkdir(parents=True, exist_ok=True)
|
|
98
|
+
with tarfile.open(tar_path, "r:gz") as tar:
|
|
99
|
+
tar.extractall(path=self.install_dir.parent)
|
|
100
|
+
|
|
101
|
+
extracted = self.install_dir.parent / f"apache-hive-{target_ver}-bin"
|
|
102
|
+
if extracted.exists() and extracted != self.install_dir:
|
|
103
|
+
extracted.rename(self.install_dir)
|
|
104
|
+
|
|
105
|
+
print(f"\n {GRAY}[3/5]{NC} Resolving Hadoop classpath and Guava dependency conflicts...")
|
|
106
|
+
hadoop_home = get_default_hadoop_home()
|
|
107
|
+
# Fix Guava conflict: Hive 3 ships old Guava 19, Hadoop 3 uses newer Guava 27+
|
|
108
|
+
hive_lib = self.install_dir / "lib"
|
|
109
|
+
for old_guava in hive_lib.glob("guava-19*.jar"):
|
|
110
|
+
old_guava.unlink()
|
|
111
|
+
print(" Removed outdated guava-19 jar from Hive lib.")
|
|
112
|
+
|
|
113
|
+
hadoop_guavas = list((hadoop_home / "share/hadoop/common/lib").glob("guava-*.jar"))
|
|
114
|
+
if hadoop_guavas:
|
|
115
|
+
shutil.copy2(hadoop_guavas[0], hive_lib / hadoop_guavas[0].name)
|
|
116
|
+
print(f" Copied compatible {hadoop_guavas[0].name} from Hadoop.")
|
|
117
|
+
|
|
118
|
+
print(f"\n {GRAY}[4/5]{NC} Configuring hive-site.xml and Derby metastore warehouse...")
|
|
119
|
+
conf_dir = self.install_dir / "conf"
|
|
120
|
+
conf_dir.mkdir(parents=True, exist_ok=True)
|
|
121
|
+
|
|
122
|
+
hive_site = """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
|
123
|
+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
|
124
|
+
<configuration>
|
|
125
|
+
<property>
|
|
126
|
+
<name>javax.jdo.option.ConnectionURL</name>
|
|
127
|
+
<value>jdbc:derby:;databaseName=metastore_db;create=true</value>
|
|
128
|
+
<description>JDBC connect string for a JDBC metastore</description>
|
|
129
|
+
</property>
|
|
130
|
+
<property>
|
|
131
|
+
<name>javax.jdo.option.ConnectionDriverName</name>
|
|
132
|
+
<value>org.apache.derby.jdbc.EmbeddedDriver</value>
|
|
133
|
+
<description>Driver class name for a JDBC metastore</description>
|
|
134
|
+
</property>
|
|
135
|
+
<property>
|
|
136
|
+
<name>hive.metastore.warehouse.dir</name>
|
|
137
|
+
<value>/user/hive/warehouse</value>
|
|
138
|
+
<description>location of default database for the warehouse</description>
|
|
139
|
+
</property>
|
|
140
|
+
<property>
|
|
141
|
+
<name>hive.metastore.uris</name>
|
|
142
|
+
<value>thrift://localhost:9083</value>
|
|
143
|
+
</property>
|
|
144
|
+
<property>
|
|
145
|
+
<name>hive.metastore.schema.verification</name>
|
|
146
|
+
<value>false</value>
|
|
147
|
+
</property>
|
|
148
|
+
</configuration>
|
|
149
|
+
"""
|
|
150
|
+
(conf_dir / "hive-site.xml").write_text(hive_site, encoding="utf-8")
|
|
151
|
+
|
|
152
|
+
# Initialize schema if schematool exists
|
|
153
|
+
schematool = self.install_dir / "bin/schematool"
|
|
154
|
+
if schematool.exists():
|
|
155
|
+
print(" Initializing Derby metastore schema...")
|
|
156
|
+
env = os.environ.copy()
|
|
157
|
+
env["HADOOP_HOME"] = str(hadoop_home)
|
|
158
|
+
env["HIVE_HOME"] = str(self.install_dir)
|
|
159
|
+
subprocess.run([str(schematool), "-dbType", "derby", "-initSchema"], env=env, capture_output=True, check=False)
|
|
160
|
+
|
|
161
|
+
print(f"\n {GRAY}[5/5]{NC} Registering HIVE_HOME in shell configuration...")
|
|
162
|
+
register_shell_env({"HIVE_HOME": str(self.install_dir)})
|
|
163
|
+
|
|
164
|
+
print(f"\n {GREEN}[✓] Apache Hive {target_ver} installed and configured successfully!{NC}")
|
|
165
|
+
shell_conf = get_active_shell_config()
|
|
166
|
+
if shell_conf:
|
|
167
|
+
print(f" Run: {YELLOW}source {shell_conf}{NC} to activate environment.")
|
|
168
|
+
send_notification("Hadoop Handler", f"Apache Hive {target_ver} addon installed.")
|
|
169
|
+
return True
|
|
170
|
+
|
|
171
|
+
def start(self) -> bool:
|
|
172
|
+
if not self.is_installed():
|
|
173
|
+
print(f" {RED}[!] Apache Hive is not installed. Run 'hadoop-handler addon install hive' first.{NC}")
|
|
174
|
+
return False
|
|
175
|
+
|
|
176
|
+
print(f" [+] Starting Apache Hive Metastore Service (Thrift on port 9083)...")
|
|
177
|
+
hive_bin = self.install_dir / "bin/hive"
|
|
178
|
+
logs_dir = self.install_dir / "logs"
|
|
179
|
+
logs_dir.mkdir(parents=True, exist_ok=True)
|
|
180
|
+
|
|
181
|
+
env = os.environ.copy()
|
|
182
|
+
env["HADOOP_HOME"] = str(get_default_hadoop_home())
|
|
183
|
+
env["HIVE_HOME"] = str(self.install_dir)
|
|
184
|
+
|
|
185
|
+
log_file = logs_dir / "metastore.log"
|
|
186
|
+
with open(log_file, "a") as f:
|
|
187
|
+
subprocess.Popen([str(hive_bin), "--service", "metastore"], stdout=f, stderr=f, env=env)
|
|
188
|
+
|
|
189
|
+
time.sleep(3)
|
|
190
|
+
if self.is_running():
|
|
191
|
+
print(f" {GREEN}[✓] Apache Hive Metastore started successfully!{NC}")
|
|
192
|
+
print(f" Thrift Endpoint: {CYAN}thrift://localhost:9083{NC}")
|
|
193
|
+
send_notification("Hadoop Handler", "Apache Hive Metastore service started.")
|
|
194
|
+
return True
|
|
195
|
+
else:
|
|
196
|
+
print(f" {YELLOW}[*] Start signal sent. Check {log_file} for startup details.{NC}")
|
|
197
|
+
return False
|
|
198
|
+
|
|
199
|
+
def stop(self) -> bool:
|
|
200
|
+
if not self.is_installed():
|
|
201
|
+
return True
|
|
202
|
+
|
|
203
|
+
print(f" [+] Stopping Apache Hive services...")
|
|
204
|
+
# Kill running metastore or hiveserver2 Java processes
|
|
205
|
+
try:
|
|
206
|
+
res = subprocess.run(["jps", "-l"], capture_output=True, text=True)
|
|
207
|
+
for line in res.stdout.splitlines():
|
|
208
|
+
parts = line.strip().split()
|
|
209
|
+
if len(parts) >= 2 and ("HiveMetaStore" in parts[1] or "HiveServer2" in parts[1]):
|
|
210
|
+
pid = parts[0]
|
|
211
|
+
subprocess.run(["kill", "-9", pid], check=False)
|
|
212
|
+
print(f" Stopped Hive process (PID {pid}).")
|
|
213
|
+
except Exception:
|
|
214
|
+
pass
|
|
215
|
+
|
|
216
|
+
print(f" {GREEN}[✓] Apache Hive stopped.{NC}")
|
|
217
|
+
send_notification("Hadoop Handler", "Apache Hive services stopped.")
|
|
218
|
+
return True
|
|
219
|
+
|
|
220
|
+
def status(self) -> dict:
|
|
221
|
+
installed = self.is_installed()
|
|
222
|
+
version = self.get_installed_version() if installed else None
|
|
223
|
+
running = self.is_running() if installed else False
|
|
224
|
+
return {
|
|
225
|
+
"id": self.id,
|
|
226
|
+
"name": self.name,
|
|
227
|
+
"installed": installed,
|
|
228
|
+
"version": version,
|
|
229
|
+
"running": running,
|
|
230
|
+
"ports": {
|
|
231
|
+
"Metastore Thrift (9083)": is_port_open(9083),
|
|
232
|
+
"HiveServer2 (10000)": is_port_open(10000),
|
|
233
|
+
},
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
def remove(self) -> bool:
|
|
237
|
+
print(f" [+] Removing Apache Hive addon...")
|
|
238
|
+
self.stop()
|
|
239
|
+
if self.install_dir.exists():
|
|
240
|
+
shutil.rmtree(self.install_dir, ignore_errors=True)
|
|
241
|
+
print(f" {GREEN}[✓] Apache Hive removed.{NC}")
|
|
242
|
+
send_notification("Hadoop Handler", "Apache Hive addon removed.")
|
|
243
|
+
return True
|