flask-stream 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flask_stream-1.0.0/LICENSE +21 -0
- flask_stream-1.0.0/PKG-INFO +146 -0
- flask_stream-1.0.0/README.md +103 -0
- flask_stream-1.0.0/flask_stream/__init__.py +3 -0
- flask_stream-1.0.0/flask_stream/blueprint.py +43 -0
- flask_stream-1.0.0/flask_stream/config.py +10 -0
- flask_stream-1.0.0/flask_stream/extension.py +35 -0
- flask_stream-1.0.0/flask_stream/jobs.py +26 -0
- flask_stream-1.0.0/flask_stream/manager.py +13 -0
- flask_stream-1.0.0/flask_stream/providers/__init__.py +0 -0
- flask_stream-1.0.0/flask_stream/providers/base.py +4 -0
- flask_stream-1.0.0/flask_stream/providers/ssh_download.py +147 -0
- flask_stream-1.0.0/flask_stream.egg-info/PKG-INFO +146 -0
- flask_stream-1.0.0/flask_stream.egg-info/SOURCES.txt +19 -0
- flask_stream-1.0.0/flask_stream.egg-info/dependency_links.txt +1 -0
- flask_stream-1.0.0/flask_stream.egg-info/requires.txt +7 -0
- flask_stream-1.0.0/flask_stream.egg-info/top_level.txt +1 -0
- flask_stream-1.0.0/pyproject.toml +33 -0
- flask_stream-1.0.0/setup.cfg +4 -0
- flask_stream-1.0.0/tests/test_basic.py +42 -0
- flask_stream-1.0.0/tests/test_routes.py +75 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Parkteknia and P3r4nD
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: flask-stream
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Flask extension for remote downloads and progress streaming
|
|
5
|
+
Author-email: Pera Andreu <perapublica@gmail.com>
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2026 Parkteknia and P3r4nD
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://github.com/P3r4nD/flask-stream
|
|
29
|
+
Classifier: Framework :: Flask
|
|
30
|
+
Classifier: Programming Language :: Python :: 3
|
|
31
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
32
|
+
Classifier: Operating System :: OS Independent
|
|
33
|
+
Requires-Python: >=3.11
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
License-File: LICENSE
|
|
36
|
+
Requires-Dist: Flask>=3.1.3
|
|
37
|
+
Requires-Dist: paramiko>=4.0
|
|
38
|
+
Provides-Extra: dev
|
|
39
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
40
|
+
Requires-Dist: black; extra == "dev"
|
|
41
|
+
Requires-Dist: isort; extra == "dev"
|
|
42
|
+
Dynamic: license-file
|
|
43
|
+
|
|
44
|
+
# Flask-Stream
|
|
45
|
+
|
|
46
|
+
[](https://pypi.org/project/flask-stream/)
|
|
47
|
+
[](https://www.python.org/)
|
|
48
|
+
[](LICENSE)
|
|
49
|
+

|
|
50
|
+
|
|
51
|
+
**Flask-Stream** is a Flask 3 extension that enables real-time streaming of file downloads from one or multiple remote servers directly in your web application's UI. It is designed to be easy to integrate, flexible, and extensible for future types of streams.
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## Features
|
|
56
|
+
|
|
57
|
+
1. **SSH File Downloads**
|
|
58
|
+
- Connect to remote servers via SSH key authentication.
|
|
59
|
+
- Recursive file listing on remote servers.
|
|
60
|
+
- Real-time file download progress updates.
|
|
61
|
+
|
|
62
|
+
2. **Multiple Servers Support**
|
|
63
|
+
- Configure one or multiple servers in the Flask app.
|
|
64
|
+
- Each server can have its own `remote_base` and download folder.
|
|
65
|
+
- The UI clearly differentiates which server each file comes from.
|
|
66
|
+
|
|
67
|
+
3. **Smart Progress Bars**
|
|
68
|
+
- **Individual file progress:** shows percentage and downloaded size (MB).
|
|
69
|
+
- **Total progress:** tracks the progress over all files in the job.
|
|
70
|
+
- For multiple simultaneous downloads, each active file has its own progress bar and filename.
|
|
71
|
+
|
|
72
|
+
4. **Parallel Downloads (Optional)**
|
|
73
|
+
- Controlled via `STREAM_BULK_DOWNLOAD` and `STREAM_MAX_SIMULTANEOUS`.
|
|
74
|
+
- Downloads multiple files at once respecting the concurrency limit.
|
|
75
|
+
- Useful for large datasets or multiple servers.
|
|
76
|
+
|
|
77
|
+
5. **Ready-to-use HTML/JavaScript UI**
|
|
78
|
+
- Includable download button via `{{ stream_button() }}`.
|
|
79
|
+
- Shows logs, errors, and dynamic progress using Server-Sent Events (SSE).
|
|
80
|
+
- Fully compatible with Bootstrap 5 for responsive styling.
|
|
81
|
+
|
|
82
|
+
6. **Extensible Architecture**
|
|
83
|
+
- Implements a `StreamProvider` base, enabling future stream types (HTTP, FTP, S3, etc.).
|
|
84
|
+
- Supports `init_app()` pattern for multiple Flask apps in the same process.
|
|
85
|
+
|
|
86
|
+
7. **Asynchronous Job Management**
|
|
87
|
+
- Each download job runs in a separate thread.
|
|
88
|
+
- Event queues per job enable real-time UI updates and multiple concurrent jobs.
|
|
89
|
+
|
|
90
|
+
8. **Error Handling**
|
|
91
|
+
- Stream errors reported in UI.
|
|
92
|
+
- Disconnects from server or app are shown as "Server or App disconnected".
|
|
93
|
+
|
|
94
|
+
9. **Centralized Configuration**
|
|
95
|
+
- Configure all parameters via `app.config` or a central config file.
|
|
96
|
+
- Main configuration variables:
|
|
97
|
+
```python
|
|
98
|
+
STREAM_PROVIDER # Currently "ssh"
|
|
99
|
+
STREAM_SERVERS # List of servers with host, user, key, remote_base, name
|
|
100
|
+
STREAM_DOWNLOAD_DIR # Local download directory
|
|
101
|
+
STREAM_BULK_DOWNLOAD # Enable parallel downloads
|
|
102
|
+
STREAM_MAX_SIMULTANEOUS # Max simultaneous downloads
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
10. **Integrated Tests**
|
|
106
|
+
- Unit tests with **pytest** covering routes, events, and basic functionality.
|
|
107
|
+
- Installable in editable mode with `pip install -e .[dev]`.
|
|
108
|
+
|
|
109
|
+
---
|
|
110
|
+
|
|
111
|
+
## Installation
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
# Clone repository
|
|
115
|
+
git clone https://github.com/yourusername/flask-stream.git
|
|
116
|
+
cd flask-stream
|
|
117
|
+
|
|
118
|
+
# Install with dev dependencies
|
|
119
|
+
pip install -e .[dev]
|
|
120
|
+
|
|
121
|
+
## Basic usage
|
|
122
|
+
```python
|
|
123
|
+
from flask import Flask
|
|
124
|
+
from flask_stream import Stream
|
|
125
|
+
|
|
126
|
+
app = Flask(__name__)
|
|
127
|
+
app.config.update({
|
|
128
|
+
"STREAM_SERVERS": [
|
|
129
|
+
{"name": "server1", "host": "example.com", "user": "ubuntu", "key": "~/.ssh/id_rsa", "remote_base": "logs"},
|
|
130
|
+
{"name": "server2", "host": "example2.com", "user": "ubuntu", "key": "~/.ssh/id_rsa", "remote_base": "logs"}
|
|
131
|
+
],
|
|
132
|
+
"STREAM_DOWNLOAD_DIR": "downloads",
|
|
133
|
+
"STREAM_BULK_DOWNLOAD": True,
|
|
134
|
+
"STREAM_MAX_SIMULTANEOUS": 2
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
stream = Stream(app)
|
|
138
|
+
|
|
139
|
+
@app.route("/")
|
|
140
|
+
def index():
|
|
141
|
+
return """
|
|
142
|
+
<h3>Download Logs</h3>
|
|
143
|
+
{{ stream_button() }}
|
|
144
|
+
"""
|
|
145
|
+
```
|
|
146
|
+
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# Flask-Stream
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/flask-stream/)
|
|
4
|
+
[](https://www.python.org/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+

|
|
7
|
+
|
|
8
|
+
**Flask-Stream** is a Flask 3 extension that enables real-time streaming of file downloads from one or multiple remote servers directly in your web application's UI. It is designed to be easy to integrate, flexible, and extensible for future types of streams.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## Features
|
|
13
|
+
|
|
14
|
+
1. **SSH File Downloads**
|
|
15
|
+
- Connect to remote servers via SSH key authentication.
|
|
16
|
+
- Recursive file listing on remote servers.
|
|
17
|
+
- Real-time file download progress updates.
|
|
18
|
+
|
|
19
|
+
2. **Multiple Servers Support**
|
|
20
|
+
- Configure one or multiple servers in the Flask app.
|
|
21
|
+
- Each server can have its own `remote_base` and download folder.
|
|
22
|
+
- The UI clearly differentiates which server each file comes from.
|
|
23
|
+
|
|
24
|
+
3. **Smart Progress Bars**
|
|
25
|
+
- **Individual file progress:** shows percentage and downloaded size (MB).
|
|
26
|
+
- **Total progress:** tracks the progress over all files in the job.
|
|
27
|
+
- For multiple simultaneous downloads, each active file has its own progress bar and filename.
|
|
28
|
+
|
|
29
|
+
4. **Parallel Downloads (Optional)**
|
|
30
|
+
- Controlled via `STREAM_BULK_DOWNLOAD` and `STREAM_MAX_SIMULTANEOUS`.
|
|
31
|
+
- Downloads multiple files at once respecting the concurrency limit.
|
|
32
|
+
- Useful for large datasets or multiple servers.
|
|
33
|
+
|
|
34
|
+
5. **Ready-to-use HTML/JavaScript UI**
|
|
35
|
+
- Includable download button via `{{ stream_button() }}`.
|
|
36
|
+
- Shows logs, errors, and dynamic progress using Server-Sent Events (SSE).
|
|
37
|
+
- Fully compatible with Bootstrap 5 for responsive styling.
|
|
38
|
+
|
|
39
|
+
6. **Extensible Architecture**
|
|
40
|
+
- Implements a `StreamProvider` base, enabling future stream types (HTTP, FTP, S3, etc.).
|
|
41
|
+
- Supports `init_app()` pattern for multiple Flask apps in the same process.
|
|
42
|
+
|
|
43
|
+
7. **Asynchronous Job Management**
|
|
44
|
+
- Each download job runs in a separate thread.
|
|
45
|
+
- Event queues per job enable real-time UI updates and multiple concurrent jobs.
|
|
46
|
+
|
|
47
|
+
8. **Error Handling**
|
|
48
|
+
- Stream errors reported in UI.
|
|
49
|
+
- Disconnects from server or app are shown as "Server or App disconnected".
|
|
50
|
+
|
|
51
|
+
9. **Centralized Configuration**
|
|
52
|
+
- Configure all parameters via `app.config` or a central config file.
|
|
53
|
+
- Main configuration variables:
|
|
54
|
+
```python
|
|
55
|
+
STREAM_PROVIDER # Currently "ssh"
|
|
56
|
+
STREAM_SERVERS # List of servers with host, user, key, remote_base, name
|
|
57
|
+
STREAM_DOWNLOAD_DIR # Local download directory
|
|
58
|
+
STREAM_BULK_DOWNLOAD # Enable parallel downloads
|
|
59
|
+
STREAM_MAX_SIMULTANEOUS # Max simultaneous downloads
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
10. **Integrated Tests**
|
|
63
|
+
- Unit tests with **pytest** covering routes, events, and basic functionality.
|
|
64
|
+
- Installable in editable mode with `pip install -e .[dev]`.
|
|
65
|
+
|
|
66
|
+
---
|
|
67
|
+
|
|
68
|
+
## Installation
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
# Clone repository
|
|
72
|
+
git clone https://github.com/yourusername/flask-stream.git
|
|
73
|
+
cd flask-stream
|
|
74
|
+
|
|
75
|
+
# Install with dev dependencies
|
|
76
|
+
pip install -e .[dev]
|
|
77
|
+
|
|
78
|
+
## Basic usage
|
|
79
|
+
```python
|
|
80
|
+
from flask import Flask
|
|
81
|
+
from flask_stream import Stream
|
|
82
|
+
|
|
83
|
+
app = Flask(__name__)
|
|
84
|
+
app.config.update({
|
|
85
|
+
"STREAM_SERVERS": [
|
|
86
|
+
{"name": "server1", "host": "example.com", "user": "ubuntu", "key": "~/.ssh/id_rsa", "remote_base": "logs"},
|
|
87
|
+
{"name": "server2", "host": "example2.com", "user": "ubuntu", "key": "~/.ssh/id_rsa", "remote_base": "logs"}
|
|
88
|
+
],
|
|
89
|
+
"STREAM_DOWNLOAD_DIR": "downloads",
|
|
90
|
+
"STREAM_BULK_DOWNLOAD": True,
|
|
91
|
+
"STREAM_MAX_SIMULTANEOUS": 2
|
|
92
|
+
})
|
|
93
|
+
|
|
94
|
+
stream = Stream(app)
|
|
95
|
+
|
|
96
|
+
@app.route("/")
|
|
97
|
+
def index():
|
|
98
|
+
return """
|
|
99
|
+
<h3>Download Logs</h3>
|
|
100
|
+
{{ stream_button() }}
|
|
101
|
+
"""
|
|
102
|
+
```
|
|
103
|
+
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
from flask import Blueprint, Response, current_app, jsonify
|
|
2
|
+
import json
|
|
3
|
+
import threading
|
|
4
|
+
from .jobs import create_job, jobs
|
|
5
|
+
|
|
6
|
+
bp = Blueprint(
|
|
7
|
+
"stream",
|
|
8
|
+
__name__,
|
|
9
|
+
url_prefix="/stream",
|
|
10
|
+
template_folder="templates",
|
|
11
|
+
static_folder="static"
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@bp.route("/start", methods=["POST"])
|
|
16
|
+
def start():
|
|
17
|
+
job_id = create_job()
|
|
18
|
+
|
|
19
|
+
provider_name = current_app.config["STREAM_PROVIDER"]
|
|
20
|
+
# use the app.extensions manager
|
|
21
|
+
provider = current_app.extensions["stream"].manager.get(provider_name)
|
|
22
|
+
|
|
23
|
+
thread = threading.Thread(
|
|
24
|
+
target=provider.run,
|
|
25
|
+
args=(current_app._get_current_object(), job_id),
|
|
26
|
+
daemon=True
|
|
27
|
+
)
|
|
28
|
+
thread.start()
|
|
29
|
+
|
|
30
|
+
return jsonify({"job_id": job_id})
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@bp.route("/events/<job_id>")
|
|
34
|
+
def events(job_id):
|
|
35
|
+
def generator():
|
|
36
|
+
q = jobs[job_id]["queue"]
|
|
37
|
+
while True:
|
|
38
|
+
item = q.get()
|
|
39
|
+
yield f"event: {item['event']}\n"
|
|
40
|
+
yield f"data: {json.dumps(item['data'])}\n\n"
|
|
41
|
+
if jobs[job_id]["done"] and q.empty():
|
|
42
|
+
break
|
|
43
|
+
return Response(generator(), mimetype="text/event-stream")
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from flask import render_template
|
|
2
|
+
from markupsafe import Markup
|
|
3
|
+
from .config import DefaultConfig
|
|
4
|
+
from .manager import StreamManager
|
|
5
|
+
from .blueprint import bp
|
|
6
|
+
|
|
7
|
+
class Stream:
|
|
8
|
+
|
|
9
|
+
def __init__(self, app=None):
|
|
10
|
+
|
|
11
|
+
self.manager = StreamManager()
|
|
12
|
+
|
|
13
|
+
if app:
|
|
14
|
+
self.init_app(app)
|
|
15
|
+
|
|
16
|
+
def init_app(self, app):
|
|
17
|
+
|
|
18
|
+
for k, v in DefaultConfig.__dict__.items():
|
|
19
|
+
if k.isupper():
|
|
20
|
+
app.config.setdefault(k, v)
|
|
21
|
+
|
|
22
|
+
app.extensions["stream"] = self
|
|
23
|
+
|
|
24
|
+
app.register_blueprint(bp)
|
|
25
|
+
|
|
26
|
+
app.context_processor(lambda: {
|
|
27
|
+
"stream_button": self.button,
|
|
28
|
+
"stream_config": {
|
|
29
|
+
"bulk": app.config["STREAM_BULK_DOWNLOAD"],
|
|
30
|
+
"max_simultaneous": app.config["STREAM_MAX_SIMULTANEOUS"]
|
|
31
|
+
}
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
def button(self):
|
|
35
|
+
return Markup(render_template("stream_button.html"))
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import uuid
|
|
2
|
+
from queue import Queue
|
|
3
|
+
|
|
4
|
+
jobs = {}
|
|
5
|
+
|
|
6
|
+
def create_job():
|
|
7
|
+
|
|
8
|
+
job_id = str(uuid.uuid4())
|
|
9
|
+
|
|
10
|
+
jobs[job_id] = {
|
|
11
|
+
"queue": Queue(),
|
|
12
|
+
"done": False
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
return job_id
|
|
16
|
+
|
|
17
|
+
def push_event(job_id, event, data):
|
|
18
|
+
|
|
19
|
+
jobs[job_id]["queue"].put({
|
|
20
|
+
"event": event,
|
|
21
|
+
"data": data
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
def finish_job(job_id):
|
|
25
|
+
|
|
26
|
+
jobs[job_id]["done"] = True
|
|
File without changes
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import stat
|
|
3
|
+
import paramiko
|
|
4
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
5
|
+
|
|
6
|
+
from ..jobs import push_event, finish_job
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class SSHDownloadProvider:
|
|
10
|
+
|
|
11
|
+
def connect(self, server):
|
|
12
|
+
|
|
13
|
+
client = paramiko.SSHClient()
|
|
14
|
+
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
15
|
+
|
|
16
|
+
client.connect(
|
|
17
|
+
server["host"],
|
|
18
|
+
port=server.get("port", 22),
|
|
19
|
+
username=server["user"],
|
|
20
|
+
key_filename=os.path.expanduser(server["key"])
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
return client
|
|
24
|
+
|
|
25
|
+
def is_dir(self, entry):
|
|
26
|
+
return stat.S_ISDIR(entry.st_mode)
|
|
27
|
+
|
|
28
|
+
def list_recursive(self, sftp, base):
|
|
29
|
+
|
|
30
|
+
files = []
|
|
31
|
+
|
|
32
|
+
def walk(path, prefix=""):
|
|
33
|
+
|
|
34
|
+
for entry in sftp.listdir_attr(path):
|
|
35
|
+
|
|
36
|
+
name = entry.filename
|
|
37
|
+
|
|
38
|
+
if name in (".", ".."):
|
|
39
|
+
continue
|
|
40
|
+
|
|
41
|
+
full = f"{path}/{name}"
|
|
42
|
+
rel = f"{prefix}{name}"
|
|
43
|
+
|
|
44
|
+
if self.is_dir(entry):
|
|
45
|
+
walk(full, rel + "/")
|
|
46
|
+
else:
|
|
47
|
+
files.append(rel)
|
|
48
|
+
|
|
49
|
+
walk(base)
|
|
50
|
+
|
|
51
|
+
return files
|
|
52
|
+
|
|
53
|
+
def run(self, app, job_id):
|
|
54
|
+
|
|
55
|
+
download_dir = app.config["STREAM_DOWNLOAD_DIR"]
|
|
56
|
+
servers = app.config["STREAM_SERVERS"]
|
|
57
|
+
bulk = app.config.get("STREAM_BULK_DOWNLOAD", False)
|
|
58
|
+
max_sim = app.config.get("STREAM_MAX_SIMULTANEOUS", 2)
|
|
59
|
+
|
|
60
|
+
for server in servers:
|
|
61
|
+
|
|
62
|
+
push_event(job_id, "debug", {
|
|
63
|
+
"msg": f"Connecting {server['name']}",
|
|
64
|
+
"server": server["name"]
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
base = server["remote_base"]
|
|
68
|
+
|
|
69
|
+
# Creamos un client temporal solo para listar archivos
|
|
70
|
+
client_list = self.connect(server)
|
|
71
|
+
sftp_list = client_list.open_sftp()
|
|
72
|
+
files = self.list_recursive(sftp_list, base)
|
|
73
|
+
sftp_list.close()
|
|
74
|
+
client_list.close()
|
|
75
|
+
|
|
76
|
+
total_files = len(files)
|
|
77
|
+
|
|
78
|
+
push_event(job_id, "Batch", {
|
|
79
|
+
"server": server["name"],
|
|
80
|
+
"total": total_files
|
|
81
|
+
})
|
|
82
|
+
|
|
83
|
+
push_event(job_id, "debug", {
|
|
84
|
+
"msg": f"{len(files)} files found",
|
|
85
|
+
"server": server["name"]
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
def download_file(rel):
|
|
89
|
+
|
|
90
|
+
# Each worker creates their own connection
|
|
91
|
+
client = self.connect(server)
|
|
92
|
+
sftp = client.open_sftp()
|
|
93
|
+
|
|
94
|
+
try:
|
|
95
|
+
|
|
96
|
+
remote_path = f"{base}/{rel}"
|
|
97
|
+
local_path = os.path.join(download_dir, server["name"], rel)
|
|
98
|
+
|
|
99
|
+
statinfo = sftp.stat(remote_path)
|
|
100
|
+
size = statinfo.st_size
|
|
101
|
+
|
|
102
|
+
push_event(job_id, "File", {
|
|
103
|
+
"file": rel,
|
|
104
|
+
"size": size,
|
|
105
|
+
"server": server["name"]
|
|
106
|
+
})
|
|
107
|
+
|
|
108
|
+
os.makedirs(os.path.dirname(local_path), exist_ok=True)
|
|
109
|
+
|
|
110
|
+
with sftp.open(remote_path, "rb") as remote_file, open(local_path, "wb") as f:
|
|
111
|
+
|
|
112
|
+
downloaded = 0
|
|
113
|
+
chunk = 32768
|
|
114
|
+
|
|
115
|
+
while True:
|
|
116
|
+
data = remote_file.read(chunk)
|
|
117
|
+
if not data:
|
|
118
|
+
break
|
|
119
|
+
f.write(data)
|
|
120
|
+
downloaded += len(data)
|
|
121
|
+
percent = int(downloaded / size * 100)
|
|
122
|
+
push_event(job_id, "Progress", {
|
|
123
|
+
"percent": percent,
|
|
124
|
+
"file": rel,
|
|
125
|
+
"server": server["name"]
|
|
126
|
+
})
|
|
127
|
+
|
|
128
|
+
push_event(job_id, "FileDone", {
|
|
129
|
+
"file": rel,
|
|
130
|
+
"server": server["name"]
|
|
131
|
+
})
|
|
132
|
+
|
|
133
|
+
finally:
|
|
134
|
+
sftp.close()
|
|
135
|
+
client.close()
|
|
136
|
+
|
|
137
|
+
# Bulk: parallel downloads
|
|
138
|
+
if bulk:
|
|
139
|
+
with ThreadPoolExecutor(max_workers=max_sim) as executor:
|
|
140
|
+
executor.map(download_file, files)
|
|
141
|
+
else:
|
|
142
|
+
# Sequential
|
|
143
|
+
for f in files:
|
|
144
|
+
download_file(f)
|
|
145
|
+
|
|
146
|
+
push_event(job_id, "done", {})
|
|
147
|
+
finish_job(job_id)
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: flask-stream
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Flask extension for remote downloads and progress streaming
|
|
5
|
+
Author-email: Pera Andreu <perapublica@gmail.com>
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2026 Parkteknia and P3r4nD
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://github.com/P3r4nD/flask-stream
|
|
29
|
+
Classifier: Framework :: Flask
|
|
30
|
+
Classifier: Programming Language :: Python :: 3
|
|
31
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
32
|
+
Classifier: Operating System :: OS Independent
|
|
33
|
+
Requires-Python: >=3.11
|
|
34
|
+
Description-Content-Type: text/markdown
|
|
35
|
+
License-File: LICENSE
|
|
36
|
+
Requires-Dist: Flask>=3.1.3
|
|
37
|
+
Requires-Dist: paramiko>=4.0
|
|
38
|
+
Provides-Extra: dev
|
|
39
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
40
|
+
Requires-Dist: black; extra == "dev"
|
|
41
|
+
Requires-Dist: isort; extra == "dev"
|
|
42
|
+
Dynamic: license-file
|
|
43
|
+
|
|
44
|
+
# Flask-Stream
|
|
45
|
+
|
|
46
|
+
[](https://pypi.org/project/flask-stream/)
|
|
47
|
+
[](https://www.python.org/)
|
|
48
|
+
[](LICENSE)
|
|
49
|
+

|
|
50
|
+
|
|
51
|
+
**Flask-Stream** is a Flask 3 extension that enables real-time streaming of file downloads from one or multiple remote servers directly in your web application's UI. It is designed to be easy to integrate, flexible, and extensible for future types of streams.
|
|
52
|
+
|
|
53
|
+
---
|
|
54
|
+
|
|
55
|
+
## Features
|
|
56
|
+
|
|
57
|
+
1. **SSH File Downloads**
|
|
58
|
+
- Connect to remote servers via SSH key authentication.
|
|
59
|
+
- Recursive file listing on remote servers.
|
|
60
|
+
- Real-time file download progress updates.
|
|
61
|
+
|
|
62
|
+
2. **Multiple Servers Support**
|
|
63
|
+
- Configure one or multiple servers in the Flask app.
|
|
64
|
+
- Each server can have its own `remote_base` and download folder.
|
|
65
|
+
- The UI clearly differentiates which server each file comes from.
|
|
66
|
+
|
|
67
|
+
3. **Smart Progress Bars**
|
|
68
|
+
- **Individual file progress:** shows percentage and downloaded size (MB).
|
|
69
|
+
- **Total progress:** tracks the progress over all files in the job.
|
|
70
|
+
- For multiple simultaneous downloads, each active file has its own progress bar and filename.
|
|
71
|
+
|
|
72
|
+
4. **Parallel Downloads (Optional)**
|
|
73
|
+
- Controlled via `STREAM_BULK_DOWNLOAD` and `STREAM_MAX_SIMULTANEOUS`.
|
|
74
|
+
- Downloads multiple files at once respecting the concurrency limit.
|
|
75
|
+
- Useful for large datasets or multiple servers.
|
|
76
|
+
|
|
77
|
+
5. **Ready-to-use HTML/JavaScript UI**
|
|
78
|
+
- Includable download button via `{{ stream_button() }}`.
|
|
79
|
+
- Shows logs, errors, and dynamic progress using Server-Sent Events (SSE).
|
|
80
|
+
- Fully compatible with Bootstrap 5 for responsive styling.
|
|
81
|
+
|
|
82
|
+
6. **Extensible Architecture**
|
|
83
|
+
- Implements a `StreamProvider` base, enabling future stream types (HTTP, FTP, S3, etc.).
|
|
84
|
+
- Supports `init_app()` pattern for multiple Flask apps in the same process.
|
|
85
|
+
|
|
86
|
+
7. **Asynchronous Job Management**
|
|
87
|
+
- Each download job runs in a separate thread.
|
|
88
|
+
- Event queues per job enable real-time UI updates and multiple concurrent jobs.
|
|
89
|
+
|
|
90
|
+
8. **Error Handling**
|
|
91
|
+
- Stream errors reported in UI.
|
|
92
|
+
- Disconnects from server or app are shown as "Server or App disconnected".
|
|
93
|
+
|
|
94
|
+
9. **Centralized Configuration**
|
|
95
|
+
- Configure all parameters via `app.config` or a central config file.
|
|
96
|
+
- Main configuration variables:
|
|
97
|
+
```python
|
|
98
|
+
STREAM_PROVIDER # Currently "ssh"
|
|
99
|
+
STREAM_SERVERS # List of servers with host, user, key, remote_base, name
|
|
100
|
+
STREAM_DOWNLOAD_DIR # Local download directory
|
|
101
|
+
STREAM_BULK_DOWNLOAD # Enable parallel downloads
|
|
102
|
+
STREAM_MAX_SIMULTANEOUS # Max simultaneous downloads
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
10. **Integrated Tests**
|
|
106
|
+
- Unit tests with **pytest** covering routes, events, and basic functionality.
|
|
107
|
+
- Installable in editable mode with `pip install -e .[dev]`.
|
|
108
|
+
|
|
109
|
+
---
|
|
110
|
+
|
|
111
|
+
## Installation
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
# Clone repository
|
|
115
|
+
git clone https://github.com/yourusername/flask-stream.git
|
|
116
|
+
cd flask-stream
|
|
117
|
+
|
|
118
|
+
# Install with dev dependencies
|
|
119
|
+
pip install -e .[dev]
|
|
120
|
+
|
|
121
|
+
## Basic usage
|
|
122
|
+
```python
|
|
123
|
+
from flask import Flask
|
|
124
|
+
from flask_stream import Stream
|
|
125
|
+
|
|
126
|
+
app = Flask(__name__)
|
|
127
|
+
app.config.update({
|
|
128
|
+
"STREAM_SERVERS": [
|
|
129
|
+
{"name": "server1", "host": "example.com", "user": "ubuntu", "key": "~/.ssh/id_rsa", "remote_base": "logs"},
|
|
130
|
+
{"name": "server2", "host": "example2.com", "user": "ubuntu", "key": "~/.ssh/id_rsa", "remote_base": "logs"}
|
|
131
|
+
],
|
|
132
|
+
"STREAM_DOWNLOAD_DIR": "downloads",
|
|
133
|
+
"STREAM_BULK_DOWNLOAD": True,
|
|
134
|
+
"STREAM_MAX_SIMULTANEOUS": 2
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
stream = Stream(app)
|
|
138
|
+
|
|
139
|
+
@app.route("/")
|
|
140
|
+
def index():
|
|
141
|
+
return """
|
|
142
|
+
<h3>Download Logs</h3>
|
|
143
|
+
{{ stream_button() }}
|
|
144
|
+
"""
|
|
145
|
+
```
|
|
146
|
+
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
flask_stream/__init__.py
|
|
5
|
+
flask_stream/blueprint.py
|
|
6
|
+
flask_stream/config.py
|
|
7
|
+
flask_stream/extension.py
|
|
8
|
+
flask_stream/jobs.py
|
|
9
|
+
flask_stream/manager.py
|
|
10
|
+
flask_stream.egg-info/PKG-INFO
|
|
11
|
+
flask_stream.egg-info/SOURCES.txt
|
|
12
|
+
flask_stream.egg-info/dependency_links.txt
|
|
13
|
+
flask_stream.egg-info/requires.txt
|
|
14
|
+
flask_stream.egg-info/top_level.txt
|
|
15
|
+
flask_stream/providers/__init__.py
|
|
16
|
+
flask_stream/providers/base.py
|
|
17
|
+
flask_stream/providers/ssh_download.py
|
|
18
|
+
tests/test_basic.py
|
|
19
|
+
tests/test_routes.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
flask_stream
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=66", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "flask-stream"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "Flask extension for remote downloads and progress streaming"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = {file = "LICENSE"}
|
|
12
|
+
authors = [
|
|
13
|
+
{name = "Pera Andreu", email = "perapublica@gmail.com"}
|
|
14
|
+
]
|
|
15
|
+
urls = { "Homepage" = "https://github.com/P3r4nD/flask-stream" }
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Framework :: Flask",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Operating System :: OS Independent",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
dependencies = [
|
|
24
|
+
"Flask>=3.1.3",
|
|
25
|
+
"paramiko>=4.0",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
dev = [
|
|
30
|
+
"pytest>=7.0",
|
|
31
|
+
"black",
|
|
32
|
+
"isort"
|
|
33
|
+
]
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
from flask import Flask
|
|
2
|
+
from flask_stream import Stream
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def test_init_app():
|
|
6
|
+
|
|
7
|
+
app = Flask(__name__)
|
|
8
|
+
|
|
9
|
+
stream = Stream()
|
|
10
|
+
stream.init_app(app)
|
|
11
|
+
|
|
12
|
+
# the extension is registered
|
|
13
|
+
assert "stream" in app.extensions
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_blueprint_registered():
|
|
17
|
+
|
|
18
|
+
app = Flask(__name__)
|
|
19
|
+
Stream(app)
|
|
20
|
+
|
|
21
|
+
assert "stream" in app.blueprints
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_stream_button_context():
|
|
25
|
+
|
|
26
|
+
app = Flask(__name__)
|
|
27
|
+
Stream(app)
|
|
28
|
+
|
|
29
|
+
with app.app_context():
|
|
30
|
+
|
|
31
|
+
# the context processor should expose stream_button
|
|
32
|
+
ctx = app.template_context_processors[None]
|
|
33
|
+
|
|
34
|
+
found = False
|
|
35
|
+
|
|
36
|
+
for processor in ctx:
|
|
37
|
+
result = processor()
|
|
38
|
+
if "stream_button" in result:
|
|
39
|
+
found = True
|
|
40
|
+
break
|
|
41
|
+
|
|
42
|
+
assert found
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# tests/test_routes.py
|
|
2
|
+
import time
|
|
3
|
+
import json
|
|
4
|
+
import pytest
|
|
5
|
+
from flask import Flask
|
|
6
|
+
from flask_stream import Stream
|
|
7
|
+
from flask_stream.jobs import jobs
|
|
8
|
+
|
|
9
|
+
# ---------- Dummy provider for testing ----------
|
|
10
|
+
class DummyProvider:
|
|
11
|
+
"""Simulate downloads without SSHing"""
|
|
12
|
+
def run(self, app, job_id):
|
|
13
|
+
# We simulated 3 files of different sizes.
|
|
14
|
+
files = [
|
|
15
|
+
{"file": "file1.txt", "size": 1000, "server": "dummy1"},
|
|
16
|
+
{"file": "file2.txt", "size": 2000, "server": "dummy1"},
|
|
17
|
+
{"file": "file3.txt", "size": 1500, "server": "dummy1"},
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
# sent the Batch event
|
|
21
|
+
app.logger.debug("DummyProvider: starting run")
|
|
22
|
+
from flask_stream.jobs import push_event, finish_job
|
|
23
|
+
push_event(job_id, "Batch", {"server": "dummy1", "total": len(files)})
|
|
24
|
+
|
|
25
|
+
for f in files:
|
|
26
|
+
push_event(job_id, "File", f)
|
|
27
|
+
# simulate progress
|
|
28
|
+
for p in range(0, 101, 25):
|
|
29
|
+
push_event(job_id, "Progress", {"percent": p, **f})
|
|
30
|
+
push_event(job_id, "FileDone", f)
|
|
31
|
+
|
|
32
|
+
push_event(job_id, "done", {})
|
|
33
|
+
finish_job(job_id)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# ---------- Fixture for the app ----------
|
|
37
|
+
@pytest.fixture
|
|
38
|
+
def app():
|
|
39
|
+
app = Flask(__name__)
|
|
40
|
+
stream_ext = Stream(app)
|
|
41
|
+
|
|
42
|
+
# We registered DummyProvider in the app manager
|
|
43
|
+
stream_ext.manager.providers["dummy"] = DummyProvider()
|
|
44
|
+
app.config["STREAM_PROVIDER"] = "dummy"
|
|
45
|
+
|
|
46
|
+
yield app
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# ---------- Test /stream/start ----------
|
|
50
|
+
def test_start_route(app):
|
|
51
|
+
client = app.test_client()
|
|
52
|
+
r = client.post("/stream/start")
|
|
53
|
+
assert r.status_code == 200
|
|
54
|
+
data = r.get_json()
|
|
55
|
+
assert "job_id" in data
|
|
56
|
+
job_id = data["job_id"]
|
|
57
|
+
assert job_id in jobs
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# ---------- Test /stream/events/<job_id> ----------
|
|
61
|
+
def test_events_stream(app):
|
|
62
|
+
client = app.test_client()
|
|
63
|
+
r = client.post("/stream/start")
|
|
64
|
+
job_id = r.get_json()["job_id"]
|
|
65
|
+
|
|
66
|
+
# consume the event stream
|
|
67
|
+
r2 = client.get(f"/stream/events/{job_id}", buffered=True)
|
|
68
|
+
assert r2.status_code == 200
|
|
69
|
+
data = r2.data.decode("utf-8")
|
|
70
|
+
|
|
71
|
+
# verify that some Dummy events were received
|
|
72
|
+
assert "event: File" in data
|
|
73
|
+
assert "event: Progress" in data
|
|
74
|
+
assert "event: FileDone" in data
|
|
75
|
+
assert "event: done" in data
|