dataflow-core 2.0.11__tar.gz → 2.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dataflow-core might be problematic. Click here for more details.
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/PKG-INFO +1 -1
- dataflow_core-2.1.1/authenticator/dataflowairflowauthenticator.py +58 -0
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/authenticator/dataflowhubauthenticator.py +13 -19
- dataflow_core-2.1.1/authenticator/dataflowsupersetauthenticator.py +76 -0
- dataflow_core-2.0.11/dataflow/models/database.py → dataflow_core-2.1.1/dataflow/database_manager.py +4 -3
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/dataflow/dataflow.py +46 -9
- dataflow_core-2.1.1/dataflow/db.py +51 -0
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/dataflow/environment.py +1 -1
- dataflow_core-2.1.1/dataflow/models/__init__.py +22 -0
- dataflow_core-2.1.1/dataflow/models/app_types.py +10 -0
- dataflow_core-2.1.1/dataflow/models/blacklist_library.py +26 -0
- dataflow_core-2.1.1/dataflow/models/connection.py +25 -0
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/dataflow/models/environment.py +5 -5
- dataflow_core-2.1.1/dataflow/models/environment_status.py +17 -0
- dataflow_core-2.1.1/dataflow/models/git_ssh.py +18 -0
- dataflow_core-2.1.1/dataflow/models/pinned_projects.py +15 -0
- dataflow_core-2.1.1/dataflow/models/project_details.py +23 -0
- dataflow_core-2.1.1/dataflow/models/recent_project_studio.py +19 -0
- dataflow_core-2.1.1/dataflow/models/recent_projects.py +9 -0
- dataflow_core-2.1.1/dataflow/models/role.py +19 -0
- dataflow_core-2.1.1/dataflow/models/role_server.py +14 -0
- dataflow_core-2.1.1/dataflow/models/runtime.py +11 -0
- dataflow_core-2.1.1/dataflow/models/server_config.py +33 -0
- dataflow_core-2.1.1/dataflow/models/session.py +17 -0
- dataflow_core-2.1.1/dataflow/models/team.py +17 -0
- dataflow_core-2.1.1/dataflow/models/user.py +29 -0
- dataflow_core-2.1.1/dataflow/models/user_environment.py +16 -0
- dataflow_core-2.1.1/dataflow/models/user_team.py +14 -0
- dataflow_core-2.1.1/dataflow/models/variables.py +27 -0
- dataflow_core-2.1.1/dataflow/utils/get_current_user.py +35 -0
- dataflow_core-2.1.1/dataflow/utils/logger.py +41 -0
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/dataflow_core.egg-info/PKG-INFO +1 -1
- dataflow_core-2.1.1/dataflow_core.egg-info/SOURCES.txt +46 -0
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/setup.py +1 -1
- dataflow_core-2.0.11/authenticator/dataflowairflowauthenticator.py +0 -83
- dataflow_core-2.0.11/authenticator/dataflowsupersetauthenticator.py +0 -66
- dataflow_core-2.0.11/authenticator/package/models/__init__.py +0 -0
- dataflow_core-2.0.11/authenticator/package/models/database.py +0 -32
- dataflow_core-2.0.11/authenticator/package/models/session.py +0 -20
- dataflow_core-2.0.11/authenticator/package/models/user.py +0 -22
- dataflow_core-2.0.11/dataflow/configuration.py +0 -27
- dataflow_core-2.0.11/dataflow/models/__init__.py +0 -0
- dataflow_core-2.0.11/dataflow/utils/__init__.py +0 -0
- dataflow_core-2.0.11/dataflow_core.egg-info/SOURCES.txt +0 -30
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/README.md +0 -0
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/authenticator/__init__.py +0 -0
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/dataflow/__init__.py +0 -0
- {dataflow_core-2.0.11/authenticator/package → dataflow_core-2.1.1/dataflow}/configuration.py +0 -0
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/dataflow/scripts/clone_environment.sh +0 -0
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/dataflow/scripts/create_environment.sh +0 -0
- {dataflow_core-2.0.11/authenticator/package → dataflow_core-2.1.1/dataflow/utils}/__init__.py +0 -0
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/dataflow/utils/aws_secrets_manager.py +0 -0
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/dataflow/utils/json_handler.py +0 -0
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/dataflow_core.egg-info/dependency_links.txt +0 -0
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/dataflow_core.egg-info/entry_points.txt +0 -0
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/dataflow_core.egg-info/requires.txt +0 -0
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/dataflow_core.egg-info/top_level.txt +0 -0
- {dataflow_core-2.0.11 → dataflow_core-2.1.1}/setup.cfg +0 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from flask import redirect, request
|
|
2
|
+
from flask_appbuilder.security.views import AuthDBView
|
|
3
|
+
from flask_appbuilder.security.views import expose
|
|
4
|
+
from flask_login import login_user
|
|
5
|
+
from airflow.www.security import FabAirflowSecurityManagerOverride
|
|
6
|
+
from dataflow.dataflow import Dataflow
|
|
7
|
+
import logging
|
|
8
|
+
|
|
9
|
+
logging.basicConfig(
|
|
10
|
+
level=logging.INFO,
|
|
11
|
+
format='%(asctime)s - %(levelname)s - %(message)s',
|
|
12
|
+
datefmt='%Y-%m-%d %H:%M:%S'
|
|
13
|
+
)
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
dataflow = Dataflow()
|
|
17
|
+
|
|
18
|
+
class DataflowAuthDBView(AuthDBView):
|
|
19
|
+
@expose('/login/', methods=['GET', 'POST'])
|
|
20
|
+
def login(self):
|
|
21
|
+
"""
|
|
22
|
+
Override the default login method to handle custom authentication
|
|
23
|
+
"""
|
|
24
|
+
try:
|
|
25
|
+
session_id = request.cookies.get('dataflow_session')
|
|
26
|
+
if not session_id:
|
|
27
|
+
logger.info("No session cookie found, falling back to standard login.")
|
|
28
|
+
return super().login()
|
|
29
|
+
|
|
30
|
+
user_details = dataflow.auth(session_id)
|
|
31
|
+
logger.info(f"User details retrieved for: {user_details['user_name']}")
|
|
32
|
+
user = self.appbuilder.sm.find_user(username=user_details['user_name'])
|
|
33
|
+
if user:
|
|
34
|
+
logger.info(f"User found: {user}")
|
|
35
|
+
login_user(user, remember=False)
|
|
36
|
+
else:
|
|
37
|
+
user = self.appbuilder.sm.add_user(
|
|
38
|
+
username=user_details['user_name'],
|
|
39
|
+
first_name=user_details.get("first_name", ""),
|
|
40
|
+
last_name=user_details.get("last_name", ""),
|
|
41
|
+
email=user_details.get("email", ""),
|
|
42
|
+
role=self.appbuilder.sm.find_role(user_details.get("base_role", "user").title())
|
|
43
|
+
)
|
|
44
|
+
logger.info(f"New user created: {user}")
|
|
45
|
+
if user:
|
|
46
|
+
login_user(user, remember=False)
|
|
47
|
+
|
|
48
|
+
return redirect(self.appbuilder.get_url_for_index)
|
|
49
|
+
|
|
50
|
+
except Exception as e:
|
|
51
|
+
logger.error(f"Login failed: {e}")
|
|
52
|
+
return super().login()
|
|
53
|
+
|
|
54
|
+
class DataflowAirflowAuthenticator(FabAirflowSecurityManagerOverride):
|
|
55
|
+
authdbview = DataflowAuthDBView
|
|
56
|
+
|
|
57
|
+
def __init__(self, appbuilder):
|
|
58
|
+
super().__init__(appbuilder)
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
from .
|
|
2
|
-
from .
|
|
3
|
-
from .package.models import (
|
|
1
|
+
from dataflow.db import get_db
|
|
2
|
+
from dataflow.models import (
|
|
4
3
|
user as m_user,
|
|
5
4
|
session as m_session
|
|
6
5
|
)
|
|
@@ -11,15 +10,7 @@ from jupyterhub.auth import Authenticator
|
|
|
11
10
|
class DataflowHubAuthenticator(Authenticator):
|
|
12
11
|
def __init__(self, **kwargs):
|
|
13
12
|
super().__init__(**kwargs)
|
|
14
|
-
|
|
15
|
-
self.dataflow_config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
|
|
16
|
-
self.database_url = self.dataflow_config.get_config_value('database', 'database_url')
|
|
17
|
-
|
|
18
|
-
self.db_instance = DatabaseManager(self.database_url)
|
|
19
|
-
self.db = next(self.db_instance.get_session())
|
|
20
|
-
|
|
21
|
-
m_user.Base.metadata.create_all(bind=self.db_instance.get_engine())
|
|
22
|
-
m_session.Base.metadata.create_all(bind=self.db_instance.get_engine())
|
|
13
|
+
self.db = next(get_db())
|
|
23
14
|
|
|
24
15
|
def generate_session_id(self):
|
|
25
16
|
return str(uuid.uuid4())
|
|
@@ -39,8 +30,8 @@ class DataflowHubAuthenticator(Authenticator):
|
|
|
39
30
|
|
|
40
31
|
# Check if the user already has an existing session
|
|
41
32
|
existing_session = (
|
|
42
|
-
self.db.query(m_session.
|
|
43
|
-
.filter(m_session.
|
|
33
|
+
self.db.query(m_session.Session)
|
|
34
|
+
.filter(m_session.Session.user_id == user.user_id)
|
|
44
35
|
.first()
|
|
45
36
|
)
|
|
46
37
|
|
|
@@ -50,16 +41,16 @@ class DataflowHubAuthenticator(Authenticator):
|
|
|
50
41
|
else:
|
|
51
42
|
# Generate a new session_id
|
|
52
43
|
session_id = self.generate_session_id()
|
|
53
|
-
query = self.db.query(m_session.
|
|
54
|
-
isSession = query.filter(m_session.
|
|
44
|
+
query = self.db.query(m_session.Session)
|
|
45
|
+
isSession = query.filter(m_session.Session.session_id == session_id).first()
|
|
55
46
|
|
|
56
47
|
# If session_id(uuid string) already exists in the database, generate a new one
|
|
57
48
|
while isSession is not None:
|
|
58
49
|
session_id = self.generate_session_id()
|
|
59
|
-
isSession = query.filter(m_session.
|
|
50
|
+
isSession = query.filter(m_session.Session.session_id == session_id).first()
|
|
60
51
|
|
|
61
52
|
# add session_id to the database
|
|
62
|
-
db_item = m_session.
|
|
53
|
+
db_item = m_session.Session(user_id=user.user_id, session_id=session_id)
|
|
63
54
|
self.db.add(db_item)
|
|
64
55
|
self.db.commit()
|
|
65
56
|
self.db.refresh(db_item)
|
|
@@ -86,4 +77,7 @@ class DataflowHubAuthenticator(Authenticator):
|
|
|
86
77
|
return user_dict
|
|
87
78
|
|
|
88
79
|
except Exception as e:
|
|
89
|
-
return None
|
|
80
|
+
return None
|
|
81
|
+
|
|
82
|
+
finally:
|
|
83
|
+
self.db.close()
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
from flask import redirect, request, jsonify
|
|
2
|
+
from flask_appbuilder.security.views import AuthDBView
|
|
3
|
+
from flask_appbuilder.security.views import expose
|
|
4
|
+
from flask_login import login_user
|
|
5
|
+
from flask_jwt_extended import (
|
|
6
|
+
create_access_token
|
|
7
|
+
)
|
|
8
|
+
from flask_appbuilder.const import (
|
|
9
|
+
API_SECURITY_ACCESS_TOKEN_KEY
|
|
10
|
+
)
|
|
11
|
+
from superset.security import SupersetSecurityManager
|
|
12
|
+
from dataflow.dataflow import Dataflow
|
|
13
|
+
|
|
14
|
+
class DataflowAuthDBView(AuthDBView):
|
|
15
|
+
def __init__(self):
|
|
16
|
+
self.dataflow = Dataflow()
|
|
17
|
+
|
|
18
|
+
def create_user(self, user_details):
|
|
19
|
+
"""
|
|
20
|
+
Create a new user in Superset with the given details.
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
User object: The newly created user instance
|
|
24
|
+
"""
|
|
25
|
+
return self.appbuilder.sm.add_user(
|
|
26
|
+
username=user_details['user_name'],
|
|
27
|
+
first_name=user_details.get("first_name", ""),
|
|
28
|
+
last_name=user_details.get("last_name", ""),
|
|
29
|
+
email=user_details.get("email", ""),
|
|
30
|
+
role=self.appbuilder.sm.find_role('Admin')
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
@expose('/login/', methods=['GET', "POST"])
|
|
34
|
+
def login(self):
|
|
35
|
+
"""
|
|
36
|
+
Handles both GET and POST login requests.
|
|
37
|
+
|
|
38
|
+
- GET: Used for browser-based login. Authenticates using session cookie and redirects to home.
|
|
39
|
+
- POST: Used for API-based login. Returns JWT access token for programmatic access.
|
|
40
|
+
|
|
41
|
+
Returns:
|
|
42
|
+
- Redirect to home page (GET)
|
|
43
|
+
- JSON response with access token (POST)
|
|
44
|
+
"""
|
|
45
|
+
if request.method == "GET":
|
|
46
|
+
session_id = request.cookies.get('dataflow_session')
|
|
47
|
+
user_details = self.dataflow.auth(session_id)
|
|
48
|
+
user = self.appbuilder.sm.find_user(username=user_details['user_name'])
|
|
49
|
+
try:
|
|
50
|
+
if user:
|
|
51
|
+
login_user(user, remember=False)
|
|
52
|
+
else:
|
|
53
|
+
user = self.create_user(user_details)
|
|
54
|
+
login_user(user, remember=False)
|
|
55
|
+
return redirect(self.appbuilder.get_url_for_index)
|
|
56
|
+
except Exception as e:
|
|
57
|
+
return super().login()
|
|
58
|
+
else:
|
|
59
|
+
user_details = request.get_json()
|
|
60
|
+
user = self.appbuilder.sm.find_user(username=user_details["user_name"])
|
|
61
|
+
if user:
|
|
62
|
+
login_user(user, remember=False)
|
|
63
|
+
else:
|
|
64
|
+
user = self.create_user(user_details)
|
|
65
|
+
login_user(user, remember=False)
|
|
66
|
+
resp = {
|
|
67
|
+
API_SECURITY_ACCESS_TOKEN_KEY: create_access_token(
|
|
68
|
+
identity=str(user.id), fresh=True
|
|
69
|
+
)
|
|
70
|
+
}
|
|
71
|
+
return jsonify(resp)
|
|
72
|
+
|
|
73
|
+
class DataflowSecurityManager(SupersetSecurityManager):
|
|
74
|
+
authdbview = DataflowAuthDBView
|
|
75
|
+
def __init__(self, appbuilder):
|
|
76
|
+
super(DataflowSecurityManager, self).__init__(appbuilder)
|
dataflow_core-2.0.11/dataflow/models/database.py → dataflow_core-2.1.1/dataflow/database_manager.py
RENAMED
|
@@ -7,6 +7,7 @@ from sqlalchemy.orm import sessionmaker
|
|
|
7
7
|
class DatabaseManager:
|
|
8
8
|
def __init__(self, db_url):
|
|
9
9
|
self.db_url = db_url
|
|
10
|
+
self.engine = self.get_engine()
|
|
10
11
|
|
|
11
12
|
def get_engine(self):
|
|
12
13
|
try:
|
|
@@ -17,8 +18,8 @@ class DatabaseManager:
|
|
|
17
18
|
|
|
18
19
|
def get_session(self):
|
|
19
20
|
try:
|
|
20
|
-
engine = self.
|
|
21
|
-
session = sessionmaker(autocommit=False, autoflush=False, bind=engine
|
|
21
|
+
engine = self.engine
|
|
22
|
+
session = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
|
22
23
|
db = session()
|
|
23
24
|
try:
|
|
24
25
|
yield db
|
|
@@ -29,4 +30,4 @@ class DatabaseManager:
|
|
|
29
30
|
raise e
|
|
30
31
|
|
|
31
32
|
def get_base(self):
|
|
32
|
-
return declarative_base()
|
|
33
|
+
return declarative_base()
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import os, requests
|
|
2
|
-
from .
|
|
2
|
+
from .database_manager import DatabaseManager
|
|
3
3
|
from .utils.aws_secrets_manager import SecretsManagerClient
|
|
4
4
|
import json
|
|
5
|
-
from
|
|
5
|
+
from .configuration import ConfigurationManager
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
class Dataflow:
|
|
@@ -10,7 +10,15 @@ class Dataflow:
|
|
|
10
10
|
self.secrets_manager = SecretsManagerClient()
|
|
11
11
|
|
|
12
12
|
def auth(self, session_id: str):
|
|
13
|
-
"""
|
|
13
|
+
"""
|
|
14
|
+
Retrieve and return user information using their session ID.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
session_id (str): User's session ID from cookies
|
|
18
|
+
|
|
19
|
+
Returns:
|
|
20
|
+
dict: User information including username, name, email, and role
|
|
21
|
+
"""
|
|
14
22
|
try:
|
|
15
23
|
dataflow_config = ConfigurationManager('/dataflow/app/auth_config/dataflow_auth.cfg')
|
|
16
24
|
auth_api = dataflow_config.get_config_value('auth', 'ui_auth_api')
|
|
@@ -28,7 +36,7 @@ class Dataflow:
|
|
|
28
36
|
"first_name": user_data["first_name"],
|
|
29
37
|
"last_name": user_data["last_name"] if user_data.get("last_name") else "",
|
|
30
38
|
"email": user_data["email"],
|
|
31
|
-
"role": user_data["
|
|
39
|
+
"role": user_data["base_role"]
|
|
32
40
|
}
|
|
33
41
|
return user_dict
|
|
34
42
|
|
|
@@ -37,7 +45,13 @@ class Dataflow:
|
|
|
37
45
|
|
|
38
46
|
def variable(self, variable_name: str):
|
|
39
47
|
"""
|
|
40
|
-
|
|
48
|
+
Retrieve a Dataflow variable.
|
|
49
|
+
|
|
50
|
+
Args:
|
|
51
|
+
variable_name (str): Name of the variable to retrieve
|
|
52
|
+
|
|
53
|
+
Returns:
|
|
54
|
+
str or None: Variable value if found, None otherwise
|
|
41
55
|
"""
|
|
42
56
|
try:
|
|
43
57
|
host_name = os.environ.get("HOSTNAME", "")
|
|
@@ -87,7 +101,15 @@ class Dataflow:
|
|
|
87
101
|
return None
|
|
88
102
|
|
|
89
103
|
def secret(self, secret_name: str):
|
|
90
|
-
"""
|
|
104
|
+
"""
|
|
105
|
+
Retrieve a Dataflow secret value.
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
secret_name (str): Name of the secret to retrieve
|
|
109
|
+
|
|
110
|
+
Returns:
|
|
111
|
+
str or None: Secret value if found, None otherwise
|
|
112
|
+
"""
|
|
91
113
|
try:
|
|
92
114
|
host_name = os.environ.get("HOSTNAME", "")
|
|
93
115
|
user_name = host_name.replace("jupyter-", "") if host_name.startswith("jupyter-") else host_name
|
|
@@ -121,8 +143,17 @@ class Dataflow:
|
|
|
121
143
|
print(f"[Dataflow.secret] Exception occurred: {e}")
|
|
122
144
|
return None
|
|
123
145
|
|
|
124
|
-
def connection(self, conn_id: str):
|
|
125
|
-
"""
|
|
146
|
+
def connection(self, conn_id: str, mode="session"):
|
|
147
|
+
"""
|
|
148
|
+
Connects with a Dataflow connection.
|
|
149
|
+
|
|
150
|
+
Args:
|
|
151
|
+
conn_id (str): Connection identifier
|
|
152
|
+
mode (str): Return type - "session" (default) or "engine" or "url"
|
|
153
|
+
|
|
154
|
+
Returns:
|
|
155
|
+
Session or Engine: SQLAlchemy session or engine based on mode
|
|
156
|
+
"""
|
|
126
157
|
try:
|
|
127
158
|
host_name = os.environ["HOSTNAME"]
|
|
128
159
|
user_name=host_name.replace("jupyter-","")
|
|
@@ -155,8 +186,14 @@ class Dataflow:
|
|
|
155
186
|
# If 'extra' is not valid JSON, skip adding extra parameters
|
|
156
187
|
pass
|
|
157
188
|
|
|
189
|
+
if mode == "url":
|
|
190
|
+
return connection_string
|
|
191
|
+
|
|
158
192
|
connection_instance = DatabaseManager(connection_string)
|
|
159
|
-
|
|
193
|
+
if mode == "engine":
|
|
194
|
+
return connection_instance.get_engine()
|
|
195
|
+
elif mode == "session":
|
|
196
|
+
return next(connection_instance.get_session())
|
|
160
197
|
|
|
161
198
|
except Exception as e:
|
|
162
199
|
return None
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from .database_manager import DatabaseManager
|
|
2
|
+
from .configuration import ConfigurationManager
|
|
3
|
+
from sqlalchemy.orm import declarative_base
|
|
4
|
+
from sqlalchemy.exc import SQLAlchemyError
|
|
5
|
+
from .utils.logger import CustomLogger
|
|
6
|
+
|
|
7
|
+
logger = CustomLogger().get_logger(__name__)
|
|
8
|
+
|
|
9
|
+
dataflow_config = ConfigurationManager('/dataflow/app/config/dataflow.cfg')
|
|
10
|
+
db_url = dataflow_config.get_config_value('database', 'database_url')
|
|
11
|
+
local_db_url = dataflow_config.get_config_value('database', 'single_user_db_url')
|
|
12
|
+
|
|
13
|
+
db_manager = DatabaseManager(db_url)
|
|
14
|
+
local_db_manager = None
|
|
15
|
+
|
|
16
|
+
Base = declarative_base()
|
|
17
|
+
Local_Base = declarative_base()
|
|
18
|
+
|
|
19
|
+
def create_tables(local_db=False):
|
|
20
|
+
"""
|
|
21
|
+
Create all tables in the database.
|
|
22
|
+
This is called at the start of the application.
|
|
23
|
+
"""
|
|
24
|
+
try:
|
|
25
|
+
if local_db:
|
|
26
|
+
global local_db_manager
|
|
27
|
+
if local_db_manager is None:
|
|
28
|
+
local_db_manager = DatabaseManager(local_db_url)
|
|
29
|
+
Local_Base.metadata.create_all(bind=local_db_manager.get_engine())
|
|
30
|
+
else:
|
|
31
|
+
Base.metadata.create_all(bind=db_manager.get_engine())
|
|
32
|
+
logger.info("Database tables created successfully")
|
|
33
|
+
except SQLAlchemyError as e:
|
|
34
|
+
error_message = f"Failed to create tables: {str(e)}"
|
|
35
|
+
logger.error(error_message)
|
|
36
|
+
raise e
|
|
37
|
+
|
|
38
|
+
def get_local_db():
|
|
39
|
+
"""
|
|
40
|
+
Get a local database session.
|
|
41
|
+
|
|
42
|
+
Yields:
|
|
43
|
+
Session: Local database session
|
|
44
|
+
"""
|
|
45
|
+
global local_db_manager
|
|
46
|
+
if local_db_manager is None:
|
|
47
|
+
local_db_manager = DatabaseManager(local_db_url)
|
|
48
|
+
yield from local_db_manager.get_session()
|
|
49
|
+
|
|
50
|
+
def get_db():
|
|
51
|
+
yield from db_manager.get_session()
|
|
@@ -2,7 +2,7 @@ import os, shutil, subprocess, datetime
|
|
|
2
2
|
from .models.environment import JobLogs, Environment
|
|
3
3
|
import json, asyncio, pkg_resources
|
|
4
4
|
from sqlalchemy.orm import Session
|
|
5
|
-
from
|
|
5
|
+
from .configuration import ConfigurationManager
|
|
6
6
|
|
|
7
7
|
class EnvironmentManager:
|
|
8
8
|
def __init__(self):
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# init for loading models in the application
|
|
2
|
+
|
|
3
|
+
from .role import Role
|
|
4
|
+
from .user import User
|
|
5
|
+
from .team import Team
|
|
6
|
+
from .environment import Environment
|
|
7
|
+
from .project_details import ProjectDetails
|
|
8
|
+
from .recent_projects import RecentProjects
|
|
9
|
+
from .pinned_projects import PinnedProject
|
|
10
|
+
from .app_types import AppType
|
|
11
|
+
from .blacklist_library import BlacklistedLibrary
|
|
12
|
+
from .environment_status import EnvironmentStatus
|
|
13
|
+
from .session import Session
|
|
14
|
+
from .server_config import ServerConfig
|
|
15
|
+
from .runtime import RuntimeZone
|
|
16
|
+
from .environment_status import EnvironmentStatus
|
|
17
|
+
from .user_team import UserTeam
|
|
18
|
+
from .role_server import RoleServer
|
|
19
|
+
from .variables import Variable
|
|
20
|
+
from .recent_project_studio import RecentProjectStudio
|
|
21
|
+
from .connection import Connection
|
|
22
|
+
from .git_ssh import GitSSH
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from sqlalchemy import Column, Integer, String, Boolean
|
|
2
|
+
from dataflow.db import Base
|
|
3
|
+
|
|
4
|
+
class AppType(Base):
|
|
5
|
+
__tablename__ = "RUNTIME_APP_TYPE"
|
|
6
|
+
|
|
7
|
+
id = Column(Integer, primary_key=True, autoincrement=True, unique=True)
|
|
8
|
+
name = Column(String, unique=True, nullable=True)
|
|
9
|
+
display_name = Column(String, nullable=False)
|
|
10
|
+
code_based = Column(Boolean, nullable=False)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# models/blacklist_library.py
|
|
2
|
+
from sqlalchemy import Column, Integer, String, UniqueConstraint
|
|
3
|
+
from dataflow.db import Base
|
|
4
|
+
|
|
5
|
+
class BlacklistedLibrary(Base):
|
|
6
|
+
"""
|
|
7
|
+
BlacklistedLibrary model represents a table for storing blacklisted libraries with their versions.
|
|
8
|
+
|
|
9
|
+
Attributes:
|
|
10
|
+
id (int): Primary key of the table, auto-incremented.
|
|
11
|
+
library_name (str): The name of the blacklisted library.
|
|
12
|
+
version (str): The version of the blacklisted library.
|
|
13
|
+
|
|
14
|
+
Unique constraint to ensure the combination of library_name and version is unique.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
__tablename__ = "BLACKLISTED_LIBRARY"
|
|
18
|
+
|
|
19
|
+
id = Column(Integer, primary_key=True, index=True, doc="Primary key for the library.")
|
|
20
|
+
library_name = Column(String, index=True, doc="The name of the blacklisted library.")
|
|
21
|
+
version = Column(String, doc="The version of the blacklisted library.")
|
|
22
|
+
|
|
23
|
+
__table_args__ = (
|
|
24
|
+
UniqueConstraint('library_name', 'version', name='uq_library_version'),
|
|
25
|
+
)
|
|
26
|
+
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from sqlalchemy import Column, String, Integer, Boolean, DateTime, UniqueConstraint
|
|
2
|
+
from sqlalchemy.sql import func
|
|
3
|
+
from dataflow.db import Base
|
|
4
|
+
|
|
5
|
+
class Connection(Base):
|
|
6
|
+
"""
|
|
7
|
+
Database model for storing non-sensitive connection metadata
|
|
8
|
+
"""
|
|
9
|
+
__tablename__ = "CONNECTION"
|
|
10
|
+
|
|
11
|
+
id = Column(Integer, primary_key=True, index=True)
|
|
12
|
+
conn_id = Column(String, index=True, nullable=False)
|
|
13
|
+
description = Column(String, nullable=True)
|
|
14
|
+
conn_type = Column(String, nullable=False)
|
|
15
|
+
runtime = Column(String, nullable=True)
|
|
16
|
+
slug = Column(String, nullable=True)
|
|
17
|
+
status = Column(Boolean, default=False)
|
|
18
|
+
created_by = Column(String, nullable=True)
|
|
19
|
+
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
|
20
|
+
updated_at = Column(DateTime(timezone=True), onupdate=func.now())
|
|
21
|
+
is_active = Column(Boolean, default=True)
|
|
22
|
+
|
|
23
|
+
__table_args__ = (
|
|
24
|
+
UniqueConstraint('conn_id', 'runtime', 'slug', 'is_active', 'created_by', name='uq_active_conn_with_runtime_slug'),
|
|
25
|
+
)
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
from sqlalchemy import Column, Integer, String, Boolean, Text, ForeignKey, DateTime
|
|
2
|
-
from sqlalchemy.orm import relationship
|
|
2
|
+
from sqlalchemy.orm import relationship
|
|
3
3
|
from sqlalchemy.sql import func
|
|
4
4
|
from datetime import datetime
|
|
5
|
-
|
|
6
|
-
Base = declarative_base()
|
|
5
|
+
from dataflow.db import Base
|
|
7
6
|
|
|
8
7
|
class EnvironmentAttributes(Base):
|
|
9
8
|
"""
|
|
@@ -53,7 +52,7 @@ class ArchivedEnvironment(EnvironmentAttributes):
|
|
|
53
52
|
|
|
54
53
|
|
|
55
54
|
class JobLogs(Base):
|
|
56
|
-
__tablename__ = "
|
|
55
|
+
__tablename__ = "JOB_LOG"
|
|
57
56
|
|
|
58
57
|
id = Column(Integer, primary_key=True, index=True)
|
|
59
58
|
created_at = Column(DateTime, default=datetime.now)
|
|
@@ -61,4 +60,5 @@ class JobLogs(Base):
|
|
|
61
60
|
log_file_name = Column(String, unique=True, nullable=False)
|
|
62
61
|
log_file_location = Column(String, nullable=False)
|
|
63
62
|
status = Column(String)
|
|
64
|
-
created_by = Column(String)
|
|
63
|
+
created_by = Column(String)
|
|
64
|
+
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""models.py"""
|
|
2
|
+
from sqlalchemy import Column, Integer, String, ForeignKey, DateTime
|
|
3
|
+
from sqlalchemy.sql import func
|
|
4
|
+
from dataflow.db import Base
|
|
5
|
+
|
|
6
|
+
class EnvironmentStatus(Base):
|
|
7
|
+
"""
|
|
8
|
+
Table ENVIRONMENT_STATUS
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
__tablename__='ENVIRONMENT_STATUS'
|
|
12
|
+
|
|
13
|
+
id = Column(Integer, ForeignKey('ENVIRONMENT.id'), primary_key=True, nullable=False)
|
|
14
|
+
status = Column(String, nullable=False)
|
|
15
|
+
comment = Column(String)
|
|
16
|
+
status_changed_date = Column(DateTime, server_default=func.now(), nullable=False)
|
|
17
|
+
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""models.py"""
|
|
2
|
+
from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, UniqueConstraint
|
|
3
|
+
from sqlalchemy.sql import func
|
|
4
|
+
from dataflow.db import Base
|
|
5
|
+
|
|
6
|
+
class GitSSH(Base):
|
|
7
|
+
__tablename__ = 'GIT_SSH'
|
|
8
|
+
|
|
9
|
+
id = Column(Integer, primary_key=True, index=True, autoincrement=True)
|
|
10
|
+
user_name = Column(String, ForeignKey('USER.user_name', ondelete="CASCADE"), nullable=False)
|
|
11
|
+
description = Column(String)
|
|
12
|
+
key_name = Column(String, nullable=False)
|
|
13
|
+
created_date = Column(DateTime, server_default=func.now(), nullable=False)
|
|
14
|
+
last_used_date = Column(DateTime)
|
|
15
|
+
|
|
16
|
+
__table_args__ = (
|
|
17
|
+
UniqueConstraint(user_name, key_name, name='user_name_key_name_unique'),
|
|
18
|
+
)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from sqlalchemy import Column, Integer, ForeignKey, DateTime, UniqueConstraint
|
|
2
|
+
from dataflow.db import Base
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
|
|
5
|
+
class PinnedProject(Base):
|
|
6
|
+
__tablename__ = "PINNED_PROJECT"
|
|
7
|
+
|
|
8
|
+
id = Column(Integer, primary_key=True)
|
|
9
|
+
user_id = Column(Integer, ForeignKey('USER.user_id'), index=True)
|
|
10
|
+
project_id = Column(Integer, ForeignKey('PROJECT_DETAIL.project_id'), index=True)
|
|
11
|
+
pinned_at = Column(DateTime, default=datetime.utcnow)
|
|
12
|
+
|
|
13
|
+
__table_args__ = (
|
|
14
|
+
UniqueConstraint("user_id", "project_id", name="uix_user_project"),
|
|
15
|
+
)
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from sqlalchemy import Column, String, Enum, DateTime, Integer, func, ForeignKey
|
|
2
|
+
from sqlalchemy.orm import relationship
|
|
3
|
+
from dataflow.db import Base
|
|
4
|
+
|
|
5
|
+
class ProjectDetails(Base):
|
|
6
|
+
__tablename__ = "PROJECT_DETAIL"
|
|
7
|
+
|
|
8
|
+
project_id = Column(Integer, primary_key=True, autoincrement=True)
|
|
9
|
+
project_name = Column(String, nullable=False)
|
|
10
|
+
git_url = Column(String)
|
|
11
|
+
git_branch = Column(String, nullable=True)
|
|
12
|
+
git_folder = Column(String, nullable=True)
|
|
13
|
+
type = Column(String, ForeignKey('RUNTIME_APP_TYPE.name'), nullable=False)
|
|
14
|
+
slug = Column(String, nullable=False, unique=True)
|
|
15
|
+
runtime = Column(String, nullable=False)
|
|
16
|
+
py_env = Column(String, nullable=True)
|
|
17
|
+
launch_url = Column(String, nullable=True)
|
|
18
|
+
status = Column(Enum("pending", "created" ,"deployed", "stopped", "failed", name="deployment_status"), default="created")
|
|
19
|
+
last_deployed = Column(DateTime, nullable=True)
|
|
20
|
+
created_at = Column(DateTime, nullable=False, server_default=func.now())
|
|
21
|
+
created_by = Column(String, nullable=False)
|
|
22
|
+
|
|
23
|
+
app_type = relationship("AppType")
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""models.py"""
|
|
2
|
+
from sqlalchemy import Column, Integer, String, DateTime, UniqueConstraint, Boolean
|
|
3
|
+
from sqlalchemy.sql import func
|
|
4
|
+
from dataflow.db import Local_Base as Base
|
|
5
|
+
|
|
6
|
+
class RecentProjectStudio(Base):
|
|
7
|
+
__tablename__ = 'RECENT_PROJECT_STUDIO'
|
|
8
|
+
|
|
9
|
+
id = Column(Integer, primary_key=True, index=True, autoincrement=True)
|
|
10
|
+
user_name = Column(String, nullable=False, index=True)
|
|
11
|
+
app_name = Column(String, nullable=False, index=True)
|
|
12
|
+
project_name = Column(String, nullable=False)
|
|
13
|
+
project_path = Column(String, nullable=False)
|
|
14
|
+
last_opened_date = Column(DateTime, server_default=func.now(), nullable=False)
|
|
15
|
+
remember = Column(Boolean, default=False)
|
|
16
|
+
|
|
17
|
+
__table_args__ = (
|
|
18
|
+
UniqueConstraint(user_name, project_path, app_name, name='user_name_project_path_app_name_unique'),
|
|
19
|
+
)
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
from sqlalchemy import Column, Integer, ForeignKey
|
|
2
|
+
from dataflow.db import Base
|
|
3
|
+
|
|
4
|
+
class RecentProjects(Base):
|
|
5
|
+
__tablename__ = "RECENT_PROJECT"
|
|
6
|
+
|
|
7
|
+
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
8
|
+
project_id = Column(Integer, ForeignKey('PROJECT_DETAIL.project_id'), nullable=False)
|
|
9
|
+
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""models.py"""
|
|
2
|
+
from sqlalchemy import Column, Integer, String, Enum
|
|
3
|
+
from sqlalchemy.orm import relationship
|
|
4
|
+
from dataflow.db import Base
|
|
5
|
+
|
|
6
|
+
class Role(Base):
|
|
7
|
+
"""
|
|
8
|
+
Table Role
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
__tablename__='ROLE'
|
|
12
|
+
|
|
13
|
+
id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
|
|
14
|
+
name = Column(String, unique=True, nullable=False)
|
|
15
|
+
description = Column(String, nullable=True)
|
|
16
|
+
base_role = Column(Enum('admin', 'user', name='base_role_field'), default='user', nullable=False)
|
|
17
|
+
|
|
18
|
+
users = relationship("User", back_populates="role_details", cascade="all, delete-orphan")
|
|
19
|
+
role_server_assocs = relationship("RoleServer", back_populates="role")
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# models/user_team.py
|
|
2
|
+
from sqlalchemy import Column, Integer, ForeignKey, UniqueConstraint
|
|
3
|
+
from sqlalchemy.orm import relationship
|
|
4
|
+
from dataflow.db import Base
|
|
5
|
+
|
|
6
|
+
class RoleServer(Base):
|
|
7
|
+
__tablename__ = 'ROLE_SERVER'
|
|
8
|
+
__table_args__ = (UniqueConstraint('role_id', 'server_id', name='_role_server_uc'),)
|
|
9
|
+
|
|
10
|
+
role_id = Column(Integer, ForeignKey('ROLE.id', ondelete="CASCADE"), nullable=False, primary_key=True)
|
|
11
|
+
server_id = Column(Integer, ForeignKey('CUSTOM_SERVER.id', ondelete="CASCADE"), nullable=False, primary_key=True)
|
|
12
|
+
|
|
13
|
+
role = relationship("Role", back_populates="role_server_assocs")
|
|
14
|
+
server = relationship("CustomServerConfig", back_populates="role_server_assocs")
|