dataflow-core 2.0.10__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- authenticator/dataflowairflowauthenticator.py +51 -76
- authenticator/dataflowhubauthenticator.py +13 -19
- authenticator/dataflowsupersetauthenticator.py +33 -15
- dataflow/{models/database.py → database_manager.py} +4 -3
- dataflow/dataflow.py +129 -19
- dataflow/db.py +51 -0
- dataflow/environment.py +1 -2
- dataflow/models/__init__.py +22 -0
- dataflow/models/app_types.py +10 -0
- dataflow/models/blacklist_library.py +26 -0
- dataflow/models/connection.py +25 -0
- dataflow/models/environment.py +5 -5
- dataflow/models/environment_status.py +17 -0
- dataflow/models/git_ssh.py +18 -0
- dataflow/models/pinned_projects.py +15 -0
- dataflow/models/project_details.py +23 -0
- dataflow/models/recent_project_studio.py +19 -0
- dataflow/models/recent_projects.py +9 -0
- dataflow/models/role.py +19 -0
- dataflow/models/role_server.py +14 -0
- dataflow/models/runtime.py +11 -0
- dataflow/models/server_config.py +33 -0
- dataflow/models/session.py +17 -0
- dataflow/models/team.py +17 -0
- dataflow/models/user.py +29 -0
- dataflow/models/user_environment.py +16 -0
- dataflow/models/user_team.py +14 -0
- dataflow/models/variables.py +27 -0
- dataflow/utils/aws_secrets_manager.py +16 -20
- dataflow/utils/get_current_user.py +35 -0
- dataflow/utils/json_handler.py +33 -0
- dataflow/utils/logger.py +41 -0
- {dataflow_core-2.0.10.dist-info → dataflow_core-2.1.0.dist-info}/METADATA +1 -1
- dataflow_core-2.1.0.dist-info/RECORD +43 -0
- authenticator/package/__init__.py +0 -0
- authenticator/package/configuration.py +0 -27
- authenticator/package/models/__init__.py +0 -0
- authenticator/package/models/database.py +0 -32
- authenticator/package/models/session.py +0 -20
- authenticator/package/models/user.py +0 -22
- dataflow_core-2.0.10.dist-info/RECORD +0 -26
- {dataflow_core-2.0.10.dist-info → dataflow_core-2.1.0.dist-info}/WHEEL +0 -0
- {dataflow_core-2.0.10.dist-info → dataflow_core-2.1.0.dist-info}/entry_points.txt +0 -0
- {dataflow_core-2.0.10.dist-info → dataflow_core-2.1.0.dist-info}/top_level.txt +0 -0
dataflow/models/environment.py
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
from sqlalchemy import Column, Integer, String, Boolean, Text, ForeignKey, DateTime
|
|
2
|
-
from sqlalchemy.orm import relationship
|
|
2
|
+
from sqlalchemy.orm import relationship
|
|
3
3
|
from sqlalchemy.sql import func
|
|
4
4
|
from datetime import datetime
|
|
5
|
-
|
|
6
|
-
Base = declarative_base()
|
|
5
|
+
from dataflow.db import Base
|
|
7
6
|
|
|
8
7
|
class EnvironmentAttributes(Base):
|
|
9
8
|
"""
|
|
@@ -53,7 +52,7 @@ class ArchivedEnvironment(EnvironmentAttributes):
|
|
|
53
52
|
|
|
54
53
|
|
|
55
54
|
class JobLogs(Base):
|
|
56
|
-
__tablename__ = "
|
|
55
|
+
__tablename__ = "JOB_LOG"
|
|
57
56
|
|
|
58
57
|
id = Column(Integer, primary_key=True, index=True)
|
|
59
58
|
created_at = Column(DateTime, default=datetime.now)
|
|
@@ -61,4 +60,5 @@ class JobLogs(Base):
|
|
|
61
60
|
log_file_name = Column(String, unique=True, nullable=False)
|
|
62
61
|
log_file_location = Column(String, nullable=False)
|
|
63
62
|
status = Column(String)
|
|
64
|
-
created_by = Column(String)
|
|
63
|
+
created_by = Column(String)
|
|
64
|
+
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""models.py"""
|
|
2
|
+
from sqlalchemy import Column, Integer, String, ForeignKey, DateTime
|
|
3
|
+
from sqlalchemy.sql import func
|
|
4
|
+
from dataflow.db import Base
|
|
5
|
+
|
|
6
|
+
class EnvironmentStatus(Base):
|
|
7
|
+
"""
|
|
8
|
+
Table ENVIRONMENT_STATUS
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
__tablename__='ENVIRONMENT_STATUS'
|
|
12
|
+
|
|
13
|
+
id = Column(Integer, ForeignKey('ENVIRONMENT.id'), primary_key=True, nullable=False)
|
|
14
|
+
status = Column(String, nullable=False)
|
|
15
|
+
comment = Column(String)
|
|
16
|
+
status_changed_date = Column(DateTime, server_default=func.now(), nullable=False)
|
|
17
|
+
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""models.py"""
|
|
2
|
+
from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, UniqueConstraint
|
|
3
|
+
from sqlalchemy.sql import func
|
|
4
|
+
from dataflow.db import Base
|
|
5
|
+
|
|
6
|
+
class GitSSH(Base):
|
|
7
|
+
__tablename__ = 'GIT_SSH'
|
|
8
|
+
|
|
9
|
+
id = Column(Integer, primary_key=True, index=True, autoincrement=True)
|
|
10
|
+
user_name = Column(String, ForeignKey('USER.user_name', ondelete="CASCADE"), nullable=False)
|
|
11
|
+
description = Column(String)
|
|
12
|
+
key_name = Column(String, nullable=False)
|
|
13
|
+
created_date = Column(DateTime, server_default=func.now(), nullable=False)
|
|
14
|
+
last_used_date = Column(DateTime)
|
|
15
|
+
|
|
16
|
+
__table_args__ = (
|
|
17
|
+
UniqueConstraint(user_name, key_name, name='user_name_key_name_unique'),
|
|
18
|
+
)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from sqlalchemy import Column, Integer, ForeignKey, DateTime, UniqueConstraint
|
|
2
|
+
from dataflow.db import Base
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
|
|
5
|
+
class PinnedProject(Base):
|
|
6
|
+
__tablename__ = "PINNED_PROJECT"
|
|
7
|
+
|
|
8
|
+
id = Column(Integer, primary_key=True)
|
|
9
|
+
user_id = Column(Integer, ForeignKey('USER.user_id'), index=True)
|
|
10
|
+
project_id = Column(Integer, ForeignKey('PROJECT_DETAIL.project_id'), index=True)
|
|
11
|
+
pinned_at = Column(DateTime, default=datetime.utcnow)
|
|
12
|
+
|
|
13
|
+
__table_args__ = (
|
|
14
|
+
UniqueConstraint("user_id", "project_id", name="uix_user_project"),
|
|
15
|
+
)
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from sqlalchemy import Column, String, Enum, DateTime, Integer, func, ForeignKey
|
|
2
|
+
from sqlalchemy.orm import relationship
|
|
3
|
+
from dataflow.db import Base
|
|
4
|
+
|
|
5
|
+
class ProjectDetails(Base):
|
|
6
|
+
__tablename__ = "PROJECT_DETAIL"
|
|
7
|
+
|
|
8
|
+
project_id = Column(Integer, primary_key=True, autoincrement=True)
|
|
9
|
+
project_name = Column(String, nullable=False)
|
|
10
|
+
git_url = Column(String)
|
|
11
|
+
git_branch = Column(String, nullable=True)
|
|
12
|
+
git_folder = Column(String, nullable=True)
|
|
13
|
+
type = Column(String, ForeignKey('RUNTIME_APP_TYPE.name'), nullable=False)
|
|
14
|
+
slug = Column(String, nullable=False, unique=True)
|
|
15
|
+
runtime = Column(String, nullable=False)
|
|
16
|
+
py_env = Column(String, nullable=True)
|
|
17
|
+
launch_url = Column(String, nullable=True)
|
|
18
|
+
status = Column(Enum("pending", "created" ,"deployed", "stopped", "failed", name="deployment_status"), default="created")
|
|
19
|
+
last_deployed = Column(DateTime, nullable=True)
|
|
20
|
+
created_at = Column(DateTime, nullable=False, server_default=func.now())
|
|
21
|
+
created_by = Column(String, nullable=False)
|
|
22
|
+
|
|
23
|
+
app_type = relationship("AppType")
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""models.py"""
|
|
2
|
+
from sqlalchemy import Column, Integer, String, DateTime, UniqueConstraint, Boolean
|
|
3
|
+
from sqlalchemy.sql import func
|
|
4
|
+
from dataflow.db import Local_Base as Base
|
|
5
|
+
|
|
6
|
+
class RecentProjectStudio(Base):
|
|
7
|
+
__tablename__ = 'RECENT_PROJECT_STUDIO'
|
|
8
|
+
|
|
9
|
+
id = Column(Integer, primary_key=True, index=True, autoincrement=True)
|
|
10
|
+
user_name = Column(String, nullable=False, index=True)
|
|
11
|
+
app_name = Column(String, nullable=False, index=True)
|
|
12
|
+
project_name = Column(String, nullable=False)
|
|
13
|
+
project_path = Column(String, nullable=False)
|
|
14
|
+
last_opened_date = Column(DateTime, server_default=func.now(), nullable=False)
|
|
15
|
+
remember = Column(Boolean, default=False)
|
|
16
|
+
|
|
17
|
+
__table_args__ = (
|
|
18
|
+
UniqueConstraint(user_name, project_path, app_name, name='user_name_project_path_app_name_unique'),
|
|
19
|
+
)
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
from sqlalchemy import Column, Integer, ForeignKey
|
|
2
|
+
from dataflow.db import Base
|
|
3
|
+
|
|
4
|
+
class RecentProjects(Base):
|
|
5
|
+
__tablename__ = "RECENT_PROJECT"
|
|
6
|
+
|
|
7
|
+
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
8
|
+
project_id = Column(Integer, ForeignKey('PROJECT_DETAIL.project_id'), nullable=False)
|
|
9
|
+
|
dataflow/models/role.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""models.py"""
|
|
2
|
+
from sqlalchemy import Column, Integer, String, Enum
|
|
3
|
+
from sqlalchemy.orm import relationship
|
|
4
|
+
from dataflow.db import Base
|
|
5
|
+
|
|
6
|
+
class Role(Base):
|
|
7
|
+
"""
|
|
8
|
+
Table Role
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
__tablename__='ROLE'
|
|
12
|
+
|
|
13
|
+
id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
|
|
14
|
+
name = Column(String, unique=True, nullable=False)
|
|
15
|
+
description = Column(String, nullable=True)
|
|
16
|
+
base_role = Column(Enum('admin', 'user', name='base_role_field'), default='user', nullable=False)
|
|
17
|
+
|
|
18
|
+
users = relationship("User", back_populates="role_details", cascade="all, delete-orphan")
|
|
19
|
+
role_server_assocs = relationship("RoleServer", back_populates="role")
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# models/user_team.py
|
|
2
|
+
from sqlalchemy import Column, Integer, ForeignKey, UniqueConstraint
|
|
3
|
+
from sqlalchemy.orm import relationship
|
|
4
|
+
from dataflow.db import Base
|
|
5
|
+
|
|
6
|
+
class RoleServer(Base):
|
|
7
|
+
__tablename__ = 'ROLE_SERVER'
|
|
8
|
+
__table_args__ = (UniqueConstraint('role_id', 'server_id', name='_role_server_uc'),)
|
|
9
|
+
|
|
10
|
+
role_id = Column(Integer, ForeignKey('ROLE.id', ondelete="CASCADE"), nullable=False, primary_key=True)
|
|
11
|
+
server_id = Column(Integer, ForeignKey('CUSTOM_SERVER.id', ondelete="CASCADE"), nullable=False, primary_key=True)
|
|
12
|
+
|
|
13
|
+
role = relationship("Role", back_populates="role_server_assocs")
|
|
14
|
+
server = relationship("CustomServerConfig", back_populates="role_server_assocs")
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from sqlalchemy import Column, Integer, String, Boolean
|
|
2
|
+
from dataflow.db import Base
|
|
3
|
+
|
|
4
|
+
class RuntimeZone(Base):
|
|
5
|
+
__tablename__ = "RUNTIME_ZONE"
|
|
6
|
+
|
|
7
|
+
id = Column(Integer, primary_key=True, autoincrement=True, unique=True)
|
|
8
|
+
name = Column(String, unique=True, nullable=False)
|
|
9
|
+
display_name = Column(String, nullable=False)
|
|
10
|
+
display_order = Column(Integer)
|
|
11
|
+
spark_enabled = Column(Boolean, default=False)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
from sqlalchemy import Column, Integer, String, Boolean, Text, ForeignKey
|
|
2
|
+
from sqlalchemy.dialects.postgresql import JSONB
|
|
3
|
+
from sqlalchemy.sql import func
|
|
4
|
+
from sqlalchemy.orm import relationship
|
|
5
|
+
from dataflow.db import Base
|
|
6
|
+
|
|
7
|
+
class ServerConfig(Base):
|
|
8
|
+
__tablename__ = "SERVER_CONFIG"
|
|
9
|
+
|
|
10
|
+
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
11
|
+
display_name = Column(String, nullable=False, unique=True)
|
|
12
|
+
slug = Column(String, unique=True, nullable=False)
|
|
13
|
+
price = Column(String, nullable=False)
|
|
14
|
+
ram = Column(String, nullable=False)
|
|
15
|
+
cpu = Column(String, nullable=False)
|
|
16
|
+
gpu = Column(String)
|
|
17
|
+
default = Column(Boolean, default=False)
|
|
18
|
+
tags = Column(JSONB, default=func.json([]))
|
|
19
|
+
description = Column(Text, nullable=True)
|
|
20
|
+
kubespawner_override = Column(JSONB, default=func.json({}))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class CustomServerConfig(Base):
|
|
24
|
+
__tablename__ = "CUSTOM_SERVER"
|
|
25
|
+
|
|
26
|
+
id = Column(Integer, primary_key=True, autoincrement=True)
|
|
27
|
+
base_server_id = Column(Integer, ForeignKey(ServerConfig.id), nullable=False)
|
|
28
|
+
display_name = Column(String, nullable=False, unique=True, index=True)
|
|
29
|
+
description = Column(Text, nullable=True)
|
|
30
|
+
|
|
31
|
+
# Relationship to the server_config table
|
|
32
|
+
server_config = relationship(ServerConfig)
|
|
33
|
+
role_server_assocs = relationship("RoleServer", back_populates="server", cascade="all, delete-orphan")
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""models.py"""
|
|
2
|
+
from sqlalchemy import Column, Integer, String, ForeignKey
|
|
3
|
+
from dataflow.db import Base
|
|
4
|
+
|
|
5
|
+
class Session(Base):
|
|
6
|
+
"""
|
|
7
|
+
Table SESSION
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
__tablename__='SESSION'
|
|
11
|
+
|
|
12
|
+
id = Column(Integer, primary_key=True, index=True, unique=True, nullable=False, autoincrement=True)
|
|
13
|
+
session_id = Column(String, unique=True, nullable=False)
|
|
14
|
+
user_id = Column(Integer, ForeignKey('USER.user_id', ondelete="CASCADE"), nullable=False)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
dataflow/models/team.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""models.py"""
|
|
2
|
+
from sqlalchemy import Column, Integer, String
|
|
3
|
+
from sqlalchemy.orm import relationship
|
|
4
|
+
from dataflow.db import Base
|
|
5
|
+
|
|
6
|
+
class Team(Base):
|
|
7
|
+
"""
|
|
8
|
+
Table TEAM
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
__tablename__='TEAM'
|
|
12
|
+
|
|
13
|
+
team_id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
|
|
14
|
+
team_name = Column(String, unique=True, nullable=False)
|
|
15
|
+
description = Column(String, nullable=True)
|
|
16
|
+
|
|
17
|
+
user_team_assocs = relationship("UserTeam", back_populates="team")
|
dataflow/models/user.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""models.py"""
|
|
2
|
+
from sqlalchemy import Column, Integer, String, Boolean, LargeBinary, Enum, ForeignKey
|
|
3
|
+
from sqlalchemy.orm import relationship
|
|
4
|
+
from dataflow.db import Base
|
|
5
|
+
|
|
6
|
+
class User(Base):
|
|
7
|
+
"""
|
|
8
|
+
Table USER
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
__tablename__='USER'
|
|
12
|
+
|
|
13
|
+
user_id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
|
|
14
|
+
user_name = Column(String, unique=True, nullable=False)
|
|
15
|
+
first_name = Column(String)
|
|
16
|
+
last_name = Column(String)
|
|
17
|
+
email = Column(String, unique=True)
|
|
18
|
+
role_id = Column(Integer, ForeignKey('ROLE.id'), nullable=False)
|
|
19
|
+
image = Column(LargeBinary)
|
|
20
|
+
image_url = Column(String, nullable=True)
|
|
21
|
+
active = Column(Enum('N', 'Y', name='active_field'), nullable=False, server_default=str("N"))
|
|
22
|
+
password = Column(String, nullable=False)
|
|
23
|
+
active_env = Column(String, ForeignKey('ENVIRONMENT.short_name'))
|
|
24
|
+
current_server = Column(String)
|
|
25
|
+
show_server_page = Column(Boolean, default = True)
|
|
26
|
+
|
|
27
|
+
role_details = relationship("Role")
|
|
28
|
+
|
|
29
|
+
user_team_assocs = relationship("UserTeam", back_populates="user")
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from sqlalchemy import Column, Integer, String, DateTime
|
|
2
|
+
from sqlalchemy.sql.schema import ForeignKey
|
|
3
|
+
from sqlalchemy.sql.expression import func
|
|
4
|
+
from dataflow.db import Base
|
|
5
|
+
|
|
6
|
+
class UserEnvironment(Base):
|
|
7
|
+
"""
|
|
8
|
+
Table USER_ENVIRONMENT
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
__tablename__ = 'USER_ENVIRONMENT'
|
|
12
|
+
|
|
13
|
+
id = Column(Integer, primary_key=True, index=True, autoincrement=True)
|
|
14
|
+
user_id = Column(Integer, ForeignKey('USER.user_id', ondelete="CASCADE"), nullable=False)
|
|
15
|
+
env_name = Column(String)
|
|
16
|
+
timestamp = Column(DateTime, server_default=func.now(), onupdate=func.now())
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# models/user_team.py
|
|
2
|
+
from sqlalchemy import Column, Integer, ForeignKey, UniqueConstraint
|
|
3
|
+
from sqlalchemy.orm import relationship
|
|
4
|
+
from dataflow.db import Base
|
|
5
|
+
|
|
6
|
+
class UserTeam(Base):
|
|
7
|
+
__tablename__ = 'USER_TEAM'
|
|
8
|
+
__table_args__ = (UniqueConstraint('user_id', 'team_id', name='_user_team_uc'),)
|
|
9
|
+
|
|
10
|
+
user_id = Column(Integer, ForeignKey('USER.user_id', ondelete="CASCADE"), nullable=False, primary_key=True)
|
|
11
|
+
team_id = Column(Integer, ForeignKey('TEAM.team_id', ondelete="CASCADE"), nullable=False, primary_key=True)
|
|
12
|
+
|
|
13
|
+
user = relationship("User", back_populates="user_team_assocs")
|
|
14
|
+
team = relationship("Team", back_populates="user_team_assocs")
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from sqlalchemy import Column, Integer, String, ForeignKey, Text, DateTime, func, UniqueConstraint, CheckConstraint, Boolean
|
|
2
|
+
from dataflow.db import Base
|
|
3
|
+
|
|
4
|
+
class Variable(Base):
|
|
5
|
+
"""
|
|
6
|
+
Unified VARIABLE table to support both Studio and Runtime environments.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
__tablename__ = 'VARIABLE'
|
|
10
|
+
|
|
11
|
+
id = Column(Integer, primary_key=True, index=True, autoincrement=True, nullable=False)
|
|
12
|
+
key = Column(String, nullable=False)
|
|
13
|
+
value = Column(Text, nullable=False)
|
|
14
|
+
type = Column(String, nullable=False)
|
|
15
|
+
description = Column(Text, nullable=True)
|
|
16
|
+
runtime = Column(String, nullable=True)
|
|
17
|
+
slug = Column(String, nullable=True)
|
|
18
|
+
created_at = Column(DateTime, server_default=func.now())
|
|
19
|
+
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
|
|
20
|
+
created_by = Column(String, ForeignKey('USER.user_name'), nullable=True)
|
|
21
|
+
is_active = Column(Boolean, default=True, nullable=False)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
__table_args__ = (
|
|
25
|
+
CheckConstraint(type.in_(['variable', 'secret']), name='check_variable_type'),
|
|
26
|
+
UniqueConstraint('key', 'runtime', 'slug', 'created_by', name='unique_key'),
|
|
27
|
+
)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import boto3
|
|
2
2
|
from botocore.exceptions import ClientError, EndpointConnectionError, NoCredentialsError
|
|
3
3
|
import json
|
|
4
|
+
from .json_handler import JsonHandler
|
|
4
5
|
|
|
5
6
|
class SecretsManagerClient:
|
|
6
7
|
"""
|
|
@@ -11,6 +12,7 @@ class SecretsManagerClient:
|
|
|
11
12
|
json_handler: An instance of JsonHandler for handling JSON operations.
|
|
12
13
|
"""
|
|
13
14
|
def __init__(self):
|
|
15
|
+
self.json_handler = JsonHandler()
|
|
14
16
|
try:
|
|
15
17
|
self.client = boto3.client('secretsmanager')
|
|
16
18
|
except EndpointConnectionError as e:
|
|
@@ -21,14 +23,16 @@ class SecretsManagerClient:
|
|
|
21
23
|
raise Exception(f"Failed to initialize SecretsManagerClient: No AWS credentials found. {e}")
|
|
22
24
|
|
|
23
25
|
|
|
24
|
-
def get_secret_by_key(self, vault_path, user_name,
|
|
26
|
+
def get_secret_by_key(self, vault_path, user_name, conn_id: str, runtime, slug):
|
|
25
27
|
"""
|
|
26
|
-
Get information about a specific secret.
|
|
28
|
+
Get information about a specific secret using a dynamically constructed vault path.
|
|
27
29
|
|
|
28
30
|
Args:
|
|
29
|
-
vault_path (str): The vault path.
|
|
31
|
+
vault_path (str): The base vault path.
|
|
30
32
|
user_name (str): The user name.
|
|
31
|
-
|
|
33
|
+
conn_id (str): The key of the secret to retrieve.
|
|
34
|
+
runtime (str, optional): The runtime environment. Defaults to None.
|
|
35
|
+
slug (str, optional): The slug identifier. Defaults to None.
|
|
32
36
|
|
|
33
37
|
Returns:
|
|
34
38
|
str: Information about the secret in JSON format.
|
|
@@ -37,25 +41,17 @@ class SecretsManagerClient:
|
|
|
37
41
|
Exception: If the operation fails.
|
|
38
42
|
"""
|
|
39
43
|
try:
|
|
40
|
-
if
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
+
if runtime and slug:
|
|
45
|
+
secret_name = f"{runtime}/{slug}/{vault_path}/{conn_id}"
|
|
46
|
+
else:
|
|
47
|
+
secret_name = f"{user_name}/{vault_path}/{conn_id}"
|
|
48
|
+
|
|
44
49
|
response = self.client.get_secret_value(SecretId=secret_name)
|
|
45
|
-
secret_metadata = self.client.describe_secret(SecretId=secret_name)
|
|
46
50
|
secret_data = json.loads(response.get('SecretString'))
|
|
47
51
|
|
|
48
|
-
|
|
49
|
-
secret_info={
|
|
50
|
-
"Name": secret_key,
|
|
51
|
-
"Description": secret_metadata.get('Description')
|
|
52
|
-
}
|
|
53
|
-
secret_info.update(secret_data)
|
|
54
|
-
return secret_info
|
|
55
|
-
else:
|
|
56
|
-
raise Exception(f"Secret named '{secret_key}' is not active")
|
|
52
|
+
return secret_data
|
|
57
53
|
except ClientError as e:
|
|
58
54
|
if e.response['Error']['Code'] == 'ResourceNotFoundException':
|
|
59
|
-
raise Exception(f"Secret named '{
|
|
55
|
+
raise Exception(f"Secret named '{conn_id}' not found with path '{secret_name}'")
|
|
60
56
|
else:
|
|
61
|
-
raise Exception(f"Failed to get secret '{
|
|
57
|
+
raise Exception(f"Failed to get secret '{conn_id}': {e}")
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from fastapi import HTTPException, status
|
|
2
|
+
from sqlalchemy.orm import Session
|
|
3
|
+
from dataflow.models import user as m_user
|
|
4
|
+
from dataflow.models import session as m_session
|
|
5
|
+
|
|
6
|
+
def get_user_from_session(session_id: str, db: Session):
|
|
7
|
+
"""
|
|
8
|
+
Retrieve a user based on session ID.
|
|
9
|
+
|
|
10
|
+
Args:
|
|
11
|
+
session_id (str): The unique session identifier
|
|
12
|
+
db (Session): Database session
|
|
13
|
+
|
|
14
|
+
Returns:
|
|
15
|
+
User: User object if found
|
|
16
|
+
|
|
17
|
+
Raises:
|
|
18
|
+
HTTPException: If session is invalid or user not found
|
|
19
|
+
"""
|
|
20
|
+
session_record = db.query(m_session.Session).filter(m_session.Session.session_id == session_id).first()
|
|
21
|
+
if not session_record:
|
|
22
|
+
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid session")
|
|
23
|
+
|
|
24
|
+
user = db.query(m_user.User).filter(m_user.User.user_id == session_record.user_id).first()
|
|
25
|
+
if not user:
|
|
26
|
+
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="User not found")
|
|
27
|
+
|
|
28
|
+
base_role = user.role_details.base_role
|
|
29
|
+
role_name = user.role_details.name
|
|
30
|
+
user.base_role = base_role
|
|
31
|
+
user.role = role_name
|
|
32
|
+
|
|
33
|
+
return user
|
|
34
|
+
|
|
35
|
+
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
class JsonHandler:
|
|
4
|
+
"""
|
|
5
|
+
Helper class for handling JSON serialization and deserialization.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
def __init__(self):
|
|
9
|
+
pass
|
|
10
|
+
|
|
11
|
+
def dict_to_json(self, data_dict):
|
|
12
|
+
"""
|
|
13
|
+
Serialize a dictionary to JSON string.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
data_dict (dict): The dictionary to serialize.
|
|
17
|
+
|
|
18
|
+
Returns:
|
|
19
|
+
str: The JSON string representation of the dictionary.
|
|
20
|
+
"""
|
|
21
|
+
return json.dumps(data_dict)
|
|
22
|
+
|
|
23
|
+
def json_to_dict(self, json_string):
|
|
24
|
+
"""
|
|
25
|
+
Deserialize a JSON string to dictionary.
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
json_string (str): The JSON string to deserialize.
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
dict: The dictionary representation of the JSON string.
|
|
32
|
+
"""
|
|
33
|
+
return json.loads(json_string)
|
dataflow/utils/logger.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
class CustomLogger:
|
|
5
|
+
"""
|
|
6
|
+
Custom logger class for configuring and using console-based logging functionality.
|
|
7
|
+
"""
|
|
8
|
+
def __init__(self):
|
|
9
|
+
self._configure_handler()
|
|
10
|
+
|
|
11
|
+
def _configure_handler(self):
|
|
12
|
+
"""
|
|
13
|
+
Configure the console handler once.
|
|
14
|
+
"""
|
|
15
|
+
self.formatter = logging.Formatter(
|
|
16
|
+
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
self.console_handler = logging.StreamHandler(sys.stdout)
|
|
20
|
+
self.console_handler.setLevel(logging.INFO)
|
|
21
|
+
self.console_handler.setFormatter(self.formatter)
|
|
22
|
+
|
|
23
|
+
def get_logger(self, name=None):
|
|
24
|
+
"""
|
|
25
|
+
Retrieve a configured logger instance with a dynamic name.
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
name (str): The name of the logger. Defaults to 'dataflow_logger'.
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
logging.Logger: Configured logger instance.
|
|
32
|
+
"""
|
|
33
|
+
logger_name = name or "dataflow_logger"
|
|
34
|
+
logger = logging.getLogger(logger_name)
|
|
35
|
+
logger.setLevel(logging.INFO)
|
|
36
|
+
|
|
37
|
+
# Prevent duplicate handlers if logger already configured
|
|
38
|
+
if not logger.handlers:
|
|
39
|
+
logger.addHandler(self.console_handler)
|
|
40
|
+
|
|
41
|
+
return logger
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
authenticator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
authenticator/dataflowairflowauthenticator.py,sha256=gEdCiL2yJQ7lYvAwbrjcAkccVMfehoMJldw9eU7cc2s,2243
|
|
3
|
+
authenticator/dataflowhubauthenticator.py,sha256=A0S0_PJVCPiTfuoLormmO0xNDqjIh00Cf93P0-mF0cw,2913
|
|
4
|
+
authenticator/dataflowsupersetauthenticator.py,sha256=Qx8hUGSOfUTgffRCaVf88vmE9bFGNBvd9mEMHcpD488,2463
|
|
5
|
+
dataflow/__init__.py,sha256=WTRg8HMpMWSgxYJ9ZGVldx4k07fAbta3mBmZ1hG9mWE,30
|
|
6
|
+
dataflow/configuration.py,sha256=7To6XwH1eESiYp39eqPcswXWwrdBUdPF6xN6WnazOF0,663
|
|
7
|
+
dataflow/database_manager.py,sha256=tJHMuOZ9Muskrh9t4uLRlTuFU0VkHAzoHlGP5DORIC4,899
|
|
8
|
+
dataflow/dataflow.py,sha256=-UYZst7EO1GgaOjlAkKu-tu7RC6XsgadGeDp1MOvZiA,7439
|
|
9
|
+
dataflow/db.py,sha256=5UwE4w5Vn9RqFIhr8ARlu2haZX-PtHDLRPjmn5BG2m8,1649
|
|
10
|
+
dataflow/environment.py,sha256=04LwJwAjciBOz-EJJGzBA_BSQRKf587TJ30R8cDzvhg,19582
|
|
11
|
+
dataflow/models/__init__.py,sha256=QMLiKj8BMhfScWMm8kgHkMjwAlFeg5Cym3_AI1NvBUA,783
|
|
12
|
+
dataflow/models/app_types.py,sha256=yE_ZB13lhpK7AZ7PyBwnQlf0RlIHYs_-vdMKx7_RMlY,379
|
|
13
|
+
dataflow/models/blacklist_library.py,sha256=B2oi3Z8GcR_glhLAyinFk0W8c9txXvm3uOER6dY-q7I,991
|
|
14
|
+
dataflow/models/connection.py,sha256=_VJL3KuIrm8t4lJmtunIL3-AXF9Yvi5wUolzdR3tE0E,1017
|
|
15
|
+
dataflow/models/environment.py,sha256=HQW1L0qKksFR0qeQQbwcPjd-zcZwS1PEvzMOFrKgnpE,2142
|
|
16
|
+
dataflow/models/environment_status.py,sha256=GnoAKI8GdCTqTq4HLhx16K2k--LMpjeJuRRRjGfAXoA,516
|
|
17
|
+
dataflow/models/git_ssh.py,sha256=W15SDypxzGOz_aZkHEnVZ6DIMVsjAsbSIXVIEt2mPYU,694
|
|
18
|
+
dataflow/models/pinned_projects.py,sha256=rkpPX_f2U9HjmrRo7_K8rnZIeXuQKGq6hYTrtLmu21c,566
|
|
19
|
+
dataflow/models/project_details.py,sha256=94wTygXv9iGB0w8g_6vtkB5ZqIzpEv1W9uWwCA4hM0Y,1078
|
|
20
|
+
dataflow/models/recent_project_studio.py,sha256=m12KGCsv453C1ijHjfVD8E7cJ7Og_0N8uc7_9VlfkYw,812
|
|
21
|
+
dataflow/models/recent_projects.py,sha256=QqDlk3ll7tBaQl5hqvRarlB9_SUBuN44muLIuTVbPe0,301
|
|
22
|
+
dataflow/models/role.py,sha256=i068kzy1UYfl2XNPGl8O4h-1Dcpxb_zzrbI9IW2FAxI,680
|
|
23
|
+
dataflow/models/role_server.py,sha256=mMcfjsGX1cY8hOAOBBmrZgw8ozdfuvjKJoBlR6F0Kdc,689
|
|
24
|
+
dataflow/models/runtime.py,sha256=OiuBfZTMg81U10GS00DxfhiAmHlcyQUw5LBR8RaPl7s,415
|
|
25
|
+
dataflow/models/server_config.py,sha256=GTMtQfgtuvKUbxV16VhEpKGhYoNISFLRWdUPqBJmYbM,1365
|
|
26
|
+
dataflow/models/session.py,sha256=c8TI6qXsM8utzp5vSQtAOXJSbgasnyu-a0qSAvA-rWs,459
|
|
27
|
+
dataflow/models/team.py,sha256=fjkqF0N4PSwSYTgHjEQl9wuC7yumd0iOb5nNFePI6q4,488
|
|
28
|
+
dataflow/models/user.py,sha256=PT-zwZj7NWUubIj_7EY2EsjduMbI_42EyMclWMLESGk,1073
|
|
29
|
+
dataflow/models/user_environment.py,sha256=yI9NutULcLiwlycuEin6ROe6o1Sjdv_sgw2MEkJFeYg,568
|
|
30
|
+
dataflow/models/user_team.py,sha256=r_fmKvf6JuGgiiI9TXWjVG2QZ3WOvDrOwYWVQ3r8oWo,659
|
|
31
|
+
dataflow/models/variables.py,sha256=Sinvv3zFYni5i_GrL69cVfhCh4tOOaIHiEzWYRJ-i10,1132
|
|
32
|
+
dataflow/scripts/clone_environment.sh,sha256=PJfMWPgiWSkvY-x98WmjeAkRREjC9824JzTazHe2iQQ,390
|
|
33
|
+
dataflow/scripts/create_environment.sh,sha256=ams50MD1r53cHRYDfpAvmEAMsCaCFvlL0cmnRVXhFgY,496
|
|
34
|
+
dataflow/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
35
|
+
dataflow/utils/aws_secrets_manager.py,sha256=A_fNs9VNah9dDdl9NhqizJamYU7xr2v_GXlw9InEDFk,2380
|
|
36
|
+
dataflow/utils/get_current_user.py,sha256=akjcUyTpmMdAZj9LFGSTs76hjBRjltNk9hLUqC_BdkA,1140
|
|
37
|
+
dataflow/utils/json_handler.py,sha256=5_7WdypegRBDe2HSqBXyrJAdd92wsha8qRcmQvCj1TA,782
|
|
38
|
+
dataflow/utils/logger.py,sha256=7BFrOq5Oiqn8P4XZbgJzMP5O07d2fpdECbbfsjrUuHw,1213
|
|
39
|
+
dataflow_core-2.1.0.dist-info/METADATA,sha256=x9qvy7PPIKQCRyYEnGRP47R1yMpicivWON4Re7rWj6w,301
|
|
40
|
+
dataflow_core-2.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
41
|
+
dataflow_core-2.1.0.dist-info/entry_points.txt,sha256=ppj_EIbYrJJwCPg1kfdsZk5q1N-Ejfis1neYrnjhO8o,117
|
|
42
|
+
dataflow_core-2.1.0.dist-info/top_level.txt,sha256=SZsUOpSCK9ntUy-3Tusxzf5A2e8ebwD8vouPb1dPt_8,23
|
|
43
|
+
dataflow_core-2.1.0.dist-info/RECORD,,
|
|
File without changes
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
"""configuration.py"""
|
|
2
|
-
import configparser
|
|
3
|
-
from configparser import NoOptionError, NoSectionError
|
|
4
|
-
|
|
5
|
-
class ConfigurationManager:
|
|
6
|
-
"""
|
|
7
|
-
Configuration Manager
|
|
8
|
-
"""
|
|
9
|
-
|
|
10
|
-
def __init__(self, config_file):
|
|
11
|
-
|
|
12
|
-
self.config_file = config_file
|
|
13
|
-
self.config = configparser.ConfigParser()
|
|
14
|
-
try:
|
|
15
|
-
self.config.read(self.config_file)
|
|
16
|
-
|
|
17
|
-
except Exception as e:
|
|
18
|
-
return None
|
|
19
|
-
|
|
20
|
-
def get_config_value(self, section, option):
|
|
21
|
-
"""
|
|
22
|
-
Get configuration value
|
|
23
|
-
"""
|
|
24
|
-
try:
|
|
25
|
-
return self.config.get(section, option)
|
|
26
|
-
except (NoOptionError, NoSectionError):
|
|
27
|
-
return None
|
|
File without changes
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
"""models/database.py"""
|
|
2
|
-
from sqlalchemy.exc import SQLAlchemyError
|
|
3
|
-
from sqlalchemy.ext.declarative import declarative_base
|
|
4
|
-
from sqlalchemy import create_engine
|
|
5
|
-
from sqlalchemy.orm import sessionmaker
|
|
6
|
-
|
|
7
|
-
class DatabaseManager:
|
|
8
|
-
def __init__(self, db_url):
|
|
9
|
-
self.db_url = db_url
|
|
10
|
-
|
|
11
|
-
def get_engine(self):
|
|
12
|
-
try:
|
|
13
|
-
engine = create_engine(self.db_url)
|
|
14
|
-
return engine
|
|
15
|
-
except SQLAlchemyError as e:
|
|
16
|
-
raise e
|
|
17
|
-
|
|
18
|
-
def get_session(self):
|
|
19
|
-
try:
|
|
20
|
-
engine = self.get_engine()
|
|
21
|
-
session = sessionmaker(autocommit=False, autoflush=False, bind=engine or create_engine(self.db_url))
|
|
22
|
-
db = session()
|
|
23
|
-
try:
|
|
24
|
-
yield db
|
|
25
|
-
finally:
|
|
26
|
-
db.close()
|
|
27
|
-
|
|
28
|
-
except SQLAlchemyError as e:
|
|
29
|
-
raise e
|
|
30
|
-
|
|
31
|
-
def get_base(self):
|
|
32
|
-
return declarative_base()
|