berryworld 1.0.0.194183__py3-none-any.whl → 1.0.0.194584__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- berryworld/__init__.py +1 -1
- berryworld/sharepoint_con.py +442 -0
- berryworld/snowflake_conn.py +32 -27
- {berryworld-1.0.0.194183.dist-info → berryworld-1.0.0.194584.dist-info}/METADATA +1 -1
- {berryworld-1.0.0.194183.dist-info → berryworld-1.0.0.194584.dist-info}/RECORD +8 -8
- {berryworld-1.0.0.194183.dist-info → berryworld-1.0.0.194584.dist-info}/WHEEL +0 -0
- {berryworld-1.0.0.194183.dist-info → berryworld-1.0.0.194584.dist-info}/licenses/LICENSE +0 -0
- {berryworld-1.0.0.194183.dist-info → berryworld-1.0.0.194584.dist-info}/top_level.txt +0 -0
berryworld/__init__.py
CHANGED
|
@@ -9,7 +9,7 @@ from .credentials import SnowflakeCredentials, SQLCredentials, BCCredentials, We
|
|
|
9
9
|
MicrosoftTeamsCredentials
|
|
10
10
|
from .persistent_storage import PersistentStorage
|
|
11
11
|
from .generate_env import EnvVariables
|
|
12
|
-
from .sharepoint_con import SharepointConnection
|
|
12
|
+
from .sharepoint_con import SharepointConnection, SharepointConn
|
|
13
13
|
from .email_con import EmailConnection
|
|
14
14
|
from .app_logs import AppLogs
|
|
15
15
|
from .app_logs_query import AppLogsQuery
|
berryworld/sharepoint_con.py
CHANGED
|
@@ -449,3 +449,445 @@ class SharepointConnection:
|
|
|
449
449
|
path = os.path.split(path)[0]
|
|
450
450
|
|
|
451
451
|
return final_path
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
class SharepointConn:
|
|
455
|
+
"""Class to connect to sharepoint
|
|
456
|
+
"""
|
|
457
|
+
|
|
458
|
+
def __init__(self, env_name=None, api_version=None):
|
|
459
|
+
""" Initialize the class
|
|
460
|
+
:param env_name: Environment Variable name
|
|
461
|
+
:param api_version: SharePoint API version to use
|
|
462
|
+
"""
|
|
463
|
+
if os.environ.get(f"SHAREPOINT-{env_name.upper()}") is None:
|
|
464
|
+
raise ValueError(f'Sharepoint environment {env_name} not found in the environment variables')
|
|
465
|
+
|
|
466
|
+
if api_version is None:
|
|
467
|
+
self.api_version = 'v1.0'
|
|
468
|
+
else:
|
|
469
|
+
self.api_version = api_version
|
|
470
|
+
|
|
471
|
+
try:
|
|
472
|
+
self.client_id = os.environ.get(f"SHAREPOINT-AUTH").split(' ')[0]
|
|
473
|
+
self.organisation_id = os.environ.get(f"SHAREPOINT-AUTH").split(' ')[1]
|
|
474
|
+
except Exception as e:
|
|
475
|
+
print(e)
|
|
476
|
+
raise ValueError(f"Sharepoint Auth not found in the environment variables")
|
|
477
|
+
|
|
478
|
+
self.env_name = os.environ.get(f"SHAREPOINT-{env_name.upper()}").split(' ')[0]
|
|
479
|
+
self.username = os.environ.get(f"SHAREPOINT-{env_name.upper()}").split(' ')[1]
|
|
480
|
+
self.password = os.environ.get(f"SHAREPOINT-{env_name.upper()}").split(' ')[2]
|
|
481
|
+
self.scopes = ['Sites.ReadWrite.All', 'Files.ReadWrite.All']
|
|
482
|
+
|
|
483
|
+
self.graph_url = f'https://graph.microsoft.com/{self.api_version}'
|
|
484
|
+
self.headers = self.get_headers()
|
|
485
|
+
self.children = 'children?select=id,name,webUrl,file'
|
|
486
|
+
|
|
487
|
+
self.site_id = self.list_sites(site_name=self.env_name)['id'].values[0]
|
|
488
|
+
self.drive_id = self.set_drive_id()
|
|
489
|
+
|
|
490
|
+
def get_headers(self):
|
|
491
|
+
""" Query the Headers to be sent on the queries to SharePoint """
|
|
492
|
+
try:
|
|
493
|
+
base_url = f'https://login.microsoftonline.com/{self.organisation_id}'
|
|
494
|
+
pca = msal.PublicClientApplication(self.client_id, authority=base_url)
|
|
495
|
+
token = pca.acquire_token_by_username_password(self.username, self.password, self.scopes)
|
|
496
|
+
header = {"Authorization": f"Bearer {token['access_token']}"}
|
|
497
|
+
return header
|
|
498
|
+
except Exception as e:
|
|
499
|
+
Exception(f"Error when authenticating in SharePoint: {str(e)}")
|
|
500
|
+
|
|
501
|
+
def set_drive_id(self):
|
|
502
|
+
drive_url = f'{self.graph_url}/sites/{self.site_id}/drives'
|
|
503
|
+
try:
|
|
504
|
+
result = requests.get(drive_url, headers=self.headers)
|
|
505
|
+
drive_id = pd.DataFrame(result.json()['value'])[['id', 'name']]
|
|
506
|
+
except ValueError:
|
|
507
|
+
raise ValueError(f"Could not retrieve the root folders: {traceback.format_exc()}")
|
|
508
|
+
return drive_id
|
|
509
|
+
|
|
510
|
+
def list_sites(self, site_name=None):
|
|
511
|
+
""" List Sites in SharePoint
|
|
512
|
+
:param site_name: Name of the site to retrieve the information
|
|
513
|
+
"""
|
|
514
|
+
site_url = f'{self.graph_url}/sites?search=*'
|
|
515
|
+
try:
|
|
516
|
+
result = requests.get(site_url, headers=self.headers)
|
|
517
|
+
site_values = result.json()['value']
|
|
518
|
+
sites_df = pd.DataFrame(site_values)[['id', 'name', 'webUrl']]
|
|
519
|
+
|
|
520
|
+
if site_name is not None:
|
|
521
|
+
sites_df = sites_df.loc[
|
|
522
|
+
sites_df['name'].str.lower().str.replace(' ', '') == site_name.lower().replace(' ', '')]
|
|
523
|
+
if sites_df.shape[0] == 0:
|
|
524
|
+
raise ValueError(f"The site {site_name} was not found")
|
|
525
|
+
|
|
526
|
+
return sites_df
|
|
527
|
+
|
|
528
|
+
except ValueError as e:
|
|
529
|
+
raise ValueError(f"Could not retrieve the sites: {str(e)}")
|
|
530
|
+
|
|
531
|
+
def list_items(self, path, file_type='All'):
|
|
532
|
+
""" List Folders in SharePoint
|
|
533
|
+
:param path: Full path to the directory to list the items. Use "/" to separate the directories (if path is not
|
|
534
|
+
provided, it will show the root drives)
|
|
535
|
+
:param file_type: Indicate which kind of items to retrieve:
|
|
536
|
+
- "All" will retrieve all the items in the directory
|
|
537
|
+
- "Folder" will retrieve all the folders in the directory
|
|
538
|
+
- "File" will retrieve all the files in the directory
|
|
539
|
+
"""
|
|
540
|
+
try:
|
|
541
|
+
folders = self.resolve_path(path)
|
|
542
|
+
try:
|
|
543
|
+
drive_id = self.drive_id.loc[self.drive_id['name'].str.lower().str.replace(' ', '') ==
|
|
544
|
+
folders[0].lower().replace(' ', ''), 'id'].values[0]
|
|
545
|
+
except Exception as e:
|
|
546
|
+
print(e)
|
|
547
|
+
raise Exception(f"There are no items in: {path}. Please check self.drive_id to provide a valid folder")
|
|
548
|
+
|
|
549
|
+
if len(folders) == 1:
|
|
550
|
+
request_url = f'{self.graph_url}/drives/{drive_id}/root/children?select=id,name,webUrl,file'
|
|
551
|
+
result = requests.get(request_url, headers=self.headers)
|
|
552
|
+
if str(result.status_code).startswith("2"):
|
|
553
|
+
items_df = pd.DataFrame(result.json()['value'])
|
|
554
|
+
if 'file' in items_df.columns:
|
|
555
|
+
items_df = items_df[['id', 'name', 'webUrl', 'file']]
|
|
556
|
+
items_df['file'] = items_df['file'].fillna('folder')
|
|
557
|
+
if file_type.lower() == 'folder':
|
|
558
|
+
return items_df.loc[items_df['file'] == 'folder']
|
|
559
|
+
elif file_type.lower() == 'file':
|
|
560
|
+
return items_df.loc[items_df['file'] != 'folder']
|
|
561
|
+
return items_df
|
|
562
|
+
else:
|
|
563
|
+
items_df = items_df[['id', 'name', 'webUrl']]
|
|
564
|
+
items_df = items_df.assign(**{'file': 'file'})
|
|
565
|
+
return items_df
|
|
566
|
+
else:
|
|
567
|
+
raise Exception(f"Status Code - {str(result.status_code)}. "
|
|
568
|
+
f"There are no items in the root folder: {path}")
|
|
569
|
+
|
|
570
|
+
url_path = urllib.parse.quote("/".join(folders[1:]))
|
|
571
|
+
query_url = f'{self.graph_url}/drives/{drive_id}/root:/{url_path}'
|
|
572
|
+
result = requests.get(query_url, headers=self.headers)
|
|
573
|
+
if str(result.status_code).startswith("2"):
|
|
574
|
+
folder_id = result.json()['id']
|
|
575
|
+
drive_id = result.json()['parentReference']['driveId']
|
|
576
|
+
|
|
577
|
+
request_url = f'{self.graph_url}/drives/{drive_id}/items/{folder_id}/children' \
|
|
578
|
+
f'?select=id,name,webUrl,file'
|
|
579
|
+
result = requests.get(request_url, headers=self.headers)
|
|
580
|
+
if str(result.status_code).startswith("2"):
|
|
581
|
+
items_df = pd.DataFrame(result.json()['value'])
|
|
582
|
+
if items_df.shape[0] > 0:
|
|
583
|
+
if 'file' in items_df.columns:
|
|
584
|
+
items_df = items_df[['id', 'name', 'webUrl', 'file']]
|
|
585
|
+
items_df['file'] = items_df['file'].fillna('folder')
|
|
586
|
+
if file_type.lower() == 'folder':
|
|
587
|
+
return items_df.loc[items_df['file'] == 'folder']
|
|
588
|
+
elif file_type.lower() == 'file':
|
|
589
|
+
return items_df.loc[items_df['file'] != 'folder']
|
|
590
|
+
return items_df
|
|
591
|
+
else:
|
|
592
|
+
items_df = items_df[['id', 'name', 'webUrl']]
|
|
593
|
+
items_df = items_df.assign(**{'file': 'file'})
|
|
594
|
+
return items_df
|
|
595
|
+
else:
|
|
596
|
+
return items_df
|
|
597
|
+
else:
|
|
598
|
+
raise Exception(f"Status Code - {str(result.status_code)}. "
|
|
599
|
+
f"There are no items in: {path}")
|
|
600
|
+
else:
|
|
601
|
+
raise Exception(f"Status Code - {str(result.status_code)}. "
|
|
602
|
+
f"There are no folders in: {path}")
|
|
603
|
+
except Exception as e:
|
|
604
|
+
raise Exception(str(e))
|
|
605
|
+
|
|
606
|
+
def get_file_content(self, file_path):
|
|
607
|
+
""" Get the file content from a file in SharePoint
|
|
608
|
+
:param file_path: Full path to the directory where the file is. Use "/" to separate the directories
|
|
609
|
+
"""
|
|
610
|
+
try:
|
|
611
|
+
folders = self.resolve_path(file_path)
|
|
612
|
+
if len(folders) < 2:
|
|
613
|
+
raise Exception("Please provide a valid full path to the file")
|
|
614
|
+
|
|
615
|
+
drive_id = self.drive_id.loc[self.drive_id['name'].str.lower().str.replace(' ', '') ==
|
|
616
|
+
folders[0].lower().replace(' ', ''), 'id'].values[0]
|
|
617
|
+
|
|
618
|
+
url_path = urllib.parse.quote("/".join(folders[1:]))
|
|
619
|
+
query_url = f'{self.graph_url}/drives/{drive_id}/root:/{url_path}'
|
|
620
|
+
result = requests.get(query_url, headers=self.headers)
|
|
621
|
+
if str(result.status_code).startswith("2"):
|
|
622
|
+
folder_id = result.json()['id']
|
|
623
|
+
drive_id = result.json()['parentReference']['driveId']
|
|
624
|
+
|
|
625
|
+
result = requests.get(f'{self.graph_url}/drives/{drive_id}/items/{folder_id}/content',
|
|
626
|
+
headers=self.headers)
|
|
627
|
+
file_content = result.content
|
|
628
|
+
return file_content
|
|
629
|
+
else:
|
|
630
|
+
raise Exception(f"Status Code - {str(result.status_code)}. "
|
|
631
|
+
f"There are no files called: {folders[-1]}")
|
|
632
|
+
except Exception as e:
|
|
633
|
+
raise Exception(str(e))
|
|
634
|
+
|
|
635
|
+
def download_file(self, file_path, download_name=None, download_path=None):
|
|
636
|
+
""" Get the file content from a file in SharePoint
|
|
637
|
+
:param file_path: Full path to the directory where the file is. Use "/" to separate the directories
|
|
638
|
+
:param download_name: Name for the downloaded file content
|
|
639
|
+
:param download_path: Location of where to download the file content
|
|
640
|
+
"""
|
|
641
|
+
try:
|
|
642
|
+
folders = self.resolve_path(file_path)
|
|
643
|
+
if len(folders) < 2:
|
|
644
|
+
raise Exception("Please provide a valid full path to the file")
|
|
645
|
+
|
|
646
|
+
drive_id = self.drive_id.loc[self.drive_id['name'].str.lower().str.replace(' ', '') ==
|
|
647
|
+
folders[0].lower().replace(' ', ''), 'id'].values[0]
|
|
648
|
+
|
|
649
|
+
url_path = urllib.parse.quote("/".join(folders[1:]))
|
|
650
|
+
query_url = f'{self.graph_url}/drives/{drive_id}/root:/{url_path}'
|
|
651
|
+
result = requests.get(query_url, headers=self.headers)
|
|
652
|
+
if str(result.status_code).startswith("2"):
|
|
653
|
+
folder_id = result.json()['id']
|
|
654
|
+
drive_id = result.json()['parentReference']['driveId']
|
|
655
|
+
|
|
656
|
+
result = requests.get(f'{self.graph_url}/drives/{drive_id}/items/{folder_id}/content',
|
|
657
|
+
headers=self.headers)
|
|
658
|
+
|
|
659
|
+
if download_name is None:
|
|
660
|
+
download_name = folders[-1]
|
|
661
|
+
|
|
662
|
+
if download_path is None:
|
|
663
|
+
open(download_name, 'wb').write(result.content)
|
|
664
|
+
else:
|
|
665
|
+
download_path = download_path + download_name
|
|
666
|
+
with open(download_path, 'wb') as f:
|
|
667
|
+
f.write(result.content)
|
|
668
|
+
else:
|
|
669
|
+
raise Exception(f"Status Code - {str(result.status_code)}. "
|
|
670
|
+
f"There are no files called: {folders[-1]}")
|
|
671
|
+
except Exception as e:
|
|
672
|
+
raise Exception(str(e))
|
|
673
|
+
|
|
674
|
+
def create_folder(self, folder_path):
|
|
675
|
+
""" Create a folder in SharePoint
|
|
676
|
+
:param folder_path: Full path to the directory to be created. Use "/" to separate the directories
|
|
677
|
+
"""
|
|
678
|
+
try:
|
|
679
|
+
folders = self.resolve_path(folder_path)
|
|
680
|
+
if len(folders) < 2:
|
|
681
|
+
raise Exception("Please provide a valid full path to the file")
|
|
682
|
+
|
|
683
|
+
drive_id = self.drive_id.loc[self.drive_id['name'].str.lower().str.replace(' ', '') ==
|
|
684
|
+
folders[0].lower().replace(' ', ''), 'id'].values[0]
|
|
685
|
+
|
|
686
|
+
url_path = urllib.parse.quote("/".join(folders[1:]))
|
|
687
|
+
query_url = f'{self.graph_url}/drives/{drive_id}/root:/{url_path}'
|
|
688
|
+
result = requests.get(query_url, headers=self.headers)
|
|
689
|
+
if not str(result.status_code).startswith("2"):
|
|
690
|
+
url_path = urllib.parse.quote("/".join(folders[1:-1]))
|
|
691
|
+
query_url = f'{self.graph_url}/drives/{drive_id}/root:/{url_path}'
|
|
692
|
+
result = requests.get(query_url, headers=self.headers)
|
|
693
|
+
if str(result.status_code).startswith("2"):
|
|
694
|
+
folder_id = result.json()['id']
|
|
695
|
+
drive_id = result.json()['parentReference']['driveId']
|
|
696
|
+
|
|
697
|
+
query_url = f'{self.graph_url}/drives/{drive_id}/items/{folder_id}/children'
|
|
698
|
+
result = requests.post(query_url, headers=self.headers,
|
|
699
|
+
json={
|
|
700
|
+
"name": urllib.parse.quote(folders[-1]),
|
|
701
|
+
"folder": {},
|
|
702
|
+
"@microsoft.graph.conflictBehavior": "rename"
|
|
703
|
+
})
|
|
704
|
+
print(f"Folder Created: {result.status_code}")
|
|
705
|
+
|
|
706
|
+
else:
|
|
707
|
+
raise Exception(f"Status Code - {str(result.status_code)}. "
|
|
708
|
+
f"Please provide a valid full path. The folders indicated do not exist")
|
|
709
|
+
else:
|
|
710
|
+
raise Exception(f"Status Code - {str(result.status_code)}. "
|
|
711
|
+
f"The folder already exists in path: {folders}")
|
|
712
|
+
except Exception as e:
|
|
713
|
+
raise Exception(str(e))
|
|
714
|
+
|
|
715
|
+
def create_file(self, local_file, file_path, overwrite=True):
|
|
716
|
+
""" Create a file in SharePoint folder
|
|
717
|
+
:param local_file: Local file path to be uploaded to SharePoint
|
|
718
|
+
:param file_path: Full path to the directory where the file is to be created. Use "/" to separate
|
|
719
|
+
the directories. ONLY SUPPORTS uploading csv
|
|
720
|
+
:param overwrite: Whether the file should be overwritten or not if already exists in the path
|
|
721
|
+
"""
|
|
722
|
+
local_file = local_file.rstrip("/")
|
|
723
|
+
file_path = file_path.rstrip("/")
|
|
724
|
+
try:
|
|
725
|
+
# Check files extension
|
|
726
|
+
remote_extension = file_path.split('.')[-1]
|
|
727
|
+
local_extension = local_file.split('.')[-1]
|
|
728
|
+
if (remote_extension == '') | ('/' in remote_extension):
|
|
729
|
+
file_path += '.' + local_extension
|
|
730
|
+
else:
|
|
731
|
+
if remote_extension != local_extension:
|
|
732
|
+
print('## Fixing remote extension file to match local extension file ##')
|
|
733
|
+
file_path = file_path.replace(remote_extension, local_extension)
|
|
734
|
+
|
|
735
|
+
# Read local file data
|
|
736
|
+
with open(local_file, 'rb') as f:
|
|
737
|
+
contents = f.read()
|
|
738
|
+
f.close()
|
|
739
|
+
if len(contents) == 0:
|
|
740
|
+
raise Exception(f'The local file is empty')
|
|
741
|
+
except Exception as e:
|
|
742
|
+
raise Exception(f'The local file path was not valid: {str(e)}')
|
|
743
|
+
|
|
744
|
+
try:
|
|
745
|
+
folders = self.resolve_path(file_path)
|
|
746
|
+
if len(folders) < 2:
|
|
747
|
+
raise Exception("Please provide a valid full path to the file")
|
|
748
|
+
|
|
749
|
+
drive_id = self.drive_id.loc[self.drive_id['name'].str.lower().str.replace(' ', '') ==
|
|
750
|
+
folders[0].lower().replace(' ', ''), 'id'].values[0]
|
|
751
|
+
|
|
752
|
+
if len(folders) == 2:
|
|
753
|
+
query_url = f'{self.graph_url}/drives/{drive_id}/root/children?select=parentReference'
|
|
754
|
+
result = requests.get(query_url, headers=self.headers)
|
|
755
|
+
folder_id = result.json()['value'][0]['parentReference']['id']
|
|
756
|
+
file_name = urllib.parse.quote(folders[-1])
|
|
757
|
+
|
|
758
|
+
query_url = f"{self.graph_url}/drives/{drive_id}/items/{folder_id}/children" \
|
|
759
|
+
f"?$filter=name eq '{file_name}'"
|
|
760
|
+
result = requests.get(query_url, headers=self.headers)
|
|
761
|
+
if len(result.json()['value']) > 0:
|
|
762
|
+
file_id = result.json()['value'][0]['id']
|
|
763
|
+
if overwrite:
|
|
764
|
+
query_url = f'{self.graph_url}/drives/{drive_id}/items/{file_id}/content'
|
|
765
|
+
result = requests.put(
|
|
766
|
+
query_url,
|
|
767
|
+
headers=self.headers,
|
|
768
|
+
data=contents
|
|
769
|
+
)
|
|
770
|
+
print(f"File Overwritten: {result.status_code}")
|
|
771
|
+
else:
|
|
772
|
+
raise Exception("The file is already created and won't be overwritten")
|
|
773
|
+
else:
|
|
774
|
+
query_url = f'{self.graph_url}/drives/{drive_id}/items/{folder_id}:/{file_name}:/content'
|
|
775
|
+
result = requests.put(
|
|
776
|
+
query_url,
|
|
777
|
+
headers=self.headers,
|
|
778
|
+
data=contents
|
|
779
|
+
)
|
|
780
|
+
print(f"File Created: {result.status_code}")
|
|
781
|
+
|
|
782
|
+
else:
|
|
783
|
+
url_path = urllib.parse.quote("/".join(folders[1:-1]))
|
|
784
|
+
query_url = f'{self.graph_url}/drives/{drive_id}/root:/{url_path}'
|
|
785
|
+
result = requests.get(query_url, headers=self.headers)
|
|
786
|
+
if str(result.status_code).startswith("2"):
|
|
787
|
+
folder_id = result.json()['id']
|
|
788
|
+
drive_id = result.json()['parentReference']['driveId']
|
|
789
|
+
file_name = urllib.parse.quote(folders[-1])
|
|
790
|
+
|
|
791
|
+
query_url = f"{self.graph_url}/drives/{drive_id}/items/{folder_id}/children" \
|
|
792
|
+
f"?$filter=name eq '{file_name}'"
|
|
793
|
+
result = requests.get(query_url, headers=self.headers)
|
|
794
|
+
if len(result.json()['value']) > 0:
|
|
795
|
+
file_id = result.json()['value'][0]['id']
|
|
796
|
+
if overwrite:
|
|
797
|
+
query_url = f'{self.graph_url}/drives/{drive_id}/items/{file_id}/content'
|
|
798
|
+
result = requests.put(
|
|
799
|
+
query_url,
|
|
800
|
+
headers=self.headers,
|
|
801
|
+
data=contents
|
|
802
|
+
)
|
|
803
|
+
print(f"File Overwritten: {result.status_code}")
|
|
804
|
+
else:
|
|
805
|
+
raise Exception("The file is already created and won't be overwritten")
|
|
806
|
+
else:
|
|
807
|
+
query_url = f'{self.graph_url}/drives/{drive_id}/items/{folder_id}:/{file_name}:/content'
|
|
808
|
+
result = requests.put(
|
|
809
|
+
query_url,
|
|
810
|
+
headers=self.headers,
|
|
811
|
+
data=contents
|
|
812
|
+
)
|
|
813
|
+
print(f"File Created: {result.status_code}")
|
|
814
|
+
else:
|
|
815
|
+
raise Exception(f"Status Code - {str(result.status_code)}. "
|
|
816
|
+
f"The path where the file is to be created does not exist: {file_path}")
|
|
817
|
+
|
|
818
|
+
except Exception as e:
|
|
819
|
+
raise Exception(str(e))
|
|
820
|
+
|
|
821
|
+
def delete_file(self, file_path):
|
|
822
|
+
""" Delete a file in SharePoint
|
|
823
|
+
:param file_path: Full path to the file to be removed. Use "/" to separate the directories
|
|
824
|
+
"""
|
|
825
|
+
try:
|
|
826
|
+
folders = self.resolve_path(file_path)
|
|
827
|
+
if len(folders) < 2:
|
|
828
|
+
raise Exception("Please provide a valid full path to the file")
|
|
829
|
+
|
|
830
|
+
drive_id = self.drive_id.loc[self.drive_id['name'].str.lower().str.replace(' ', '') ==
|
|
831
|
+
folders[0].lower().replace(' ', ''), 'id'].values[0]
|
|
832
|
+
|
|
833
|
+
url_path = urllib.parse.quote("/".join(folders[1:]))
|
|
834
|
+
query_url = f'{self.graph_url}/drives/{drive_id}/root:/{url_path}'
|
|
835
|
+
result = requests.get(query_url, headers=self.headers)
|
|
836
|
+
if str(result.status_code).startswith("2"):
|
|
837
|
+
drive_id = result.json()['parentReference']['driveId']
|
|
838
|
+
|
|
839
|
+
url_path = urllib.parse.quote("/".join(folders[1:]))
|
|
840
|
+
query_url = f'{self.graph_url}/drives/{drive_id}/root:/{url_path}'
|
|
841
|
+
result = requests.delete(query_url, headers=self.headers)
|
|
842
|
+
|
|
843
|
+
print(f"File Deleted: {result.status_code}")
|
|
844
|
+
|
|
845
|
+
else:
|
|
846
|
+
raise Exception(f"Status Code - {str(result.status_code)}. "
|
|
847
|
+
f"The path indicated does not exist {file_path}")
|
|
848
|
+
except Exception as e:
|
|
849
|
+
raise Exception(str(e))
|
|
850
|
+
|
|
851
|
+
def delete_folder(self, folder_path):
|
|
852
|
+
""" Delete a folder in SharePoint
|
|
853
|
+
:param folder_path: Full path to the folder to be removed. Use "/" to separate the directories
|
|
854
|
+
"""
|
|
855
|
+
try:
|
|
856
|
+
folders = self.resolve_path(folder_path)
|
|
857
|
+
if len(folders) < 2:
|
|
858
|
+
raise Exception("Please provide a valid full path to the file")
|
|
859
|
+
|
|
860
|
+
drive_id = self.drive_id.loc[self.drive_id['name'].str.lower().str.replace(' ', '') ==
|
|
861
|
+
folders[0].lower().replace(' ', ''), 'id'].values[0]
|
|
862
|
+
|
|
863
|
+
url_path = urllib.parse.quote("/".join(folders[1:]))
|
|
864
|
+
query_url = f'{self.graph_url}/drives/{drive_id}/root:/{url_path}'
|
|
865
|
+
result = requests.get(query_url, headers=self.headers)
|
|
866
|
+
if str(result.status_code).startswith("2"):
|
|
867
|
+
drive_id = result.json()['parentReference']['driveId']
|
|
868
|
+
|
|
869
|
+
url_path = urllib.parse.quote("/".join(folders[1:]))
|
|
870
|
+
query_url = f'{self.graph_url}/drives/{drive_id}/root:/{url_path}'
|
|
871
|
+
result = requests.delete(query_url, headers=self.headers)
|
|
872
|
+
|
|
873
|
+
print(f"Folder Deleted: {result.status_code}")
|
|
874
|
+
|
|
875
|
+
else:
|
|
876
|
+
raise Exception(f"Status Code - {str(result.status_code)}. "
|
|
877
|
+
f"The path indicated does not exist {folder_path}")
|
|
878
|
+
except Exception as e:
|
|
879
|
+
raise Exception(str(e))
|
|
880
|
+
|
|
881
|
+
@staticmethod
|
|
882
|
+
def resolve_path(path):
|
|
883
|
+
""" Resolve the path to a list of folders / files
|
|
884
|
+
:param path: Full path to the directory to be created. Use "/" to separate the directories
|
|
885
|
+
"""
|
|
886
|
+
final_path = []
|
|
887
|
+
|
|
888
|
+
path = path.rstrip("/").lstrip("/")
|
|
889
|
+
while os.path.split(path)[-1] != '':
|
|
890
|
+
final_path.insert(0, os.path.split(path)[-1])
|
|
891
|
+
path = os.path.split(path)[0]
|
|
892
|
+
|
|
893
|
+
return final_path
|
berryworld/snowflake_conn.py
CHANGED
|
@@ -1,49 +1,54 @@
|
|
|
1
|
+
import os
|
|
1
2
|
import pandas as pd
|
|
2
3
|
import snowflake.connector
|
|
3
|
-
from .credentials import SnowflakeCredentials
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
class SnowflakeConn:
|
|
7
|
-
def __init__(self,
|
|
7
|
+
def __init__(self, db_reference, server=None):
|
|
8
8
|
""" Initialize the class
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
"account": "",
|
|
12
|
-
"db_name": "",
|
|
13
|
-
"user_name": "",
|
|
14
|
-
"password": ""
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
-----------------------------
|
|
18
|
-
:param server_creds: Dictionary containing the info to connect to the Server
|
|
19
|
-
:param kwargs: Additional parameters to be passed to the connection
|
|
9
|
+
:param db_reference: Reference name for the DB to be used
|
|
10
|
+
:param server: Dictionary containing the info to connect to the Server
|
|
20
11
|
"""
|
|
21
12
|
|
|
22
13
|
self.conn_str = None
|
|
23
14
|
self.conn = None
|
|
15
|
+
self.db_reference = db_reference
|
|
16
|
+
self.server = server
|
|
17
|
+
|
|
18
|
+
try:
|
|
19
|
+
snowflake_ref = os.environ.get("SNOWFLAKE-" + self.db_reference.upper() + '-' + self.server.upper())
|
|
20
|
+
except Exception:
|
|
21
|
+
raise Exception(f"Snowflake reference {self.db_reference} not found in the environment variables")
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
self.account = snowflake_ref.split(' ')[0]
|
|
25
|
+
except Exception:
|
|
26
|
+
raise Exception(f"Snowflake account not provided for Reference {self.db_reference}")
|
|
27
|
+
|
|
28
|
+
try:
|
|
29
|
+
self.db_name = snowflake_ref.split(' ')[1]
|
|
30
|
+
except Exception:
|
|
31
|
+
raise Exception(f"Snowflake db name not provided for Reference {self.db_reference}")
|
|
24
32
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
except KeyError:
|
|
30
|
-
raise KeyError('Please provide a valid db_name and server_type')
|
|
33
|
+
try:
|
|
34
|
+
self.user_name = snowflake_ref.split(' ')[2]
|
|
35
|
+
except Exception:
|
|
36
|
+
raise Exception(f"Snowflake username not provided for Reference {self.db_reference}")
|
|
31
37
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
38
|
+
try:
|
|
39
|
+
self.pw = snowflake_ref.split(' ')[3]
|
|
40
|
+
except Exception:
|
|
41
|
+
raise Exception(f"Snowflake password not provided for Reference {self.db_reference}")
|
|
36
42
|
|
|
37
|
-
|
|
38
|
-
self.connect()
|
|
43
|
+
self.connect()
|
|
39
44
|
|
|
40
45
|
def connect(self):
|
|
41
46
|
""" Open the connection to Snowflake """
|
|
42
47
|
self.conn = snowflake.connector.connect(
|
|
43
|
-
user=self.
|
|
48
|
+
user=self.user_name,
|
|
44
49
|
password=self.pw,
|
|
45
50
|
account=self.account,
|
|
46
|
-
database=self.
|
|
51
|
+
database=self.db_name)
|
|
47
52
|
|
|
48
53
|
def close(self):
|
|
49
54
|
""" Close the connection to Snowflake """
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
berryworld/__init__.py,sha256=
|
|
1
|
+
berryworld/__init__.py,sha256=x0BONOWraAH1SG_BRHWpZNarRMUqoRHKkdASKMtAdhc,1140
|
|
2
2
|
berryworld/aks_logs.py,sha256=Gb2_cokiZbEX01Yoptd0MxpDociaug-GrXdwliyxFBo,2293
|
|
3
3
|
berryworld/allocation_solver.py,sha256=asFtaCAze6-eHUGWXA0kAp67UBS-Upj1KKdrVLj_ttQ,8513
|
|
4
4
|
berryworld/app_logs.py,sha256=MKzKPYd3JuPfOQNAapIgaeZeFHw1z_w2mbn9I6QCADE,4180
|
|
@@ -16,8 +16,8 @@ berryworld/microsoft_teams.py,sha256=8uPo0yku-euBj2VdzBoZCeX3IcsCCOqISLqaVZUVxfA
|
|
|
16
16
|
berryworld/persistent_storage.py,sha256=L15kLyzN42T6UB1WAg8rFXJq3Mdb1M8Sw4P5YQaUN84,8711
|
|
17
17
|
berryworld/pickle_management.py,sha256=5o6UuXBpTj23Jgpz6sj9V-vdcdWBK1xMEckWxT-Whj4,2436
|
|
18
18
|
berryworld/power_automate.py,sha256=V86QEGG9H36DrDvod9Q6yp8OUu307hfYcXJhw06pYrA,27912
|
|
19
|
-
berryworld/sharepoint_con.py,sha256=
|
|
20
|
-
berryworld/snowflake_conn.py,sha256=
|
|
19
|
+
berryworld/sharepoint_con.py,sha256=DSGiS1J_c2PHtGNaC6XkgY8WAFU3KMD6Wx-d723XOHU,44522
|
|
20
|
+
berryworld/snowflake_conn.py,sha256=hhh8pxN3TSg3baJ61olC_oEROSE-yp3MKsG50llLglc,2708
|
|
21
21
|
berryworld/sql_conn.py,sha256=tYKgD8ja7NQuvLB1WBjdsJbPcm3eX1Y76QPTEgx8R8Q,47564
|
|
22
22
|
berryworld/sql_connenction.py,sha256=ku2f4_4JY-X-G9dY7h1eKEkRgZY_Ps-1pC1_1ZvF1ng,48411
|
|
23
23
|
berryworld/teams_logging.py,sha256=8NwXyWr4fLj7W6GzAm2nRQCGFDxibQpAHDHHD24FrP8,6997
|
|
@@ -26,12 +26,12 @@ berryworld/verify_keys.py,sha256=X7VUHqYDklWPDO0bGVHIOXeLq5Qi4fZRZbHYw5x7UnA,435
|
|
|
26
26
|
berryworld/vivantio.py,sha256=QfZo0UKqkzVRg_LyiwivNd3aEup4TH57x4KxLZkCJwc,10627
|
|
27
27
|
berryworld/vivantio_logging.py,sha256=ciy7gA4u3FrgUIpEBnMgocbNPp6jcu9TPoy-kLcrTZU,5736
|
|
28
28
|
berryworld/xml_parser.py,sha256=HWD71NaTN3DaIOGT6Wzxs4CEsroFhGQwe9iPLIL80Co,957
|
|
29
|
-
berryworld-1.0.0.
|
|
29
|
+
berryworld-1.0.0.194584.dist-info/licenses/LICENSE,sha256=vtkVCJM6E2af2gnsi2XxKPr4WY-uIbvzVLXieFND0UU,1074
|
|
30
30
|
tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
31
31
|
tests/test_allocation_config.py,sha256=e12l6fE9U57eSPS35g6ekJ_hol7-RHg89JV60_m1BlE,4633
|
|
32
32
|
tests/test_handy_mix_config.py,sha256=Un56mz9KJmdn4K4OwzHAHLSRzDU1Xv2nFrONNuzOG04,2594
|
|
33
33
|
tests/test_xml_parser.py,sha256=3QTlhFEd6KbK6nRFKZnc35tad6wqukTbe4QrFi8mr_8,859
|
|
34
|
-
berryworld-1.0.0.
|
|
35
|
-
berryworld-1.0.0.
|
|
36
|
-
berryworld-1.0.0.
|
|
37
|
-
berryworld-1.0.0.
|
|
34
|
+
berryworld-1.0.0.194584.dist-info/METADATA,sha256=SsAZin66xZtADTeiNa-TzOFORm2lWCEr6vTcDJmLW4U,1445
|
|
35
|
+
berryworld-1.0.0.194584.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
36
|
+
berryworld-1.0.0.194584.dist-info/top_level.txt,sha256=GIZ5qy-P5oxfEH755vA1IMFeTVdX3-40JxMe6nOe5I8,17
|
|
37
|
+
berryworld-1.0.0.194584.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|