move-data 0.1.5__py3-none-any.whl

move_data/__init__.py

@@ -0,0 +1,25 @@
+"""
+move-data: A Python package for moving data between various sources and destinations.
+
+This package provides utilities for:
+- Google Sheets data extraction
+- SharePoint file operations
+- Google Cloud Storage file operations
+- Snowflake data loading and extraction
+"""
+
+from .move_data import (
+    get_googlesheets_data,
+    sharepoint,
+    snowflake,
+    googlestorage
+)
+
+__version__ = "0.1.5"
+__all__ = [
+    "get_googlesheets_data",
+    "sharepoint",
+    "snowflake",
+    "googlestorage"
+]
+

move_data/move_data.py

@@ -0,0 +1,304 @@
+import requests
+from io import StringIO, BytesIO
+import datetime, time
+import warnings
+warnings.filterwarnings('ignore')
+import snowflake.connector as sf
+from snowflake.connector.pandas_tools import write_pandas
+import base64
+import logging,json, pygsheets, pandas as pd
+import chardet
+from zipfile import BadZipFile
+import re
+import os.path
+from google.cloud import storage
+
+def get_googlesheets_data(name,sheet,service_account_path,skip_rows=0):
+  global new_columns, old_columns,file
+  gc = pygsheets.authorize(service_account_file = service_account_path)
+
+  sh = gc.open(name)
+  wks = sh.worksheet_by_title(sheet)
+  data = wks.get_as_df()
+  
+  if skip_rows > 0:
+    data = data.iloc[skip_rows:].reset_index(drop=True)
+
+  new_columns = []
+  old_columns = data.columns.tolist()
+
+  for item in old_columns:
+    if type(item) == str:
+      new_item = re.sub(r'^order$','"order"',item.replace(" ($)","_").replace(" \+ ","_").replace(":","_").replace(" ","_").replace(".","").replace("(","").replace(")","").replace("/","_").replace(",","_").\
+      replace("-","_").replace("%","per").replace('unnamed__',"").lstrip('0123456789').replace('unique','a_unique').lower().replace('#','').replace("+","").replace('&','_').replace('___','_').replace('__','_'))
+      new_columns.append(new_item)
+    elif type(item) == datetime.datetime:
+      new_item = item.strftime("%b_%Y").lower()
+      new_columns.append(new_item)
+      
+  try: 
+    data = data.drop('',axis=1)
+  except KeyError:
+    pass
+
+  for col in data.columns:
+    try:
+      data[col] = data[col].astype(str)
+    except (ValueError,KeyError):
+      continue
+
+  for i in range(len(new_columns)):
+    new_columns[i] = new_columns[i].lower()
+
+  data.columns = new_columns
+
+  drop_cols = ['',',','_']
+  data = data.drop(drop_cols, axis=1, errors = 'ignore')
+  data = data.loc[:,~data.columns.duplicated()]
+  new_columns = data.columns.tolist()
+
+  new_columns = data.columns
+
+  sf_cols = []
+  sf_tr = []
+
+  for i in range(len(new_columns)):
+    new_value = new_columns[i].lower() + ' ' + 'string'
+    transform = 'nullif(' + new_columns[i].lower() + ',\'\') as ' + new_columns[i].lower()        # + ' ' + 'string'
+    sf_cols.append(new_value)
+    sf_tr.append(transform)
+
+  sf_query = "\n,".join(sf_cols)
+  sf_tr_query = "\n,".join(sf_tr)
+
+  return data, sf_query, sf_tr_query
+
+class sharepoint:
+  def __init__(self,client_id,client_secret,tenant_id,site_id,library_name,drive_id):
+    self.client_id = client_id
+    self.client_secret = client_secret
+    self.tenant_id = tenant_id
+
+    # SharePoint Online site URL and library name
+
+    self.site_id = site_id
+    self.library_name = library_name
+    self.drive_id = drive_id
+
+    # Authenticate and get an access token
+    auth_url = f'https://login.microsoftonline.com/{self.tenant_id}/oauth2/v2.0/token'
+    data = {
+        'grant_type': 'client_credentials',
+        'client_id': self.client_id,
+        'client_secret': self.client_secret,
+        'scope': 'https://graph.microsoft.com/.default'
+    }
+    response = requests.post(auth_url, data=data)
+    self.access_token = response.json()['access_token']
+
+    self.headers = {
+        'Authorization': f'Bearer {self.access_token}',
+        'Content-Type': 'application/octet-stream',
+    }
+
+  def get_data(self,search_query,relative_path,date_col,sheet_name,skip_rows):
+    api_url = f'https://graph.microsoft.com/v1.0/sites/{self.site_id}/drives/{self.drive_id}/items/root:/{relative_path}:/children'
+    response = requests.get(api_url, headers=self.headers)
+
+    data = response.json()
+
+    df = pd.DataFrame(data = data['value'].copy())
+    cols = df.columns.to_list()
+    df_filtered = df[df[cols]['name'].str.lower().str.contains(search_query.lower())].sort_values('lastModifiedDateTime',ascending=False).head(1).reset_index()
+
+    file_name = df_filtered['name'].values[0]
+    api_url_content = f'https://graph.microsoft.com/v1.0/sites/{self.site_id}/drives/{self.drive_id}/items/root:/{relative_path}/{file_name}:/content'
+
+    output = StringIO()
+    file = requests.get(api_url_content,headers=self.headers)
+    
+    try:
+      output = StringIO()
+      data = pd.read_csv(StringIO(file.content.decode('utf-8')),skiprows=skip_rows)
+    except (UnicodeDecodeError,BadZipFile) as err:
+      print(err,'\nNow processing as excel file')
+      output = BytesIO()
+      dict = pd.read_excel(BytesIO(file.content),sheet_name=[sheet_name],engine='openpyxl',skiprows=skip_rows)
+      data = dict[sheet_name]
+
+    data['insertion_datetime'] = datetime.datetime.now().strftime('%Y-%m-%d %I:%M:%S')
+  
+    new_columns = []
+    old_columns = data.columns.tolist()
+
+    for item in old_columns:
+      if type(item) == str:
+        new_item = re.sub('^_|_$','',item.replace(" ($)","_").replace('\n','_').replace(" \+ ","_").replace(":","_").replace(" ","_").replace(".","")\
+        .replace("(","").replace(")","").replace("/","_").replace(",","_").\
+        replace("-","_").replace('__','_').replace('___','_').replace("%","per").replace('unnamed__',"").lstrip('0123456789')\
+        .replace('unique','a_unique').lower().replace('#','').replace('?','').replace("+","").replace('^_','').replace('_$',''))
+        new_columns.append(new_item)
+      elif type(item) == datetime.datetime:
+        new_item = item.strftime("%b_%Y").lower()
+        new_columns.append(new_item)
+        
+    try: 
+      data = data.drop('',axis=1)
+    except KeyError:
+      pass
+
+    for col in data.columns:
+      try:
+        data[col] = data[col].astype(str)
+      except (ValueError,KeyError):
+        continue
+
+    for i in range(len(new_columns)):
+      new_columns[i] = new_columns[i].lower()
+
+    data.columns = new_columns
+
+    drop_cols = ['',',','_']
+    data = data.drop(drop_cols, axis=1, errors = 'ignore')
+    data = data.loc[:,~data.columns.duplicated()]
+    new_columns = data.columns.tolist()
+
+    sf_cols = []
+    sf_tr = []
+
+    for i in range(len(new_columns)):
+      new_value = new_columns[i].lower() + ' ' + 'string'
+      transform = 'nullif(' + new_columns[i].lower() + ',\'nan\') as ' + new_columns[i].lower()        # + ' ' + 'string'
+      sf_cols.append(new_value)
+      sf_tr.append(transform)
+
+    sf_query = "\n,".join(sf_cols)
+    sf_tr_query = "\n,".join(sf_tr)
+
+    return data, sf_query, sf_tr_query, file, api_url_content
+
+  def upload_file(self,upload_url,modified_data,content_type='application/octet-stream'):
+    headers = {
+        'Authorization': f'Bearer {self.access_token}',
+        'Content-Type': content_type,
+        'Content-Length': str(len(modified_data)),
+    }
+    upload_response = requests.put(upload_url, data=modified_data, headers=headers)
+    if upload_response.status_code == 200:
+      print("File uploaded successfully.")
+    else:
+      print(f"Failed to upload file. Status code: {upload_response.status_code}")
+    upload_response.close()
+    return headers
+  
+
+class snowflake:
+  def __init__(self,user,pw,database,schema,role):
+
+    self.database=database
+    self.schema=schema
+    self.role=role
+
+    if 'airbyte' in role.lower():
+      warehouse = 'airbyte_warehouse'
+    else:
+      warehouse = 'cart_dev_compute_wh'
+
+    self.cnn = sf.connect(
+            user= user,
+            password = pw,
+            account = 'og64234.us-central1.gcp',
+            warehouse = warehouse,
+            database = database,
+            role = role,
+            schema = schema)
+    
+  def load_data(self,sf_query,sf_tr_query,table_name,data,change_tracking=None):
+    print('Table Name: {}'.format(self.database + '.' + self.schema + '.' + table_name))
+    print('Start: load to Snowflake...')
+    data.reset_index(drop=True, inplace=True)
+    print('opening snowflake...')
+
+    self.cnn.cursor().execute(
+        "CREATE SCHEMA IF NOT EXISTS " + self.database + "." + self.schema
+    ) 
+    
+    self.cnn.cursor().execute(
+      "CREATE OR REPLACE TABLE " +
+      table_name + "("  + sf_query + ")"
+    )
+
+    success, nchunks, nrows, _ = write_pandas(self.cnn, data, table_name, on_error = "CONTINUE",quote_identifiers=False)
+    print(str(success) + ', ' + str(nchunks) + ', ' + str(nrows))
+
+    self.cnn.cursor().execute(
+      "CREATE OR REPLACE TABLE " + table_name + " as" + "\nselect\n" + sf_tr_query + '\nfrom\n' + table_name 
+    )
+
+    if change_tracking:
+      self.cnn.cursor().execute("ALTER TABLE " + table_name + " set CHANGE_TRACKING=TRUE")
+      print("Change Tracking Enabled")
+
+    if self.database.casefold() != "maas_db":
+      print(self.database)
+      self.cnn.cursor().execute("EXECUTE TASK ENRICHMENT_DB.TASKS.SPROC5_TRIGGER")
+
+    print('Started: Executed SPROC5...\n\n')
+
+    self.cnn.close()
+    print('Done: Load to Snowflake\n\n')
+
+  def get_data(self,sheet_name,search_query):
+    global df, sqlText, file_path
+    print('Start: download from Snowflake for sheet {}'.format(sheet_name))
+
+    print('opening snowflake...')
+
+    sqlText = search_query
+    print(sqlText)
+
+    # Create a cursor object
+    cur = self.cnn.cursor().execute(sqlText)
+
+    # Fetch the result set from the cursor and deliver it as the Pandas DataFrame
+    self.df = cur.fetch_pandas_all()
+
+    # Process the DataFrame as needed for each sheet
+    columns = self.df.columns.tolist()
+
+    # Create a new dataframe with dynamic column names
+    cumm_df = pd.DataFrame(columns=columns,data=self.df)
+    # cumm_df = cumm_df.append(self.df, ignore_index=True)
+
+    self.cnn.close()
+    print('End: download from Snowflake for sheet {}'.format(sheet_name))
+
+    # Return the new dataframe
+    return cumm_df
+
+class googlestorage:
+  def __init__(self,service_account):
+    self.client = storage.Client.from_service_account_json(service_account)
+
+  def get_data(self,bucket_name,path,search_query,sheet_name,skip_rows):
+    bucket = self.client.get_bucket(bucket_name)
+    blobs = bucket.list_blobs(prefix=f'{path}')
+    max_modified_date = None
+
+    for blob in blobs:
+      if search_query.lower() in blob.name.lower():
+        modified_time = blob.updated
+        if max_modified_date is None or modified_time > max_modified_date:
+          max_modified_date = modified_time
+          fblob = blob
+          print(f'Object Name: {fblob.name}, Modified: {fblob.updated}')
+
+    try:
+      csv_data = fblob.download_as_string()
+      csv_string = csv_data.decode('utf-8')
+      df = pd.read_csv(csv_string,skiprows=skip_rows)
+    except UnicodeDecodeError:
+      csv_data = fblob.download_as_bytes()
+      df = pd.read_excel(csv_data,sheet_name=sheet_name,skiprows=skip_rows)
+
+    return df

move_data/setup.py

@@ -0,0 +1,37 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="move-data",
+    version="0.1.5",
+    description="A Python package for moving data between Google Sheets, SharePoint, Google Cloud Storage, and Snowflake",
+    long_description=open("README.md").read(),
+    long_description_content_type="text/markdown",
+    author="Your Name",
+    author_email="your.email@example.com",
+    url="https://github.com/yourusername/move-data",
+    packages=["move_data"],
+    package_dir={"move_data": "."},
+    include_package_data=True,
+    python_requires=">=3.7",
+    install_requires=[
+        "requests>=2.25.0",
+        "pandas>=1.3.0",
+        "pygsheets>=2.0.0",
+        "snowflake-connector-python>=2.7.0",
+        "google-cloud-storage>=2.0.0",
+        "chardet>=4.0.0",
+        "openpyxl>=3.0.0",
+    ],
+    classifiers=[
+        "Development Status :: 4 - Beta",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: MIT License",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+    ],
+)
+

move_data-0.1.5.dist-info/METADATA

@@ -0,0 +1,139 @@
+Metadata-Version: 2.4
+Name: move-data
+Version: 0.1.5
+Summary: A Python package for moving data between Google Sheets, SharePoint, Google Cloud Storage, and Snowflake
+Home-page: https://github.com/yourusername/move-data
+Author: Your Name
+Author-email: Your Name <your.email@example.com>
+License: MIT
+Project-URL: Homepage, https://github.com/yourusername/move-data
+Project-URL: Documentation, https://github.com/yourusername/move-data#readme
+Project-URL: Repository, https://github.com/yourusername/move-data
+Project-URL: Issues, https://github.com/yourusername/move-data/issues
+Keywords: data,etl,snowflake,google-sheets,sharepoint,google-cloud-storage
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: requests>=2.25.0
+Requires-Dist: pandas>=1.3.0
+Requires-Dist: pygsheets>=2.0.0
+Requires-Dist: snowflake-connector-python>=2.7.0
+Requires-Dist: google-cloud-storage>=2.0.0
+Requires-Dist: chardet>=4.0.0
+Requires-Dist: openpyxl>=3.0.0
+Dynamic: author
+Dynamic: home-page
+Dynamic: requires-python
+
+# move-data
+
+A Python package for moving data between Google Sheets, SharePoint, Google Cloud Storage, and Snowflake.
+
+## Features
+
+- **Google Sheets Integration**: Extract data from Google Sheets with automatic column name normalization
+- **SharePoint Integration**: Download and upload files from SharePoint Online
+- **Google Cloud Storage**: Retrieve files from GCS buckets
+- **Snowflake Integration**: Load data to and extract data from Snowflake databases
+
+## Installation
+
+```bash
+pip install move-data
+```
+
+## Usage
+
+### Google Sheets
+
+```python
+from move_data import get_googlesheets_data
+
+data, sf_query, sf_tr_query = get_googlesheets_data(
+    name="My Spreadsheet",
+    sheet="Sheet1",
+    service_account_path="/path/to/service_account.json",
+    skip_rows=0  # Optional: skip first N rows
+)
+```
+
+### SharePoint
+
+```python
+from move_data import sharepoint
+
+sp = sharepoint(
+    client_id="your_client_id",
+    client_secret="your_client_secret",
+    tenant_id="your_tenant_id",
+    site_id="your_site_id",
+    library_name="Documents",
+    drive_id="your_drive_id"
+)
+
+data, sf_query, sf_tr_query, file, api_url = sp.get_data(
+    search_query="filename",
+    relative_path="folder/path",
+    date_col="date_column",
+    sheet_name="Sheet1",
+    skip_rows=0
+)
+```
+
+### Snowflake
+
+```python
+from move_data import snowflake
+
+sf = snowflake(
+    user="username",
+    pw="password",
+    database="database_name",
+    schema="schema_name",
+    role="role_name"
+)
+
+# Load data to Snowflake
+sf.load_data(sf_query, sf_tr_query, "table_name", data, change_tracking=True)
+
+# Get data from Snowflake
+df = sf.get_data(sheet_name="Sheet1", search_query="SELECT * FROM table")
+```
+
+### Google Cloud Storage
+
+```python
+from move_data import googlestorage
+
+gs = googlestorage(service_account="/path/to/service_account.json")
+
+df = gs.get_data(
+    bucket_name="my-bucket",
+    path="folder/path",
+    search_query="filename",
+    sheet_name="Sheet1",
+    skip_rows=0
+)
+```
+
+## Requirements
+
+- Python 3.7+
+- See `pyproject.toml` for full dependency list
+
+## License
+
+MIT
+
+## Contributing
+
+Contributions are welcome! Please open an issue or submit a pull request.
+

move_data-0.1.5.dist-info/RECORD

@@ -0,0 +1,7 @@
+move_data/__init__.py,sha256=3w0EZT8rFCSk4pN5uw1pNmG4wtPHY77TtYH4ecaelXQ,498
+move_data/move_data.py,sha256=VqqNJ8lCDUrO2hF4H-XclrOR5XuhvB8AGMeMl4JDeDw,10799
+move_data/setup.py,sha256=s-pIY-SDMvjDziuYfcNFM6ekadchCjBdpdJd6IHN9u4,1265
+move_data-0.1.5.dist-info/METADATA,sha256=VCXT8UkD3-vq3kG85iM_fV89o1eAK5s_SO1N3o9veX0,3620
+move_data-0.1.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+move_data-0.1.5.dist-info/top_level.txt,sha256=x3YHlcqp8uYWyjg_u22_o646HG9BaUjGKVb9aWDS2FU,10
+move_data-0.1.5.dist-info/RECORD,,

move_data-0.1.5.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

move_data-0.1.5.dist-info/top_level.txt

@@ -0,0 +1 @@
+move_data