ccfx 0.8.0__py3-none-any.whl → 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ccfx/__init__.py +1 -1
- ccfx/ccfx.py +216 -6
- ccfx-1.0.0.dist-info/METADATA +180 -0
- ccfx-1.0.0.dist-info/RECORD +11 -0
- {ccfx-0.8.0.dist-info → ccfx-1.0.0.dist-info}/WHEEL +1 -1
- {ccfx-0.8.0.dist-info → ccfx-1.0.0.dist-info}/licenses/LICENSE +2 -2
- ccfx-0.8.0.dist-info/METADATA +0 -150
- ccfx-0.8.0.dist-info/RECORD +0 -11
- {ccfx-0.8.0.dist-info → ccfx-1.0.0.dist-info}/top_level.txt +0 -0
ccfx/__init__.py
CHANGED
ccfx/ccfx.py
CHANGED
@@ -14,7 +14,7 @@ import os, sys
|
|
14
14
|
import glob
|
15
15
|
import warnings
|
16
16
|
from netCDF4 import Dataset
|
17
|
-
from osgeo import gdal, osr
|
17
|
+
from osgeo import gdal, ogr, osr
|
18
18
|
import numpy
|
19
19
|
from genericpath import exists
|
20
20
|
import shutil
|
@@ -23,7 +23,6 @@ import pickle
|
|
23
23
|
import time
|
24
24
|
from shapely.geometry import box, Point
|
25
25
|
import geopandas, pandas
|
26
|
-
from osgeo import gdal, ogr, osr
|
27
26
|
import py7zr
|
28
27
|
import subprocess
|
29
28
|
import multiprocessing
|
@@ -33,6 +32,8 @@ from concurrent.futures import ThreadPoolExecutor
|
|
33
32
|
import math
|
34
33
|
import requests
|
35
34
|
from tqdm import tqdm
|
35
|
+
import yt_dlp
|
36
|
+
from typing import Optional
|
36
37
|
|
37
38
|
# functions
|
38
39
|
def listFiles(path: str, ext: str = None) -> list:
|
@@ -119,8 +120,8 @@ def getMp3Metadata(fn, imagePath=None):
|
|
119
120
|
'year': "Unknown Year",
|
120
121
|
'genre': "Unknown Genre"
|
121
122
|
}
|
122
|
-
|
123
123
|
return metadata
|
124
|
+
|
124
125
|
|
125
126
|
def guessMimeType(imagePath):
|
126
127
|
ext = os.path.splitext(imagePath.lower())[1]
|
@@ -130,6 +131,111 @@ def guessMimeType(imagePath):
|
|
130
131
|
return 'image/png'
|
131
132
|
return 'image/png'
|
132
133
|
|
134
|
+
|
135
|
+
def downloadYoutubeVideo(url: str, dstDir: str, audioOnly: bool = False, cookiesFile: str = None, dstFileName: Optional[str] = None ) -> str:
|
136
|
+
"""
|
137
|
+
Download from YouTube via yt-dlp.
|
138
|
+
|
139
|
+
Args:
|
140
|
+
url: YouTube URL
|
141
|
+
dstDir: output directory (created if missing)
|
142
|
+
audioOnly: if True, extract MP3
|
143
|
+
dstFileName: exact filename (with extension). If None, uses title.
|
144
|
+
|
145
|
+
Returns: Full path to downloaded file.
|
146
|
+
"""
|
147
|
+
os.makedirs(dstDir, exist_ok=True)
|
148
|
+
|
149
|
+
if dstFileName is None:
|
150
|
+
template = os.path.join(dstDir, "%(title)s.%(ext)s")
|
151
|
+
else:
|
152
|
+
template = os.path.join(dstDir, dstFileName)
|
153
|
+
|
154
|
+
opts = {"outtmpl": template}
|
155
|
+
|
156
|
+
if cookiesFile:
|
157
|
+
opts["cookiefile"] = cookiesFile
|
158
|
+
|
159
|
+
if audioOnly:
|
160
|
+
opts.update({
|
161
|
+
"format": "bestaudio/best",
|
162
|
+
"postprocessors": [{
|
163
|
+
"key": "FFmpegExtractAudio",
|
164
|
+
"preferredcodec": "mp3",
|
165
|
+
"preferredquality": "192",
|
166
|
+
}],
|
167
|
+
})
|
168
|
+
|
169
|
+
else:
|
170
|
+
# prefer a single MP4 file (progressive), fallback to any best if none
|
171
|
+
opts["format"] = "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best"
|
172
|
+
|
173
|
+
with yt_dlp.YoutubeDL(opts) as ytdl:
|
174
|
+
info = ytdl.extract_info(url, download=True)
|
175
|
+
|
176
|
+
# determine final filename
|
177
|
+
if dstFileName:
|
178
|
+
final = dstFileName
|
179
|
+
else:
|
180
|
+
ext = "mp3" if audioOnly else info.get("ext", "mp4")
|
181
|
+
title = info.get("title", "video")
|
182
|
+
final = f"{title}.{ext}"
|
183
|
+
|
184
|
+
return os.path.join(dstDir, final)
|
185
|
+
|
186
|
+
|
187
|
+
def parseYoutubePlaylist(playlistUrl: str) -> list[str]:
|
188
|
+
"""
|
189
|
+
Return a list of full video URLs contained in a YouTube playlist.
|
190
|
+
|
191
|
+
Args:
|
192
|
+
playlistUrl: Full URL of the playlist (the one with &list=… or /playlist?list=…).
|
193
|
+
|
194
|
+
Returns:
|
195
|
+
List of video URLs in the order reported by YouTube.
|
196
|
+
"""
|
197
|
+
opts = {
|
198
|
+
"quiet": True,
|
199
|
+
"extract_flat": "in_playlist", # don’t recurse into each video
|
200
|
+
}
|
201
|
+
|
202
|
+
with yt_dlp.YoutubeDL(opts) as ytdl:
|
203
|
+
info = ytdl.extract_info(playlistUrl, download=False)
|
204
|
+
|
205
|
+
entries = info.get("entries", [])
|
206
|
+
return [f"https://www.youtube.com/watch?v={e['id']}" for e in entries if e.get("id")]
|
207
|
+
|
208
|
+
|
209
|
+
def parseYoutubeChannelVideos(channelUrl: str, maxItems: Optional[int] = None) -> list[str]:
|
210
|
+
"""
|
211
|
+
Return a list of video URLs published on a channel.
|
212
|
+
|
213
|
+
Args:
|
214
|
+
channelUrl: Any canonical channel URL, e.g.
|
215
|
+
- https://www.youtube.com/@LinusTechTips
|
216
|
+
- https://www.youtube.com/channel/UCXuqSBlHAE6Xw-yeJA0Tunw
|
217
|
+
- https://www.youtube.com/c/NASA/videos
|
218
|
+
maxItems: Optional hard limit. If None, returns every video the API exposes.
|
219
|
+
|
220
|
+
Returns:
|
221
|
+
List of video URLs, newest-first (YouTube’s default order).
|
222
|
+
"""
|
223
|
+
opts = {
|
224
|
+
"quiet": True,
|
225
|
+
"extract_flat": True, # treat the channel as one big “playlist”
|
226
|
+
"skip_download": True,
|
227
|
+
}
|
228
|
+
|
229
|
+
with yt_dlp.YoutubeDL(opts) as ytdl:
|
230
|
+
info = ytdl.extract_info(channelUrl, download=False)
|
231
|
+
|
232
|
+
entries = info.get("entries", [])
|
233
|
+
if maxItems is not None:
|
234
|
+
entries = entries[:maxItems]
|
235
|
+
|
236
|
+
return [f"https://www.youtube.com/watch?v={e['id']}" for e in entries if e.get("id")]
|
237
|
+
|
238
|
+
|
133
239
|
def setMp3Metadata(fn, metadata, imagePath=None):
|
134
240
|
'''
|
135
241
|
This function takes a path to an mp3 and a metadata dictionary,
|
@@ -225,6 +331,39 @@ def deleteFile(filePath:str, v:bool = False) -> bool:
|
|
225
331
|
|
226
332
|
return deleted
|
227
333
|
|
334
|
+
|
335
|
+
def alert(message:str, server:str = "http://ntfy.sh", topic:str = "pythonAlerts", attachment:str = None, messageTitle:str = "info", priority:int = None, tags:list = [], printIt:bool = True, v:bool = False) -> bool:
|
336
|
+
'''
|
337
|
+
This sends an alert to a given server in case you want to be notified of something
|
338
|
+
message : the message to send
|
339
|
+
server : the server to send the message to (default is http://ntfy.sh)
|
340
|
+
topic : the topic to send the message to (default is pythonAlerts)
|
341
|
+
attachment : a file to attach to the message (optional)
|
342
|
+
messageTitle : the title of the message (optional, default is info)
|
343
|
+
priority : the priority of the message (optional, default is None)
|
344
|
+
tags : a list of tags to add to the message (optional, default is empty list)
|
345
|
+
printIt : whether to print the message to the console (default is True)
|
346
|
+
v : verbose (default is False, set to True to print debug info)
|
347
|
+
|
348
|
+
return: True if the alert was sent successfully, False otherwise
|
349
|
+
'''
|
350
|
+
print(message) if printIt else None; header_data = {}
|
351
|
+
if not messageTitle is None: header_data["Title"] = messageTitle
|
352
|
+
if not priority is None: header_data["Priority"] = priority
|
353
|
+
if not len(tags) == 0: header_data["Tags"] = ",".join(tags)
|
354
|
+
|
355
|
+
try:
|
356
|
+
if v: print(f"sending alert to {server}/{topic}")
|
357
|
+
if not attachment is None:
|
358
|
+
header_data["Filename"] = getFileBaseName(attachment)
|
359
|
+
requests.put( f"{server}/{topic}", data=open(attachment, 'rb'), headers=header_data )
|
360
|
+
return True
|
361
|
+
try: requests.post(f"{server}/{topic}",data=message, headers=header_data )
|
362
|
+
except: return False
|
363
|
+
except: return False
|
364
|
+
return True
|
365
|
+
|
366
|
+
|
228
367
|
def deletePath(path:str, v:bool = False) -> bool:
|
229
368
|
'''
|
230
369
|
Delete a directory
|
@@ -258,7 +397,39 @@ def downloadChunk(url, start, end, path):
|
|
258
397
|
if chunk:
|
259
398
|
f.write(chunk)
|
260
399
|
|
261
|
-
|
400
|
+
|
401
|
+
def formatStringBlock(input_str, max_chars=70):
|
402
|
+
'''
|
403
|
+
This function takes a string and formats it into a block of text
|
404
|
+
with a maximum number of characters per line.
|
405
|
+
|
406
|
+
input_str: the string to format
|
407
|
+
max_chars: the maximum number of characters per line (default is 70)
|
408
|
+
|
409
|
+
'''
|
410
|
+
words = input_str.split(' ')
|
411
|
+
lines = []
|
412
|
+
current_line = ""
|
413
|
+
|
414
|
+
for word in words:
|
415
|
+
# If adding the next word to the current line would exceed the max_chars limit
|
416
|
+
if len(current_line) + len(word) > max_chars:
|
417
|
+
# Append current line to lines and start a new one
|
418
|
+
lines.append(current_line.strip())
|
419
|
+
current_line = word
|
420
|
+
else:
|
421
|
+
# Add the word to the current line
|
422
|
+
current_line += " " + word
|
423
|
+
|
424
|
+
# Append any remaining words
|
425
|
+
lines.append(current_line.strip())
|
426
|
+
|
427
|
+
return '\n'.join(lines)
|
428
|
+
|
429
|
+
|
430
|
+
|
431
|
+
|
432
|
+
def downloadFile(url, save_path, exists_action='resume', num_connections=5, v=False):
|
262
433
|
if v:
|
263
434
|
print(f"\ndownloading {url}")
|
264
435
|
fname = getFileBaseName(url, extension=True)
|
@@ -592,21 +763,45 @@ def renameNetCDFvariable(input_file: str, output_file: str, old_var_name: str, n
|
|
592
763
|
except subprocess.CalledProcessError as e:
|
593
764
|
print(f"Error: {e.stderr}")
|
594
765
|
|
595
|
-
|
766
|
+
|
767
|
+
def compressTo7z(input_dir: str, output_file: str, compressionLevel: int = 4, excludeExt: list = None, v: bool = False) -> None:
|
596
768
|
"""
|
597
769
|
Compresses the contents of a directory to a .7z archive with maximum compression.
|
598
770
|
|
599
771
|
:param input_dir: Path to the directory to compress
|
600
772
|
:param output_file: Output .7z file path
|
773
|
+
:param compressionLevel: Compression level (0-9), default is 4 (maximum compression)
|
774
|
+
:param excludeExt: List of file extensions to exclude from compression
|
601
775
|
"""
|
776
|
+
if excludeExt is None:
|
777
|
+
excludeExt = []
|
778
|
+
|
602
779
|
# Create the .7z archive with LZMA2 compression
|
603
|
-
with py7zr.SevenZipFile(output_file, 'w', filters=[{'id': py7zr.FILTER_LZMA2, 'preset':
|
780
|
+
with py7zr.SevenZipFile(output_file, 'w', filters=[{'id': py7zr.FILTER_LZMA2, 'preset': compressionLevel}]) as archive:
|
604
781
|
# Add each item in the input directory, avoiding the top-level folder in the archive
|
605
782
|
for root, _, files in os.walk(input_dir):
|
606
783
|
for file in files:
|
607
784
|
file_path = os.path.join(root, file)
|
785
|
+
|
786
|
+
# Skip excluded file extensions
|
787
|
+
if any(file.endswith(ext) for ext in excludeExt):
|
788
|
+
continue
|
608
789
|
# Add file to the archive with a relative path to avoid including the 'tmp' folder itself
|
609
790
|
archive.write(file_path, arcname=os.path.relpath(file_path, start=input_dir))
|
791
|
+
if v:
|
792
|
+
print(f"compressed {input_dir} to {output_file} with compression level {compressionLevel}.")
|
793
|
+
|
794
|
+
|
795
|
+
def uncompress(inputFile: str, outputDir: str, v: bool = False) -> None:
|
796
|
+
"""
|
797
|
+
Extracts an archive supported by py7zr (.7z, .zip, .tar, .tar.gz, .tar.bz2, .xz, .tar.xz) to outputDir.
|
798
|
+
inputFile: Path to the input archive file
|
799
|
+
outputDir: Directory where the contents will be extracted
|
800
|
+
v: Verbose flag to print extraction status (default is False)
|
801
|
+
"""
|
802
|
+
if not exists(outputDir): createPath(outputDir)
|
803
|
+
with py7zr.SevenZipFile(inputFile, 'r') as archive: archive.extractall(path=outputDir)
|
804
|
+
if v: print(f"extracted {inputFile} to {outputDir}.")
|
610
805
|
|
611
806
|
|
612
807
|
def moveDirectory(srcDir:str, destDir:str, v:bool = False) -> bool:
|
@@ -693,6 +888,21 @@ def clipRasterByExtent(inFile: str, outFile: str, bounds: tuple) -> str:
|
|
693
888
|
ds = None
|
694
889
|
return outFile
|
695
890
|
|
891
|
+
|
892
|
+
def clipRasterByVector(inFile: str, outFile: str, vectorFile: str) -> str:
|
893
|
+
'''
|
894
|
+
Clips a raster using GDAL warp with a vector file
|
895
|
+
inFile: input raster path
|
896
|
+
outFile: output path
|
897
|
+
vectorFile: vector file path (e.g., shapefile or GeoJSON)
|
898
|
+
return: output path
|
899
|
+
'''
|
900
|
+
ds = gdal.Open(inFile)
|
901
|
+
gdal.Warp(outFile, ds, cutlineDSName=vectorFile, cropToCutline=True)
|
902
|
+
ds = None
|
903
|
+
return outFile
|
904
|
+
|
905
|
+
|
696
906
|
def clipVectorByExtent(inFile: str, outFile: str, bounds: tuple) -> str:
|
697
907
|
'''
|
698
908
|
Clips a vector using GeoPandas
|
@@ -0,0 +1,180 @@
|
|
1
|
+
Metadata-Version: 2.4
|
2
|
+
Name: ccfx
|
3
|
+
Version: 1.0.0
|
4
|
+
Summary: This package simplifies regular common actions for quick prototyping in a user friendly way
|
5
|
+
Author-email: Celray James CHAWANDA <celray@chawanda.com>
|
6
|
+
License-Expression: MIT
|
7
|
+
Project-URL: Homepage, https://github.com/celray/ccfx
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
9
|
+
Classifier: Operating System :: OS Independent
|
10
|
+
Requires-Python: >=3.10
|
11
|
+
Description-Content-Type: text/markdown
|
12
|
+
License-File: LICENSE
|
13
|
+
Requires-Dist: netCDF4
|
14
|
+
Requires-Dist: yt_dlp
|
15
|
+
Requires-Dist: gdal
|
16
|
+
Requires-Dist: numpy
|
17
|
+
Requires-Dist: shapely
|
18
|
+
Requires-Dist: geopandas
|
19
|
+
Requires-Dist: pandas
|
20
|
+
Requires-Dist: xlsxwriter
|
21
|
+
Requires-Dist: pyodbc
|
22
|
+
Requires-Dist: sqlalchemy
|
23
|
+
Requires-Dist: python-docx
|
24
|
+
Requires-Dist: py7zr
|
25
|
+
Requires-Dist: mutagen
|
26
|
+
Requires-Dist: requests
|
27
|
+
Requires-Dist: tqdm
|
28
|
+
Dynamic: license-file
|
29
|
+
|
30
|
+
# ccfx
|
31
|
+
|
32
|
+
`ccfx` is a comprehensive Python package designed to streamline file and data management, geospatial analysis, NetCDF file processing, database interactions, document generation, and multimedia handling for rapid prototyping and development workflows.
|
33
|
+
|
34
|
+
## Features
|
35
|
+
|
36
|
+
1. **File Management**:
|
37
|
+
* List, delete, move, copy, and count files/directories.
|
38
|
+
* Monitor file count over time.
|
39
|
+
* Save, load, and manage Python variables via pickle serialization.
|
40
|
+
* Compress directories to `.7z` archives.
|
41
|
+
* Read/write text files with encoding support.
|
42
|
+
* Download files from URLs with resume and multi-connection support.
|
43
|
+
|
44
|
+
2. **Geospatial Data Processing**:
|
45
|
+
* Read, write, clip (by extent/feature), resample, reproject, merge, and rasterize raster data (GeoTIFF, NetCDF).
|
46
|
+
* Read, write, and clip vector data (Shapefile, GeoPackage).
|
47
|
+
* Create grids of polygons based on shapefile boundaries.
|
48
|
+
* Convert coordinates between Coordinate Reference Systems (CRS).
|
49
|
+
* Extract raster values at specific coordinates.
|
50
|
+
* Convert point lists to GeoDataFrames.
|
51
|
+
* Get vector layer bounds.
|
52
|
+
|
53
|
+
3. **NetCDF File Handling**:
|
54
|
+
* List variables and dimensions.
|
55
|
+
* Export NetCDF variables to GeoTIFF format (single or multiple bands).
|
56
|
+
* Calculate sum and average maps from NetCDF data across multiple files.
|
57
|
+
* Rename variables using CDO (if available).
|
58
|
+
|
59
|
+
4. **Database Connectivity**:
|
60
|
+
* **MS SQL Server**: Connect, list databases/tables, read tables (including spatial data into GeoDataFrames), write DataFrames/GeoDataFrames to tables, drop tables.
|
61
|
+
* **SQLite**: Connect, create/rename/drop tables, read tables (as dict, specific columns), insert data (rows, dicts, partial dicts), update values, dump tables to CSV.
|
62
|
+
|
63
|
+
5. **Document & Spreadsheet Handling**:
|
64
|
+
* **Excel**: Create `.xlsx` files, add sheets, write data (including dates), set column widths, add scatter plot charts.
|
65
|
+
* **Word**: Create `.docx` files, add headings, paragraphs (with alignment), list items, formatted text (bold/italic), images, page breaks, set margins.
|
66
|
+
|
67
|
+
6. **Multimedia & Web**:
|
68
|
+
* Read and write MP3 metadata (ID3 tags), including album art.
|
69
|
+
* Download videos/audio from YouTube using `yt-dlp`.
|
70
|
+
|
71
|
+
7. **Data Analysis & Utilities**:
|
72
|
+
* Calculate timeseries statistics (NSE, KGE, PBIAS, LNSE, R2, RMSE, MAE, MSE, MAPE, alpha, beta) with resampling options.
|
73
|
+
* Display dynamic progress bars.
|
74
|
+
* Check system platform information.
|
75
|
+
* Enable or disable warnings programmatically.
|
76
|
+
* Set the working directory.
|
77
|
+
|
78
|
+
## Installation
|
79
|
+
|
80
|
+
Install `ccfx` via pip:
|
81
|
+
```bash
|
82
|
+
pip install ccfx
|
83
|
+
```
|
84
|
+
|
85
|
+
## Dependencies
|
86
|
+
|
87
|
+
`ccfx` relies on the following libraries:
|
88
|
+
|
89
|
+
* **gdal**: For geospatial raster and vector data manipulation.
|
90
|
+
* **numpy**: For array processing and numerical operations.
|
91
|
+
* **pandas**: For data manipulation and analysis.
|
92
|
+
* **geopandas**: Extends pandas to handle geospatial vector data.
|
93
|
+
* **shapely**: Provides geometric objects and operations.
|
94
|
+
* **netCDF4**: For working with NetCDF files.
|
95
|
+
* **xlsxwriter**: For creating and writing Excel `.xlsx` files.
|
96
|
+
* **python-docx**: Enables creation and manipulation of Word `.docx` documents.
|
97
|
+
* **pyodbc**: Enables connectivity to databases through ODBC (e.g., MS SQL Server).
|
98
|
+
* **sqlalchemy**: Provides SQL toolkit and ORM features for database access (used with MS SQL).
|
99
|
+
* **py7zr**: For creating `.7z` archives.
|
100
|
+
* **mutagen**: For reading and writing MP3 metadata (ID3 tags).
|
101
|
+
* **requests**: For downloading files via HTTP.
|
102
|
+
* **tqdm**: For displaying progress bars.
|
103
|
+
* **yt-dlp**: For downloading YouTube content.
|
104
|
+
* **matplotlib** (Optional, often used with geospatial/data analysis): For plotting.
|
105
|
+
|
106
|
+
These dependencies should be installed automatically when `ccfx` is installed via pip, but GDAL might require manual installation steps depending on your OS.
|
107
|
+
|
108
|
+
## API Reference (Selected Functions)
|
109
|
+
|
110
|
+
### File Management (`ccfx.py`)
|
111
|
+
|
112
|
+
* **`listFiles(path: str, ext: str = None) -> list`**: Lists files in a directory, optionally filtering by extension.
|
113
|
+
* **`deleteFile(filePath: str, v: bool = False) -> bool`**: Deletes a specified file.
|
114
|
+
* **`deletePath(path: str, v: bool = False) -> bool`**: Deletes a directory and its contents.
|
115
|
+
* **`createPath(pathName, v = False)`**: Creates a directory path if it doesn't exist.
|
116
|
+
* **`copyFile(source: str, destination: str, v: bool = True)`**: Copies a single file.
|
117
|
+
* **`copyDirectory(source: str, destination: str, recursive=True, v=True, filter=[])`**: Copies a directory's contents.
|
118
|
+
* **`moveDirectoryFiles(srcDir: str, destDir: str, v: bool = False) -> bool`**: Moves files and subdirectories from source to destination.
|
119
|
+
* **`pythonVariable(filename, option, variable=None)`**: Saves ('dump') or loads ('load') Python variables using pickle.
|
120
|
+
* **`compressTo7z(input_dir: str, output_file: str)`**: Compresses a directory into a .7z file.
|
121
|
+
* **`downloadFile(url, save_path, exists_action='resume', num_connections=5, v=False)`**: Downloads a file from a URL with advanced options.
|
122
|
+
* **`listAllFiles(folder, extension="*")`**: Recursively lists all files in a folder and its subfolders.
|
123
|
+
|
124
|
+
### Geospatial (`ccfx.py`)
|
125
|
+
|
126
|
+
* **`createGrid(shapefile_path: str, resolution: float, useDegree: bool = True) -> tuple`**: Generates a grid of polygons based on a shapefile extent.
|
127
|
+
* **`clipRasterByExtent(inFile: str, outFile: str, bounds: tuple) -> str`**: Clips a raster using bounding box coordinates.
|
128
|
+
* **`clipVectorByExtent(inFile: str, outFile: str, bounds: tuple) -> str`**: Clips a vector file using bounding box coordinates.
|
129
|
+
* **`clipFeatures(inputFeaturePath:str, boundaryFeature:str, outputFeature:str, keepOnlyTypes = None, v = False) -> geopandas.GeoDataFrame`**: Clips input features by a boundary feature.
|
130
|
+
* **`resampleRaster(inFile:str, outFile:str, resolution:float, dstSRS = None, resamplingMethod = 'bilinear', replaceOutput:bool = True, v:bool = True) -> str`**: Resamples a raster to a new resolution and optionally CRS.
|
131
|
+
* **`reprojectRaster(inFile: str, outFile: str, dstProjection: str, resamplingMethod: str = 'mode') -> str`**: Reprojects a raster to a new CRS.
|
132
|
+
* **`mergeRasterTiles(tileList:list, outFile:str) -> str`**: Merges multiple raster files into one.
|
133
|
+
* **`rasterizeRaster(inFile: str, outFile: str, targetField: str, targetResolution: float) -> str`**: Rasterizes a vector layer based on an attribute field.
|
134
|
+
* **`extractRasterValue(rasterPath: str, lat: float, lon: float, coordProj: str = 'EPSG:4326') -> float`**: Extracts the raster value at a specific point.
|
135
|
+
* **`convertCoordinates(lon, lat, srcEPSG, dstCRS) -> tuple`**: Converts coordinates between CRSs.
|
136
|
+
* **`tiffWriteArray(array: numpy.ndarray, outputFile: str, geoTransform: tuple, projection: str, noData:float = None, v:bool = False) -> gdal.Dataset`**: Writes a NumPy array to a GeoTIFF file.
|
137
|
+
* **`pointsToGeodataframe(point_pairs_list, columns = ['latitude', 'longitude'], auth = "EPSG", code = '4326', out_shape = '', format = 'gpkg', v = False, get_geometry_only = False)`**: Converts a list of point coordinates to a GeoDataFrame.
|
138
|
+
|
139
|
+
### NetCDF (`ccfx.py`)
|
140
|
+
|
141
|
+
* **`netcdfVariablesList(ncFile: str) -> list`**: Lists variables in a NetCDF file.
|
142
|
+
* **`netcdfVariableDimensions(ncFile: str, variable: str) -> dict`**: Gets dimensions and their sizes for a NetCDF variable.
|
143
|
+
* **`netcdfExportTif(ncFile: str, variable: str, outputFile: str = None, band: int = None, v:bool = True) -> gdal.Dataset`**: Exports a NetCDF variable (optionally a specific band) to GeoTIFF.
|
144
|
+
* **`netcdfAverageMap(ncFiles:list, variable:str, band:int = 1) -> numpy.ndarray`**: Calculates the average map from a variable across multiple NetCDF files.
|
145
|
+
* **`netcdfSumMaps(ncFiles:list, variable:str, band:int = 1) -> numpy.ndarray`**: Calculates the sum map from a variable across multiple NetCDF files.
|
146
|
+
* **`renameNetCDFvariable(input_file: str, output_file: str, old_var_name: str, new_var_name: str, v = False)`**: Renames a variable in a NetCDF file using CDO.
|
147
|
+
|
148
|
+
### Database (`mssqlConnection.py`, `sqliteConnection.py`)
|
149
|
+
|
150
|
+
* **`mssql_connection(server, username, password, driver, ...)`**: Class for MS SQL Server interactions.
|
151
|
+
* `connect()`, `listDatabases()`, `listTables()`, `readTable()`, `connectDB()`, `dataframeToSql()`, `dropTable()`, `close()`
|
152
|
+
* **`sqliteConnection(sqlite_database, connect=False)`**: Class for SQLite interactions.
|
153
|
+
* `connect()`, `createTable()`, `renameTable()`, `deleteTable()`, `readTableAsDict()`, `insertDict()`, `insertRow()`, `updateValue()`, `dumpCSV()`, `commitChanges()`, `closeConnection()`
|
154
|
+
|
155
|
+
### Document/Spreadsheet (`word.py`, `excel.py`)
|
156
|
+
|
157
|
+
* **`word_document(path)`**: Class for creating Word documents.
|
158
|
+
* `addHeading()`, `addParagraph()`, `addListItem()`, `addText()`, `addImage()`, `addPageBreak()`, `setMargins()`, `save()`
|
159
|
+
* **`excel(path)`**: Class for creating Excel spreadsheets.
|
160
|
+
* `create()`, `addSheet()`, `write()`, `writeDate()`, `setColumnWidth()`, `addFigure()`, `writeColumn()`, `save()`, `open()`
|
161
|
+
|
162
|
+
### Multimedia & Web (`ccfx.py`)
|
163
|
+
|
164
|
+
* **`getMp3Metadata(fn, imagePath=None)`**: Extracts ID3 metadata from an MP3 file.
|
165
|
+
* **`setMp3Metadata(fn, metadata, imagePath=None)`**: Writes ID3 metadata (including album art) to an MP3 file.
|
166
|
+
* **`downloadYoutubeVideo(url: str, dstDir: str, audioOnly: bool = False, dstFileName: Optional[str] = None ) -> str`**: Downloads video or audio from a YouTube URL.
|
167
|
+
|
168
|
+
### Data Analysis & Utilities (`ccfx.py`)
|
169
|
+
|
170
|
+
* **`calculateTimeseriesStats(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> dict`**: Calculates various statistics between observed and simulated timeseries. (Wrappers like `getNSE`, `getKGE`, etc., are also available).
|
171
|
+
* **`progressBar(count, total, message="")`**: Displays a simple console progress bar.
|
172
|
+
* **`showProgress(count: int, end: int, message: str, barLength: int = 100)`**: Displays a more detailed console progress bar.
|
173
|
+
* **`ignoreWarnings(ignore:bool = True, v:bool = False)`**: Suppresses or enables Python warnings.
|
174
|
+
|
175
|
+
## Contributing
|
176
|
+
|
177
|
+
Contributions are welcome! Please fork the repository, make your changes, and submit a pull request. Ensure code is well-documented and includes tests where applicable.
|
178
|
+
|
179
|
+
## License
|
180
|
+
This project is licensed under the MIT License. See the LICENSE file for details.
|
@@ -0,0 +1,11 @@
|
|
1
|
+
ccfx/__init__.py,sha256=UK62VcGS84SJyGVg1bK4FltZj7OkpdoyhoFWeXcKsX0,144
|
2
|
+
ccfx/ccfx.py,sha256=hUeU8i3v3Y5PTfHhAfXNeoRjacdeWrYUeznwArXsEOk,64046
|
3
|
+
ccfx/excel.py,sha256=cQ4TQW49XqbMB3sSS0IOhO3-WArIolEBIrvOvhFyPtI,4757
|
4
|
+
ccfx/mssqlConnection.py,sha256=TwyZXhHHI7zy6BSfH1pszuHVJ5cmndRC5dVxvEtSTks,7904
|
5
|
+
ccfx/sqliteConnection.py,sha256=jEJ94D5ySt84N7AeDpa27Rclt1NaKhkX6nYzidwApIg,11104
|
6
|
+
ccfx/word.py,sha256=AGa64jX5Zl5qotZh5L0QmrsjTnktIBhmj_ByRKZ88vw,3061
|
7
|
+
ccfx-1.0.0.dist-info/licenses/LICENSE,sha256=EuxaawJg_OOCLfikkCGgfXPZmxR-x_5PH7_2e9M-3eA,1099
|
8
|
+
ccfx-1.0.0.dist-info/METADATA,sha256=Uoy4Br0JKKZxW8cd9Bj-dguujuMX2CRRVbiZ_lk-8QQ,11260
|
9
|
+
ccfx-1.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
10
|
+
ccfx-1.0.0.dist-info/top_level.txt,sha256=_cSvSA1WX2K8TgoV3iBJUdUZZqMKJbOPLNnKLYSLHaw,5
|
11
|
+
ccfx-1.0.0.dist-info/RECORD,,
|
@@ -1,6 +1,6 @@
|
|
1
1
|
MIT License
|
2
2
|
|
3
|
-
Copyright (c)
|
3
|
+
Copyright (c) 2025 Celray James CHAWANDA
|
4
4
|
|
5
5
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
6
|
of this software and associated documentation files (the "Software"), to deal
|
@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
18
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
19
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
20
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
-
SOFTWARE.
|
21
|
+
SOFTWARE.
|
ccfx-0.8.0.dist-info/METADATA
DELETED
@@ -1,150 +0,0 @@
|
|
1
|
-
Metadata-Version: 2.4
|
2
|
-
Name: ccfx
|
3
|
-
Version: 0.8.0
|
4
|
-
Summary: This package simplifies regular common actions for quick prototyping in a user friendly way
|
5
|
-
Author-email: Celray James CHAWANDA <celray@chawanda.com>
|
6
|
-
License: MIT
|
7
|
-
Project-URL: Homepage, https://github.com/celray/ccfx
|
8
|
-
Classifier: Programming Language :: Python :: 3
|
9
|
-
Classifier: License :: OSI Approved :: MIT License
|
10
|
-
Classifier: Operating System :: OS Independent
|
11
|
-
Requires-Python: >=3.10
|
12
|
-
Description-Content-Type: text/markdown
|
13
|
-
License-File: LICENSE
|
14
|
-
Requires-Dist: netCDF4
|
15
|
-
Requires-Dist: gdal
|
16
|
-
Requires-Dist: numpy
|
17
|
-
Requires-Dist: shapely
|
18
|
-
Requires-Dist: geopandas
|
19
|
-
Requires-Dist: pandas
|
20
|
-
Requires-Dist: xlsxwriter
|
21
|
-
Requires-Dist: pyodbc
|
22
|
-
Requires-Dist: sqlalchemy
|
23
|
-
Requires-Dist: python-docx
|
24
|
-
Requires-Dist: py7zr
|
25
|
-
Requires-Dist: mutagen
|
26
|
-
Requires-Dist: requests
|
27
|
-
Requires-Dist: tqdm
|
28
|
-
Dynamic: license-file
|
29
|
-
|
30
|
-
# ccfx
|
31
|
-
|
32
|
-
`ccfx` is a comprehensive Python package designed to streamline file and data management, geospatial analysis, and NetCDF file processing for quick prototyping. The library provides versatile tools for file handling, raster and vector manipulation, database connectivity, and data export for geospatial and scientific computing projects.
|
33
|
-
|
34
|
-
## Features
|
35
|
-
|
36
|
-
1. **File Management**:
|
37
|
-
- List, delete, move, and count files within directories.
|
38
|
-
- Monitor file count over time in a specific directory.
|
39
|
-
- Save, load, and manage Python variables via pickle serialization.
|
40
|
-
|
41
|
-
2. **Geospatial Data Processing**:
|
42
|
-
- Read, write, and manage raster and vector geospatial data.
|
43
|
-
- Clip rasters by bounding boxes and extract raster values at specified coordinates.
|
44
|
-
- Create grids of polygons based on shapefile boundaries with user-defined resolutions.
|
45
|
-
- Convert coordinates between coordinate reference systems (CRS).
|
46
|
-
- Write NumPy arrays to GeoTIFF files with projection and geotransform settings.
|
47
|
-
|
48
|
-
3. **NetCDF File Handling**:
|
49
|
-
- List available variables and dimensions in NetCDF files.
|
50
|
-
- Export NetCDF variables to GeoTIFF format.
|
51
|
-
- Calculate sum and average maps from NetCDF data across multiple files.
|
52
|
-
|
53
|
-
4. **Database Connectivity**:
|
54
|
-
- Access and interact with databases using ODBC and SQLAlchemy for flexible database management.
|
55
|
-
- Connect to both SQL Server and SQLite databases.
|
56
|
-
|
57
|
-
5. **Progress Tracking and System Info**:
|
58
|
-
- Display dynamic progress bars for long-running operations.
|
59
|
-
- Check the system’s platform information.
|
60
|
-
- Enable or disable warnings programmatically.
|
61
|
-
|
62
|
-
6. **Excel and Word File Handling**:
|
63
|
-
- Create and modify Excel files using xlsxwriter.
|
64
|
-
- Generate Word documents with advanced formatting options using python-docx.
|
65
|
-
|
66
|
-
## Installation
|
67
|
-
|
68
|
-
Install `ccfx` via pip:
|
69
|
-
```bash
|
70
|
-
pip install ccfx
|
71
|
-
```
|
72
|
-
|
73
|
-
## Dependencies
|
74
|
-
|
75
|
-
`ccfx` relies on the following libraries:
|
76
|
-
|
77
|
-
- **netCDF4**: For working with NetCDF files.
|
78
|
-
- **gdal**: Required for geospatial raster data manipulation.
|
79
|
-
- **numpy**: For array processing and numerical operations.
|
80
|
-
- **pandas**: For data manipulation and analysis.
|
81
|
-
- **shapely**: Provides geometric operations for spatial data.
|
82
|
-
- **geopandas**: Extends pandas to handle geospatial data.
|
83
|
-
- **xlsxwriter**: For creating and writing Excel files.
|
84
|
-
- **pyodbc**: Enables connectivity to databases through ODBC.
|
85
|
-
- **sqlalchemy**: Provides SQL toolkit and ORM features for database access.
|
86
|
-
- **python-docx**: Enables creation and manipulation of Word documents.
|
87
|
-
|
88
|
-
These dependencies will be installed automatically when `ccfx` is installed.
|
89
|
-
|
90
|
-
## API Reference
|
91
|
-
|
92
|
-
### `listFiles(path: str, ext: str = None) -> list`
|
93
|
-
Lists all files in a directory with a specified extension.
|
94
|
-
|
95
|
-
- **Parameters**:
|
96
|
-
- `path` (str): The directory to search.
|
97
|
-
- `ext` (str, optional): File extension to filter by, e.g., `'txt'`, `'.txt'`, `'*txt'`, or `'*.txt'`.
|
98
|
-
|
99
|
-
- **Returns**:
|
100
|
-
- `list`: A list of file paths matching the criteria.
|
101
|
-
|
102
|
-
### `deleteFile(filePath: str, v: bool = False) -> bool`
|
103
|
-
Deletes a specified file with optional verbose output.
|
104
|
-
|
105
|
-
- **Parameters**:
|
106
|
-
- `filePath` (str): Path to the file to be deleted.
|
107
|
-
- `v` (bool, optional): If `True`, prints a confirmation message. Defaults to `False`.
|
108
|
-
|
109
|
-
- **Returns**:
|
110
|
-
- `bool`: `True` if deletion was successful; `False` otherwise.
|
111
|
-
|
112
|
-
### `createGrid(shapefile_path: str, resolution: float, useDegree: bool = True) -> tuple`
|
113
|
-
Generates a grid of polygons from a shapefile at a given resolution.
|
114
|
-
|
115
|
-
- **Parameters**:
|
116
|
-
- `shapefile_path` (str): Path to the shapefile.
|
117
|
-
- `resolution` (float): Resolution of the grid.
|
118
|
-
- `useDegree` (bool, optional): If `True`, coordinates are in degrees. Defaults to `True`.
|
119
|
-
|
120
|
-
- **Returns**:
|
121
|
-
- `tuple`: Contains grid coordinates and metadata.
|
122
|
-
|
123
|
-
### `clipRasterByExtent(inFile: str, outFile: str, bounds: tuple) -> str`
|
124
|
-
Clips a raster to specified bounding box coordinates.
|
125
|
-
|
126
|
-
- **Parameters**:
|
127
|
-
- `inFile` (str): Path to the input raster file.
|
128
|
-
- `outFile` (str): Path to the output clipped raster file.
|
129
|
-
- `bounds` (tuple): Bounding box as `(minx, miny, maxx, maxy)`.
|
130
|
-
|
131
|
-
- **Returns**:
|
132
|
-
- `str`: Path to the clipped raster file.
|
133
|
-
|
134
|
-
### `netcdfVariablesList(ncFile: str) -> list`
|
135
|
-
Lists all variables in a NetCDF file.
|
136
|
-
|
137
|
-
- **Parameters**:
|
138
|
-
- `ncFile` (str): Path to the NetCDF file.
|
139
|
-
|
140
|
-
- **Returns**:
|
141
|
-
- `list`: A list of variable names in the file.
|
142
|
-
|
143
|
-
### ... And More ...
|
144
|
-
|
145
|
-
## Contributing
|
146
|
-
|
147
|
-
Contributions are welcome! Please fork the repository, make your changes, and submit a pull request.
|
148
|
-
|
149
|
-
## License
|
150
|
-
This project is licensed under the MIT License.
|
ccfx-0.8.0.dist-info/RECORD
DELETED
@@ -1,11 +0,0 @@
|
|
1
|
-
ccfx/__init__.py,sha256=VmBeF3oj6JTJ_793d4i8PvhyF8_FxaxA1L_FmHWqitc,142
|
2
|
-
ccfx/ccfx.py,sha256=L0BeQLs4H9o8MlKlmr7pHqFmjbd5A3yDBKsk3XSzU5s,56313
|
3
|
-
ccfx/excel.py,sha256=cQ4TQW49XqbMB3sSS0IOhO3-WArIolEBIrvOvhFyPtI,4757
|
4
|
-
ccfx/mssqlConnection.py,sha256=TwyZXhHHI7zy6BSfH1pszuHVJ5cmndRC5dVxvEtSTks,7904
|
5
|
-
ccfx/sqliteConnection.py,sha256=jEJ94D5ySt84N7AeDpa27Rclt1NaKhkX6nYzidwApIg,11104
|
6
|
-
ccfx/word.py,sha256=AGa64jX5Zl5qotZh5L0QmrsjTnktIBhmj_ByRKZ88vw,3061
|
7
|
-
ccfx-0.8.0.dist-info/licenses/LICENSE,sha256=2-M3fBUS3FmrSIrqd3cZDmxXxojWVJtZY-SHSRE6RxM,1098
|
8
|
-
ccfx-0.8.0.dist-info/METADATA,sha256=1BskaEpbKDSaouiPDW9xdIlLdb62qPwwFjH2mUi9mpQ,5425
|
9
|
-
ccfx-0.8.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
|
10
|
-
ccfx-0.8.0.dist-info/top_level.txt,sha256=_cSvSA1WX2K8TgoV3iBJUdUZZqMKJbOPLNnKLYSLHaw,5
|
11
|
-
ccfx-0.8.0.dist-info/RECORD,,
|
File without changes
|