ullr-data-cli 1.0.1__tar.gz → 1.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ullr_data_cli-1.0.1 → ullr_data_cli-1.0.2}/PKG-INFO +1 -1
- {ullr_data_cli-1.0.1 → ullr_data_cli-1.0.2}/README.md +18 -0
- {ullr_data_cli-1.0.1 → ullr_data_cli-1.0.2}/setup.py +1 -1
- {ullr_data_cli-1.0.1 → ullr_data_cli-1.0.2}/ullr/main.py +383 -6
- {ullr_data_cli-1.0.1 → ullr_data_cli-1.0.2}/ullr_data_cli.egg-info/PKG-INFO +1 -1
- {ullr_data_cli-1.0.1 → ullr_data_cli-1.0.2}/setup.cfg +0 -0
- {ullr_data_cli-1.0.1 → ullr_data_cli-1.0.2}/ullr/__init__.py +0 -0
- {ullr_data_cli-1.0.1 → ullr_data_cli-1.0.2}/ullr_data_cli.egg-info/SOURCES.txt +0 -0
- {ullr_data_cli-1.0.1 → ullr_data_cli-1.0.2}/ullr_data_cli.egg-info/dependency_links.txt +0 -0
- {ullr_data_cli-1.0.1 → ullr_data_cli-1.0.2}/ullr_data_cli.egg-info/entry_points.txt +0 -0
- {ullr_data_cli-1.0.1 → ullr_data_cli-1.0.2}/ullr_data_cli.egg-info/requires.txt +0 -0
- {ullr_data_cli-1.0.1 → ullr_data_cli-1.0.2}/ullr_data_cli.egg-info/top_level.txt +0 -0
|
@@ -36,6 +36,24 @@ Turn raw numbers into a full dashboard layout in seconds. Ullr calculates your k
|
|
|
36
36
|
* **Dynamic Chart Logic:** Automatically recommends Line Charts for dates, Bar Charts for categories, and Donut Charts for binary metrics.
|
|
37
37
|
* **Multi-Platform Mentor:** Generates step-by-step click instructions on how to build the recommended dashboard in **Microsoft Excel, Power BI, Google Looker Studio, and Tableau**.
|
|
38
38
|
|
|
39
|
+
### 3. 🧹 Auto-Clean Engine (`ullr clean`)
|
|
40
|
+
Automatically fixes the errors found during the audit phase.
|
|
41
|
+
* **Ghost Purge:** Instantly deletes 100% empty columns and "Unnamed" structural columns.
|
|
42
|
+
* **Regex Stripping:** Acts as a Python `=TRIM()`, vaporizing invisible leading/trailing spaces.
|
|
43
|
+
* **Math Fixer:** Hunts down numbers trapped as text (currencies `$`, `£`, commas), rips out the symbols, and converts them to pure math-ready floats.
|
|
44
|
+
* **Multi-Format Export:** Pauses after cleaning to ask if you want to save the output as `.xlsx` (Dashboards), `.csv` (PostgreSQL bulk imports), or `.json` (Web Apps).
|
|
45
|
+
|
|
46
|
+
### 4. 🗃️ Markdown Data Dictionary & Job Simulator (`ullr map`)
|
|
47
|
+
Automatically parses your dataset to identify column data types, find Primary Keys, and generate a beautiful Markdown schema dictionary.
|
|
48
|
+
* **Virtual Manager Brief:** Reads your specific columns and generates a **Simulated Stakeholder Brief** giving you exact instructions on what KPIs to calculate and what layout to build.
|
|
49
|
+
* **Auto-Documentation:** Saves the `.md` dictionary file directly next to your dataset so you never lose context.
|
|
50
|
+
|
|
51
|
+
### 5. 🏭 Practice Data Engine (`ullr generate`)
|
|
52
|
+
A built-in data synthesizer for building your portfolio.
|
|
53
|
+
* **Synthetic Generation:** Uses the `Faker` library to instantly generate up to 10,000+ rows of realistic E-commerce, Healthcare, or Real Estate data.
|
|
54
|
+
* **The Sabotage Engine:** Intentionally breaks the perfect data by injecting NULL values, invisible spaces, and numbers trapped as text, giving you the perfect dirty dataset to practice cleaning in Excel or SQL.
|
|
55
|
+
* **Dynamic Stakeholder Brief:** Automatically analyzes the generated dataset and prints a System Requirements Document, giving you specific KPIs and Business Questions to answer for your project scope.
|
|
56
|
+
|
|
39
57
|
---
|
|
40
58
|
|
|
41
59
|
## 📖 Quick Start
|
|
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
|
|
|
2
2
|
|
|
3
3
|
setup(
|
|
4
4
|
name='ullr-data-cli',
|
|
5
|
-
version='1.0.
|
|
5
|
+
version='1.0.2',
|
|
6
6
|
author='jshlydnzl',
|
|
7
7
|
author_email='jshlydnzl@users.noreply.github.com',
|
|
8
8
|
description='An offline, AI-free Data Analytics and Dashboard Auditing CLI',
|
|
@@ -21,8 +21,11 @@ def print_banner():
|
|
|
21
21
|
"""
|
|
22
22
|
console.print(banner, style="bold cyan")
|
|
23
23
|
|
|
24
|
+
import pathlib
|
|
25
|
+
|
|
24
26
|
def find_file_globally(filename):
|
|
25
|
-
|
|
27
|
+
# pathlib.Path.home() natively handles Windows (C:\Users\Name), macOS (/Users/Name), and Linux (/home/Name)
|
|
28
|
+
home_dir = str(pathlib.Path.home())
|
|
26
29
|
|
|
27
30
|
with console.status(f"[yellow]Scanning your computer for '{filename}'...[/]", spinner="dots"):
|
|
28
31
|
for root, dirs, files in os.walk(home_dir):
|
|
@@ -317,6 +320,347 @@ def analyze_data(filepath):
|
|
|
317
320
|
console.print(f" [dim]↳ Steps: Click 'Add a control' -> Select 'Drop-down list' -> Control field: your category.[/dim]")
|
|
318
321
|
console.print("")
|
|
319
322
|
|
|
323
|
+
def clean_data(filepath):
|
|
324
|
+
if not os.path.exists(filepath):
|
|
325
|
+
found_path = find_file_globally(os.path.basename(filepath))
|
|
326
|
+
if found_path:
|
|
327
|
+
filepath = found_path
|
|
328
|
+
else:
|
|
329
|
+
console.print(f"[bold red]Error:[/] Could not find '{os.path.basename(filepath)}' anywhere on your computer.")
|
|
330
|
+
return
|
|
331
|
+
|
|
332
|
+
df = load_dataframe(filepath)
|
|
333
|
+
if df is None:
|
|
334
|
+
return
|
|
335
|
+
|
|
336
|
+
console.print("\n[bold yellow]🧹 INITIATING AUTO-CLEAN ENGINE...[/]")
|
|
337
|
+
time.sleep(0.5)
|
|
338
|
+
|
|
339
|
+
# 1. Drop Ghost / Unnamed Columns
|
|
340
|
+
unnamed_cols = [col for col in df.columns if str(col).lower().startswith('unnamed')]
|
|
341
|
+
if unnamed_cols:
|
|
342
|
+
df = df.drop(columns=unnamed_cols)
|
|
343
|
+
console.print(f" [green]✔[/] Dropped {len(unnamed_cols)} 'Unnamed' ghost columns.")
|
|
344
|
+
|
|
345
|
+
# 2. Drop Completely Empty Columns
|
|
346
|
+
empty_cols = df.columns[df.isnull().all()].tolist()
|
|
347
|
+
if empty_cols:
|
|
348
|
+
df = df.drop(columns=empty_cols)
|
|
349
|
+
console.print(f" [green]✔[/] Dropped {len(empty_cols)} completely empty columns.")
|
|
350
|
+
|
|
351
|
+
# 3. Fix Text Columns (Invisible Spaces & Text Numbers)
|
|
352
|
+
text_cols = df.select_dtypes(include=['object']).columns
|
|
353
|
+
space_fixed = 0
|
|
354
|
+
dirty_num_fixed = 0
|
|
355
|
+
|
|
356
|
+
for col in text_cols:
|
|
357
|
+
# Check if column looks like a dirty number (has currency/commas and digits)
|
|
358
|
+
curr_mask = df[col].notna() & df[col].astype(str).str.contains(r'[\$£€,]', regex=True) & df[col].astype(str).str.contains(r'\d', regex=True)
|
|
359
|
+
if curr_mask.any():
|
|
360
|
+
df[col] = df[col].astype(str).str.replace(r'[^\d.-]', '', regex=True)
|
|
361
|
+
df[col] = pd.to_numeric(df[col], errors='coerce')
|
|
362
|
+
dirty_num_fixed += 1
|
|
363
|
+
console.print(f" [green]✔[/] Converted '{col}' from text to pure numbers.")
|
|
364
|
+
continue # Moved to numeric, skip string stripping
|
|
365
|
+
|
|
366
|
+
# Strip invisible spaces for remaining text columns
|
|
367
|
+
mask = df[col].notna() & df[col].astype(str).str.contains(r'^\s+|\s+$', regex=True)
|
|
368
|
+
if mask.any():
|
|
369
|
+
df[col] = df[col].astype(str).str.strip()
|
|
370
|
+
space_fixed += 1
|
|
371
|
+
console.print(f" [green]✔[/] Stripped invisible spaces from '{col}'.")
|
|
372
|
+
|
|
373
|
+
# Prompt user for output format
|
|
374
|
+
from rich.prompt import Prompt
|
|
375
|
+
console.print("\n[bold cyan]💾 How would you like to save the cleaned data?[/]")
|
|
376
|
+
console.print(" [bold green]1.[/] Excel Workbook (.xlsx) - Best for Dashboards & formatting")
|
|
377
|
+
console.print(" [bold blue]2.[/] Raw CSV (.csv) - Best for PostgreSQL Bulk Imports")
|
|
378
|
+
console.print(" [bold magenta]3.[/] JSON (.json) - Best for Web Apps & APIs")
|
|
379
|
+
|
|
380
|
+
save_choice = Prompt.ask("Select output format", choices=["1", "2", "3"], default="1")
|
|
381
|
+
base, ext = os.path.splitext(filepath)
|
|
382
|
+
|
|
383
|
+
if save_choice == "1":
|
|
384
|
+
new_filepath = f"{base}_cleaned.xlsx"
|
|
385
|
+
with console.status(f"[cyan]● Saving to '{os.path.basename(new_filepath)}'...[/]", spinner="bouncingBar"):
|
|
386
|
+
df.to_excel(new_filepath, index=False)
|
|
387
|
+
time.sleep(0.6)
|
|
388
|
+
elif save_choice == "2":
|
|
389
|
+
new_filepath = f"{base}_cleaned.csv"
|
|
390
|
+
with console.status(f"[cyan]● Saving to '{os.path.basename(new_filepath)}'...[/]", spinner="bouncingBar"):
|
|
391
|
+
df.to_csv(new_filepath, index=False)
|
|
392
|
+
time.sleep(0.6)
|
|
393
|
+
else:
|
|
394
|
+
new_filepath = f"{base}_cleaned.json"
|
|
395
|
+
with console.status(f"[cyan]● Saving to '{os.path.basename(new_filepath)}'...[/]", spinner="bouncingBar"):
|
|
396
|
+
df.to_json(new_filepath, orient="records", indent=4)
|
|
397
|
+
time.sleep(0.6)
|
|
398
|
+
|
|
399
|
+
console.print(f"\n[bold green]🎉 SUCCESS![/] Cleaned dataset saved to:\n[cyan]{new_filepath}[/]")
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def map_data(filepath):
|
|
404
|
+
if not os.path.exists(filepath):
|
|
405
|
+
found_path = find_file_globally(os.path.basename(filepath))
|
|
406
|
+
if found_path:
|
|
407
|
+
filepath = found_path
|
|
408
|
+
else:
|
|
409
|
+
console.print(f"[bold red]Error:[/] Could not find '{os.path.basename(filepath)}' anywhere on your computer.")
|
|
410
|
+
return
|
|
411
|
+
|
|
412
|
+
df = load_dataframe(filepath)
|
|
413
|
+
if df is None:
|
|
414
|
+
return
|
|
415
|
+
|
|
416
|
+
console.print("\n[bold yellow]🗺️ INITIATING MARKDOWN DATA DICTIONARY (ULLR MAP)...[/]")
|
|
417
|
+
time.sleep(0.5)
|
|
418
|
+
|
|
419
|
+
# 1. Identify Columns and Types
|
|
420
|
+
schema_data = []
|
|
421
|
+
primary_keys = []
|
|
422
|
+
|
|
423
|
+
for col in df.columns:
|
|
424
|
+
dtype = str(df[col].dtype)
|
|
425
|
+
is_unique = df[col].is_unique
|
|
426
|
+
is_id = 'id' in str(col).lower()
|
|
427
|
+
|
|
428
|
+
# Auto-detect Primary Key
|
|
429
|
+
pk_flag = ""
|
|
430
|
+
if is_unique and is_id:
|
|
431
|
+
pk_flag = "🔑 **PRIMARY KEY**"
|
|
432
|
+
primary_keys.append(col)
|
|
433
|
+
elif is_unique:
|
|
434
|
+
pk_flag = "Candidate Key"
|
|
435
|
+
|
|
436
|
+
# Map pandas dtypes to business logic
|
|
437
|
+
business_type = dtype
|
|
438
|
+
if "object" in dtype:
|
|
439
|
+
business_type = "Text / Categorical"
|
|
440
|
+
elif "int" in dtype:
|
|
441
|
+
business_type = "Integer"
|
|
442
|
+
elif "float" in dtype:
|
|
443
|
+
business_type = "Decimal / Float"
|
|
444
|
+
elif "datetime" in dtype:
|
|
445
|
+
business_type = "Datetime"
|
|
446
|
+
|
|
447
|
+
schema_data.append(f"| `{col}` | {business_type} | {pk_flag} | |")
|
|
448
|
+
|
|
449
|
+
# Generate Markdown Content
|
|
450
|
+
base_name = os.path.splitext(os.path.basename(filepath))[0]
|
|
451
|
+
md_content = f"# 🗃️ Data Dictionary: {base_name}\n\n"
|
|
452
|
+
md_content += f"**Source File:** `{os.path.basename(filepath)}`\n"
|
|
453
|
+
md_content += f"**Total Rows:** {len(df):,}\n"
|
|
454
|
+
md_content += f"**Total Columns:** {len(df.columns)}\n\n"
|
|
455
|
+
|
|
456
|
+
if primary_keys:
|
|
457
|
+
md_content += f"**Detected Primary Key(s):** `{', '.join(primary_keys)}`\n\n"
|
|
458
|
+
|
|
459
|
+
# Combined AI Stakeholder Brief Logic
|
|
460
|
+
real_cols = [col for col in df.columns if not str(col).lower().startswith('unnamed')]
|
|
461
|
+
numeric_cols = [c for c in df[real_cols].select_dtypes(include=['number']).columns.tolist() if 'id' not in c.lower() and 'zip' not in c.lower()]
|
|
462
|
+
text_cols = df[real_cols].select_dtypes(include=['object', 'category']).columns.tolist()
|
|
463
|
+
date_cols = [col for col in real_cols if 'date' in col.lower() or 'time' in col.lower() or 'year' in col.lower()]
|
|
464
|
+
|
|
465
|
+
brief = "## 👔 Simulated Stakeholder Brief\n"
|
|
466
|
+
brief += "> *\"Hey team, I just dropped the new dataset in the folder. Before the weekly standup, I need you to clean this up and build a dashboard for me. Specifically, I want to see:\"\n>\n"
|
|
467
|
+
|
|
468
|
+
req_num = 1
|
|
469
|
+
if numeric_cols:
|
|
470
|
+
brief += f"> **{req_num}. High-Level KPIs**: I need the total sum and average of `{numeric_cols[0]}`.\n"
|
|
471
|
+
req_num += 1
|
|
472
|
+
if date_cols and numeric_cols:
|
|
473
|
+
brief += f"> **{req_num}. Trend Analysis**: Build a line chart showing `{numeric_cols[0]}` tracking over time using `{date_cols[0]}`.\n"
|
|
474
|
+
req_num += 1
|
|
475
|
+
if text_cols and numeric_cols:
|
|
476
|
+
brief += f"> **{req_num}. Category Breakdown**: I need a visual showing the top performers in `{text_cols[0]}` based on `{numeric_cols[-1]}`.\n"
|
|
477
|
+
req_num += 1
|
|
478
|
+
if len(text_cols) > 1:
|
|
479
|
+
valid_slicers = [f'`{c}`' for c in text_cols if df[c].nunique() <= 20][:3]
|
|
480
|
+
if valid_slicers:
|
|
481
|
+
brief += f"> **{req_num}. Interactivity**: Make sure you include Slicers for {', '.join(valid_slicers)} so I can filter the dashboard myself.\n"
|
|
482
|
+
|
|
483
|
+
brief += "> \n> **Recommended Dashboard Layout:**\n"
|
|
484
|
+
|
|
485
|
+
if numeric_cols:
|
|
486
|
+
kpis = " and ".join([f"`Total {c}`" for c in numeric_cols[:2]])
|
|
487
|
+
brief += f"> - **Top (KPI Cards):** Put large, bold text showing {kpis} at the very top.\n"
|
|
488
|
+
if date_cols and numeric_cols:
|
|
489
|
+
brief += f"> - **Center (Line Chart):** Show the trend of `{numeric_cols[0]}` over `{date_cols[0]}`.\n"
|
|
490
|
+
if text_cols:
|
|
491
|
+
small_cats = [c for c in text_cols if 1 < df[c].nunique() <= 5]
|
|
492
|
+
if small_cats:
|
|
493
|
+
brief += f"> - **Bottom Left (Donut Chart):** Break down the percentage share of `{small_cats[0]}`.\n"
|
|
494
|
+
med_cats = [c for c in text_cols if 5 < df[c].nunique() <= 15]
|
|
495
|
+
if med_cats and numeric_cols:
|
|
496
|
+
brief += f"> - **Bottom Right (Bar Chart):** Rank the top `{med_cats[0]}` by `{numeric_cols[0]}`.\n"
|
|
497
|
+
valid_slicers = [f'`{c}`' for c in text_cols if df[c].nunique() <= 20][:3]
|
|
498
|
+
if valid_slicers:
|
|
499
|
+
brief += f"> - **Left Sidebar (Slicers):** Add clickable filters for {', '.join(valid_slicers)}.\n"
|
|
500
|
+
|
|
501
|
+
brief += "> \n> *\"Let me know when the Excel file is ready for review!\"*\n\n"
|
|
502
|
+
|
|
503
|
+
md_content += brief
|
|
504
|
+
md_content += "## 📊 Schema Map\n\n"
|
|
505
|
+
md_content += "| Column Name | Data Type | Key Type | Description/Notes |\n"
|
|
506
|
+
md_content += "|---|---|---|---|\n"
|
|
507
|
+
for row in schema_data:
|
|
508
|
+
md_content += row + "\n"
|
|
509
|
+
|
|
510
|
+
# Save next to the original file
|
|
511
|
+
file_dir = os.path.dirname(os.path.abspath(filepath))
|
|
512
|
+
out_path = os.path.join(file_dir, f"{base_name}_schema.md")
|
|
513
|
+
|
|
514
|
+
with open(out_path, "w", encoding="utf-8") as f:
|
|
515
|
+
f.write(md_content)
|
|
516
|
+
|
|
517
|
+
console.print(f"\n[bold green]🎉 SUCCESS![/] Markdown Data Dictionary generated and saved to:\n[cyan]{out_path}[/]")
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def generate_data():
|
|
522
|
+
from rich.prompt import Prompt
|
|
523
|
+
try:
|
|
524
|
+
from faker import Faker
|
|
525
|
+
except ImportError:
|
|
526
|
+
console.print("[bold red]Faker library not installed. Run 'pip install faker' first.[/]")
|
|
527
|
+
return
|
|
528
|
+
|
|
529
|
+
import random
|
|
530
|
+
from datetime import datetime
|
|
531
|
+
import numpy as np
|
|
532
|
+
|
|
533
|
+
fake = Faker()
|
|
534
|
+
|
|
535
|
+
console.print("\n[bold yellow]🏭 INITIATING PRACTICE DATA ENGINE (ULLR GENERATE)...[/]")
|
|
536
|
+
time.sleep(0.5)
|
|
537
|
+
|
|
538
|
+
console.print("\n[bold cyan]Select an Industry Context:[/]")
|
|
539
|
+
console.print(" [bold green]1.[/] E-commerce (Sales, Customers, Products)")
|
|
540
|
+
console.print(" [bold blue]2.[/] Healthcare (Patients, Treatments, Costs)")
|
|
541
|
+
console.print(" [bold magenta]3.[/] Real Estate (Properties, Agents, Prices)")
|
|
542
|
+
|
|
543
|
+
ind_choice = Prompt.ask("Select an option", choices=["1", "2", "3"], default="1")
|
|
544
|
+
rows_choice = Prompt.ask("How many rows of dirty practice data do you need?", default="1000")
|
|
545
|
+
|
|
546
|
+
try:
|
|
547
|
+
rows = int(rows_choice)
|
|
548
|
+
except:
|
|
549
|
+
rows = 1000
|
|
550
|
+
|
|
551
|
+
with console.status(f"[cyan]● Fabricating {rows:,} rows of beautifully dirty data...[/]", spinner="dots"):
|
|
552
|
+
data = []
|
|
553
|
+
if ind_choice == "1":
|
|
554
|
+
industry = "ecommerce"
|
|
555
|
+
for i in range(rows):
|
|
556
|
+
data.append({
|
|
557
|
+
"Transaction_ID": fake.uuid4()[:8],
|
|
558
|
+
"Customer_Name": fake.name(),
|
|
559
|
+
"Purchase_Date": fake.date_between(start_date='-1y', end_date='today').strftime('%Y-%m-%d'),
|
|
560
|
+
"Product_Category": random.choice(["Electronics", "Clothing", "Home", "Sports"]),
|
|
561
|
+
"Quantity": random.randint(1, 10),
|
|
562
|
+
"Unit_Price": round(random.uniform(10.0, 500.0), 2)
|
|
563
|
+
})
|
|
564
|
+
elif ind_choice == "2":
|
|
565
|
+
industry = "healthcare"
|
|
566
|
+
for i in range(rows):
|
|
567
|
+
data.append({
|
|
568
|
+
"Patient_ID": f"PT-{fake.random_int(min=1000, max=9999)}",
|
|
569
|
+
"Patient_Name": fake.name(),
|
|
570
|
+
"Admission_Date": fake.date_between(start_date='-2y', end_date='today').strftime('%Y-%m-%d'),
|
|
571
|
+
"Department": random.choice(["Cardiology", "Neurology", "Oncology", "Pediatrics"]),
|
|
572
|
+
"Length_of_Stay": random.randint(1, 30),
|
|
573
|
+
"Treatment_Cost": round(random.uniform(500.0, 15000.0), 2)
|
|
574
|
+
})
|
|
575
|
+
else:
|
|
576
|
+
industry = "realestate"
|
|
577
|
+
for i in range(rows):
|
|
578
|
+
data.append({
|
|
579
|
+
"Property_ID": f"RE-{fake.random_int(min=100, max=999)}",
|
|
580
|
+
"Agent_Name": fake.name(),
|
|
581
|
+
"Listing_Date": fake.date_between(start_date='-6m', end_date='today').strftime('%Y-%m-%d'),
|
|
582
|
+
"Property_Type": random.choice(["House", "Condo", "Townhouse", "Commercial"]),
|
|
583
|
+
"Square_Feet": random.randint(800, 5000),
|
|
584
|
+
"Listing_Price": round(random.uniform(150000.0, 2000000.0), 2)
|
|
585
|
+
})
|
|
586
|
+
|
|
587
|
+
df = pd.DataFrame(data)
|
|
588
|
+
|
|
589
|
+
# --- INJECT DIRTY DATA ---
|
|
590
|
+
cols = list(df.columns)
|
|
591
|
+
|
|
592
|
+
# 1. Nulls (Missing Values)
|
|
593
|
+
for col in cols:
|
|
594
|
+
mask = np.random.rand(len(df)) < 0.05 # 5% missing
|
|
595
|
+
df.loc[mask, col] = np.nan
|
|
596
|
+
|
|
597
|
+
# 2. Invisible Spaces (Messy Text)
|
|
598
|
+
text_cols = [c for c in cols if 'Name' in c or 'Category' in c or 'Type' in c or 'Department' in c]
|
|
599
|
+
for col in text_cols:
|
|
600
|
+
mask = np.random.rand(len(df)) < 0.15 # 15% messy text
|
|
601
|
+
def add_spaces(x):
|
|
602
|
+
if pd.isna(x): return x
|
|
603
|
+
return (" " + str(x)) if random.random() > 0.5 else (str(x) + " ")
|
|
604
|
+
|
|
605
|
+
df.loc[mask, col] = df.loc[mask, col].apply(add_spaces)
|
|
606
|
+
|
|
607
|
+
# 3. Numbers Trapped as Text
|
|
608
|
+
num_cols = [c for c in cols if 'Price' in c or 'Cost' in c]
|
|
609
|
+
for col in num_cols:
|
|
610
|
+
df[col] = df[col].astype('object')
|
|
611
|
+
mask = np.random.rand(len(df)) < 0.20 # 20% trapped numbers
|
|
612
|
+
def add_currency(x):
|
|
613
|
+
if pd.isna(x): return x
|
|
614
|
+
return f"${x:,.2f}" if random.random() > 0.5 else f"£{x:,.2f}"
|
|
615
|
+
|
|
616
|
+
df.loc[mask, col] = df.loc[mask, col].apply(add_currency)
|
|
617
|
+
|
|
618
|
+
# 4. Ghost Column
|
|
619
|
+
df["Unnamed: 4"] = np.nan
|
|
620
|
+
|
|
621
|
+
# Save File
|
|
622
|
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
623
|
+
filename = f"{industry}_raw_{timestamp}.csv"
|
|
624
|
+
|
|
625
|
+
out_dir = os.path.join(pathlib.Path.home(), "Projects", "eksel-praktis")
|
|
626
|
+
os.makedirs(out_dir, exist_ok=True)
|
|
627
|
+
out_path = os.path.join(out_dir, filename)
|
|
628
|
+
|
|
629
|
+
df.to_csv(out_path, index=False)
|
|
630
|
+
time.sleep(1)
|
|
631
|
+
|
|
632
|
+
console.print(f"\n[bold green]🎉 SUCCESS![/] {rows:,} rows of beautifully broken {industry} data generated.")
|
|
633
|
+
console.print(f"[cyan]Saved to: {out_path}[/]")
|
|
634
|
+
|
|
635
|
+
# --- DYNAMIC STAKEHOLDER BRIEF ---
|
|
636
|
+
real_cols = [col for col in df.columns if not str(col).lower().startswith('unnamed')]
|
|
637
|
+
numeric_cols = [c for c in df[real_cols].select_dtypes(include=['number', 'float64', 'int64']).columns.tolist() if 'id' not in c.lower() and 'zip' not in c.lower()]
|
|
638
|
+
# Text cols excluding specific names that usually act as unique IDs (like Patient_Name)
|
|
639
|
+
text_cols = [c for c in df[real_cols].select_dtypes(include=['object', 'string']).columns.tolist() if 'name' not in c.lower() and 'id' not in c.lower()]
|
|
640
|
+
date_cols = [col for col in real_cols if 'date' in col.lower() or 'time' in col.lower() or 'year' in col.lower()]
|
|
641
|
+
|
|
642
|
+
console.print("\n[bold magenta]👔 STAKEHOLDER BRIEF: SYSTEM REQUIREMENTS[/]")
|
|
643
|
+
console.print("[bold cyan]KPIs (The Top-Level Cards)[/]")
|
|
644
|
+
if numeric_cols:
|
|
645
|
+
for num_col in numeric_cols[:2]:
|
|
646
|
+
console.print(f" * Total {num_col.replace('_', ' ')}")
|
|
647
|
+
console.print(f" * Avg. {num_col.replace('_', ' ')}")
|
|
648
|
+
else:
|
|
649
|
+
console.print(" * Total Records")
|
|
650
|
+
|
|
651
|
+
console.print("\n[bold cyan]Business Questions (The Charts & Visuals)[/]")
|
|
652
|
+
if date_cols and numeric_cols:
|
|
653
|
+
console.print(f" * Time-Series Trend: How is {numeric_cols[0].replace('_', ' ')} trending based on {date_cols[0].replace('_', ' ')}? (Line Chart)")
|
|
654
|
+
if text_cols and numeric_cols:
|
|
655
|
+
console.print(f" * Financial/Metric Breakdown: Which {text_cols[0].replace('_', ' ')} drives the highest {numeric_cols[0].replace('_', ' ')}? (Bar Chart)")
|
|
656
|
+
if text_cols:
|
|
657
|
+
console.print(f" * Volume Breakdown: What is the distribution of total volume by {text_cols[0].replace('_', ' ')}? (Donut Chart)")
|
|
658
|
+
if len(text_cols) > 1 and numeric_cols:
|
|
659
|
+
console.print(f" * Secondary Ranking: Compare {text_cols[1].replace('_', ' ')} by {numeric_cols[-1].replace('_', ' ')}? (Bar Chart)")
|
|
660
|
+
|
|
661
|
+
console.print("\n[bold yellow]Your training mission is ready. Clean the data and build the Engine to answer these exact questions![/]")
|
|
662
|
+
|
|
663
|
+
|
|
320
664
|
def interactive_mode():
|
|
321
665
|
from rich.prompt import Prompt
|
|
322
666
|
|
|
@@ -325,21 +669,40 @@ def interactive_mode():
|
|
|
325
669
|
|
|
326
670
|
while True:
|
|
327
671
|
console.print("[bold cyan]What would you like to do?[/]")
|
|
328
|
-
console.print(" [bold green]1.[/] Audit
|
|
329
|
-
console.print(" [bold blue]2.[/] Analyze
|
|
330
|
-
console.print(" [bold
|
|
672
|
+
console.print(" [bold green]1.[/] Audit Data")
|
|
673
|
+
console.print(" [bold blue]2.[/] Analyze Data")
|
|
674
|
+
console.print(" [bold magenta]3.[/] Auto-Clean")
|
|
675
|
+
console.print(" [bold bright_cyan]4.[/] Map Schema")
|
|
676
|
+
console.print(" [bold yellow]5.[/] Generate Data")
|
|
677
|
+
console.print(" [bold white]6.[/] Help Menu")
|
|
678
|
+
console.print(" [bold red]7.[/] Exit")
|
|
331
679
|
|
|
332
|
-
choice = Prompt.ask("\nSelect an option", choices=["1", "2", "3"], default="1")
|
|
680
|
+
choice = Prompt.ask("\nSelect an option", choices=["1", "2", "3", "4", "5", "6", "7"], default="1")
|
|
333
681
|
|
|
334
|
-
if choice == "
|
|
682
|
+
if choice == "7":
|
|
335
683
|
console.print("[yellow]Goodbye![/]")
|
|
336
684
|
break
|
|
685
|
+
elif choice == "6":
|
|
686
|
+
console.print("\n[bold underline]Ullr Help Menu[/]")
|
|
687
|
+
console.print("[bold green]Audit Data:[/] Scans a raw CSV/Excel file for missing values, duplicates, and formatting errors without changing the file.")
|
|
688
|
+
console.print("[bold blue]Analyze Data:[/] Reads a cleaned dataset and provides a blueprint/layout for building a Dashboard in BI tools.")
|
|
689
|
+
console.print("[bold magenta]Auto-Clean:[/] Automatically drops empty columns, strips invisible spaces, and converts currency text back to pure numbers.")
|
|
690
|
+
console.print("[bold bright_cyan]Map Schema:[/] Generates a Markdown Data Dictionary of your dataset and saves it in the same folder as your original file.")
|
|
691
|
+
console.print("[bold yellow]Generate Data:[/] Creates synthetic, dirty practice datasets (e.g. E-commerce, Healthcare) with intentional errors for you to practice cleaning.")
|
|
692
|
+
elif choice == "5":
|
|
693
|
+
generate_data()
|
|
694
|
+
elif choice == "4":
|
|
695
|
+
filepath = Prompt.ask("\n[bold yellow]Enter the filename or path to map (e.g., data.csv)[/]")
|
|
696
|
+
map_data(filepath.strip())
|
|
337
697
|
elif choice == "1":
|
|
338
698
|
filepath = Prompt.ask("\n[bold yellow]Enter the filename or path to audit (e.g., data.csv)[/]")
|
|
339
699
|
audit_data(filepath.strip())
|
|
340
700
|
elif choice == "2":
|
|
341
701
|
filepath = Prompt.ask("\n[bold yellow]Enter the filename or path to analyze (e.g., data.csv)[/]")
|
|
342
702
|
analyze_data(filepath.strip())
|
|
703
|
+
elif choice == "3":
|
|
704
|
+
filepath = Prompt.ask("\n[bold yellow]Enter the filename or path to clean (e.g., data.csv)[/]")
|
|
705
|
+
clean_data(filepath.strip())
|
|
343
706
|
|
|
344
707
|
console.print("\n" + "-"*50 + "\n")
|
|
345
708
|
|
|
@@ -353,6 +716,14 @@ def cli():
|
|
|
353
716
|
analyze_parser = subparsers.add_parser("analyze", help="Generate dashboard blueprints for cleaned data.")
|
|
354
717
|
analyze_parser.add_argument("file", help="Path to the cleaned CSV file")
|
|
355
718
|
|
|
719
|
+
clean_parser = subparsers.add_parser("clean", help="Auto-clean data (remove empty columns, strip spaces, fix text numbers).")
|
|
720
|
+
clean_parser.add_argument("file", help="Path to the raw CSV file")
|
|
721
|
+
|
|
722
|
+
map_parser = subparsers.add_parser("map", help="Auto-generate a Markdown Data Dictionary for Obsidian.")
|
|
723
|
+
map_parser.add_argument("file", help="Path to the CSV/Excel file")
|
|
724
|
+
|
|
725
|
+
generate_parser = subparsers.add_parser("generate", help="Generate synthetic dirty datasets for Excel practice.")
|
|
726
|
+
|
|
356
727
|
args = parser.parse_args()
|
|
357
728
|
|
|
358
729
|
if not args.command:
|
|
@@ -364,6 +735,12 @@ def cli():
|
|
|
364
735
|
audit_data(args.file)
|
|
365
736
|
elif args.command == "analyze":
|
|
366
737
|
analyze_data(args.file)
|
|
738
|
+
elif args.command == "clean":
|
|
739
|
+
clean_data(args.file)
|
|
740
|
+
elif args.command == "map":
|
|
741
|
+
map_data(args.file)
|
|
742
|
+
elif args.command == "generate":
|
|
743
|
+
generate_data()
|
|
367
744
|
|
|
368
745
|
if __name__ == "__main__":
|
|
369
746
|
cli()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|