rosie-df 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rosie_df-0.1.0/PKG-INFO +11 -0
- rosie_df-0.1.0/README.md +1 -0
- rosie_df-0.1.0/pyproject.toml +23 -0
- rosie_df-0.1.0/rosie_df/__init__.py +3 -0
- rosie_df-0.1.0/rosie_df/cli.py +39 -0
- rosie_df-0.1.0/rosie_df/core.py +212 -0
- rosie_df-0.1.0/rosie_df/data/cheatsheet.yaml +279 -0
- rosie_df-0.1.0/rosie_df.egg-info/PKG-INFO +11 -0
- rosie_df-0.1.0/rosie_df.egg-info/SOURCES.txt +12 -0
- rosie_df-0.1.0/rosie_df.egg-info/dependency_links.txt +1 -0
- rosie_df-0.1.0/rosie_df.egg-info/entry_points.txt +2 -0
- rosie_df-0.1.0/rosie_df.egg-info/requires.txt +2 -0
- rosie_df-0.1.0/rosie_df.egg-info/top_level.txt +1 -0
- rosie_df-0.1.0/setup.cfg +4 -0
rosie_df-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: rosie-df
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: High-performance DataFrame Rosetta Stone powered by Polars
|
|
5
|
+
Author-email: anandlenin <anandijjina719@gmail.com>
|
|
6
|
+
Requires-Python: >=3.9
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: pyyaml>=6.0
|
|
9
|
+
Requires-Dist: polars>=1.0.0
|
|
10
|
+
|
|
11
|
+
Cheatsheet tool mapping syntax across Pandas, Polars, and PySpark.
|
rosie_df-0.1.0/README.md
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Cheatsheet tool mapping syntax across Pandas, Polars, and PySpark.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "rosie-df"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "High-performance DataFrame Rosetta Stone powered by Polars"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
authors = [
|
|
12
|
+
{ name = "anandlenin", email = "anandijjina719@gmail.com" }
|
|
13
|
+
]
|
|
14
|
+
dependencies = [
|
|
15
|
+
"pyyaml>=6.0",
|
|
16
|
+
"polars>=1.0.0"
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[project.scripts]
|
|
20
|
+
rosie = "rosie_df.cli:main"
|
|
21
|
+
|
|
22
|
+
[tool.setuptools.package-data]
|
|
23
|
+
rosie_df = ["data/*.yaml"]
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import polars as pl
|
|
3
|
+
from rosie_df.core import helper
|
|
4
|
+
|
|
5
|
+
def main():
|
|
6
|
+
parser = argparse.ArgumentParser(
|
|
7
|
+
prog="rosie",
|
|
8
|
+
description="Rosie-DF: Fast DataFrame Cheatsheet for Pandas, Polars & PySpark"
|
|
9
|
+
)
|
|
10
|
+
parser.add_argument("engines", nargs="*", default=[], help="Engines: pandas, polars, pyspark")
|
|
11
|
+
parser.add_argument("--category", "-c", type=str, default=None, help="Filter category: inspect, clean, transform, eda, export")
|
|
12
|
+
parser.add_argument("--search", "-s", type=str, default=None, help="Search query")
|
|
13
|
+
parser.add_argument("--ops", "-o", nargs="+", default=None, help="Filter specific operations")
|
|
14
|
+
parser.add_argument("--gotchas", "-g", action="store_true", help="Show performance gotchas")
|
|
15
|
+
parser.add_argument("--transpose", "-t", action="store_true", help="Transpose rows and columns")
|
|
16
|
+
parser.add_argument("--categories", action="store_true", help="List available categories")
|
|
17
|
+
|
|
18
|
+
args = parser.parse_args()
|
|
19
|
+
|
|
20
|
+
if args.categories:
|
|
21
|
+
print("\nAvailable Categories:", ", ".join(helper.categories()), "\n")
|
|
22
|
+
return
|
|
23
|
+
|
|
24
|
+
if args.search:
|
|
25
|
+
result = helper.search(args.search, *args.engines, gotchas=args.gotchas)
|
|
26
|
+
else:
|
|
27
|
+
result = helper.show(
|
|
28
|
+
*args.engines,
|
|
29
|
+
category=args.category,
|
|
30
|
+
ops=args.ops,
|
|
31
|
+
gotchas=args.gotchas,
|
|
32
|
+
transpose=True if args.transpose else None
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
with pl.Config(tbl_rows=-1, tbl_cols=-1, fmt_str_lengths=120, tbl_width_chars=180):
|
|
36
|
+
print("\n", result.df, "\n")
|
|
37
|
+
|
|
38
|
+
if __name__ == "__main__":
|
|
39
|
+
main()
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import html
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import List, Optional
|
|
4
|
+
import yaml
|
|
5
|
+
import polars as pl
|
|
6
|
+
|
|
7
|
+
YAML_FILE = Path(__file__).parent / "data" / "cheatsheet.yaml"
|
|
8
|
+
VALID_ENGINES = ["pandas", "polars", "pyspark"]
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class RosieResult:
|
|
12
|
+
def __init__(self, df: pl.DataFrame, raw_data: list, engines: list, gotchas: bool, transpose: bool):
|
|
13
|
+
self.df = df
|
|
14
|
+
self.raw_data = raw_data
|
|
15
|
+
self.engines = engines
|
|
16
|
+
self.gotchas = gotchas
|
|
17
|
+
self.transpose = transpose
|
|
18
|
+
|
|
19
|
+
def _repr_html_(self) -> str:
|
|
20
|
+
css = """
|
|
21
|
+
<style>
|
|
22
|
+
.rosie-table {
|
|
23
|
+
width: 100%;
|
|
24
|
+
border-collapse: collapse;
|
|
25
|
+
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
|
26
|
+
margin: 12px 0;
|
|
27
|
+
font-size: 13px;
|
|
28
|
+
line-height: 1.4;
|
|
29
|
+
}
|
|
30
|
+
.rosie-table th {
|
|
31
|
+
background: rgba(125, 125, 125, 0.15);
|
|
32
|
+
color: var(--jp-ui-font-color1, #e0e0e0);
|
|
33
|
+
font-weight: 600;
|
|
34
|
+
padding: 10px 14px;
|
|
35
|
+
border: 1px solid rgba(125, 125, 125, 0.25);
|
|
36
|
+
text-align: left;
|
|
37
|
+
}
|
|
38
|
+
.rosie-table td {
|
|
39
|
+
padding: 10px 14px;
|
|
40
|
+
border: 1px solid rgba(125, 125, 125, 0.25);
|
|
41
|
+
vertical-align: top;
|
|
42
|
+
}
|
|
43
|
+
.rosie-badge {
|
|
44
|
+
display: inline-block;
|
|
45
|
+
padding: 3px 8px;
|
|
46
|
+
border-radius: 4px;
|
|
47
|
+
font-weight: bold;
|
|
48
|
+
font-size: 11px;
|
|
49
|
+
text-transform: uppercase;
|
|
50
|
+
}
|
|
51
|
+
.badge-pandas { background: #306998; color: #ffffff; }
|
|
52
|
+
.badge-polars { background: #00876c; color: #ffffff; }
|
|
53
|
+
.badge-pyspark { background: #e25a1c; color: #ffffff; }
|
|
54
|
+
.rosie-code {
|
|
55
|
+
font-family: "JetBrains Mono", Consolas, Menlo, monospace;
|
|
56
|
+
font-size: 12px;
|
|
57
|
+
background: rgba(125, 125, 125, 0.1);
|
|
58
|
+
border: 1px solid rgba(125, 125, 125, 0.2);
|
|
59
|
+
border-radius: 4px;
|
|
60
|
+
padding: 8px 10px;
|
|
61
|
+
margin: 0;
|
|
62
|
+
white-space: pre-wrap;
|
|
63
|
+
word-break: break-word;
|
|
64
|
+
color: var(--jp-ui-font-color1, #ffffff);
|
|
65
|
+
}
|
|
66
|
+
.rosie-gotcha {
|
|
67
|
+
margin-top: 6px;
|
|
68
|
+
padding: 6px 8px;
|
|
69
|
+
background: rgba(245, 158, 11, 0.12);
|
|
70
|
+
border-left: 3px solid #f59e0b;
|
|
71
|
+
border-radius: 2px;
|
|
72
|
+
font-size: 11px;
|
|
73
|
+
color: var(--jp-ui-font-color2, #f3f4f6);
|
|
74
|
+
}
|
|
75
|
+
</style>
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
html_out = [css, '<table class="rosie-table">']
|
|
79
|
+
|
|
80
|
+
if not self.transpose:
|
|
81
|
+
html_out.append("<thead><tr><th>Library</th>")
|
|
82
|
+
for item in self.raw_data:
|
|
83
|
+
op_name = item.get("name", item.get("id"))
|
|
84
|
+
html_out.append(f"<th>{html.escape(op_name)}</th>")
|
|
85
|
+
html_out.append("</tr></thead><tbody>")
|
|
86
|
+
|
|
87
|
+
for eng in self.engines:
|
|
88
|
+
html_out.append("<tr>")
|
|
89
|
+
html_out.append(f'<td><span class="rosie-badge badge-{eng}">{eng}</span></td>')
|
|
90
|
+
for item in self.raw_data:
|
|
91
|
+
syntax = item.get(eng, "").strip()
|
|
92
|
+
code_html = f'<pre class="rosie-code">{html.escape(syntax)}</pre>'
|
|
93
|
+
gotcha_html = ""
|
|
94
|
+
if self.gotchas:
|
|
95
|
+
note = item.get("gotchas", {}).get(eng, "")
|
|
96
|
+
if note:
|
|
97
|
+
gotcha_html = f'<div class="rosie-gotcha">💡 <b>Gotcha:</b> {html.escape(note)}</div>'
|
|
98
|
+
html_out.append(f"<td>{code_html}{gotcha_html}</td>")
|
|
99
|
+
html_out.append("</tr>")
|
|
100
|
+
else:
|
|
101
|
+
html_out.append("<thead><tr><th>Operation</th>")
|
|
102
|
+
for eng in self.engines:
|
|
103
|
+
html_out.append(f'<th><span class="rosie-badge badge-{eng}">{eng}</span></th>')
|
|
104
|
+
html_out.append("</tr></thead><tbody>")
|
|
105
|
+
|
|
106
|
+
for item in self.raw_data:
|
|
107
|
+
op_name = item.get("name", item.get("id"))
|
|
108
|
+
html_out.append(f"<tr><td><b>{html.escape(op_name)}</b></td>")
|
|
109
|
+
for eng in self.engines:
|
|
110
|
+
syntax = item.get(eng, "").strip()
|
|
111
|
+
code_html = f'<pre class="rosie-code">{html.escape(syntax)}</pre>'
|
|
112
|
+
gotcha_html = ""
|
|
113
|
+
if self.gotchas:
|
|
114
|
+
note = item.get("gotchas", {}).get(eng, "")
|
|
115
|
+
if note:
|
|
116
|
+
gotcha_html = f'<div class="rosie-gotcha">💡 <b>Gotcha:</b> {html.escape(note)}</div>'
|
|
117
|
+
html_out.append(f"<td>{code_html}{gotcha_html}</td>")
|
|
118
|
+
html_out.append("</tr>")
|
|
119
|
+
|
|
120
|
+
html_out.append("</tbody></table>")
|
|
121
|
+
return "".join(html_out)
|
|
122
|
+
|
|
123
|
+
def to_polars(self) -> pl.DataFrame:
|
|
124
|
+
return self.df
|
|
125
|
+
|
|
126
|
+
def __repr__(self) -> str:
|
|
127
|
+
with pl.Config(tbl_rows=-1, tbl_cols=-1, fmt_str_lengths=120, tbl_width_chars=180):
|
|
128
|
+
return str(self.df)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class SyntaxLookup:
|
|
132
|
+
def __init__(self, data_path: Path = YAML_FILE):
|
|
133
|
+
self.data_path = data_path
|
|
134
|
+
self._raw_data = self._load_data()
|
|
135
|
+
|
|
136
|
+
def _load_data(self) -> list:
|
|
137
|
+
if not self.data_path.exists():
|
|
138
|
+
raise FileNotFoundError(f"Cheatsheet data not found at: {self.data_path}")
|
|
139
|
+
with open(self.data_path, "r", encoding="utf-8") as f:
|
|
140
|
+
return yaml.safe_load(f).get("operations", [])
|
|
141
|
+
|
|
142
|
+
def categories(self) -> List[str]:
|
|
143
|
+
return sorted(list(set(item.get("category", "general") for item in self._raw_data)))
|
|
144
|
+
|
|
145
|
+
def show(
|
|
146
|
+
self,
|
|
147
|
+
*engines: str,
|
|
148
|
+
category: Optional[str] = None,
|
|
149
|
+
ops: Optional[List[str]] = None,
|
|
150
|
+
gotchas: bool = False,
|
|
151
|
+
transpose: Optional[bool] = None
|
|
152
|
+
) -> RosieResult:
|
|
153
|
+
"""
|
|
154
|
+
Compare DataFrame syntax across engines.
|
|
155
|
+
|
|
156
|
+
Categories:
|
|
157
|
+
- 'inspect' : read_data, view_rows, schema_info, count_nulls, drop_duplicates, describe_stats
|
|
158
|
+
- 'clean' : filter_rows, handle_nulls, cast_types, rename_columns, string_cleaning
|
|
159
|
+
- 'transform' : add_column, groupby_agg, join_tables, reshape, window_functions
|
|
160
|
+
- 'eda' : univariate_counts, quantiles, skew_kurt, bivariate_corr, crosstab, covariance
|
|
161
|
+
- 'export' : save_parquet, save_csv
|
|
162
|
+
"""
|
|
163
|
+
selected_engines = [e.lower() for e in engines] if engines else VALID_ENGINES
|
|
164
|
+
for eng in selected_engines:
|
|
165
|
+
if eng not in VALID_ENGINES:
|
|
166
|
+
raise ValueError(f"Unknown engine '{eng}'. Choose from: {VALID_ENGINES}")
|
|
167
|
+
|
|
168
|
+
filtered_items = []
|
|
169
|
+
for item in self._raw_data:
|
|
170
|
+
if category and item.get("category", "").lower() != category.lower():
|
|
171
|
+
continue
|
|
172
|
+
op_id = item.get("id", "")
|
|
173
|
+
op_name = item.get("name", op_id)
|
|
174
|
+
if ops and not any(t.lower() in (op_id.lower(), op_name.lower()) for t in ops):
|
|
175
|
+
continue
|
|
176
|
+
filtered_items.append(item)
|
|
177
|
+
|
|
178
|
+
# Auto-transpose if table is wide (> 3 columns) unless explicitly overridden
|
|
179
|
+
should_transpose = transpose if transpose is not None else (len(filtered_items) > 3)
|
|
180
|
+
|
|
181
|
+
if not should_transpose:
|
|
182
|
+
data_dict = {"Library": selected_engines}
|
|
183
|
+
for item in filtered_items:
|
|
184
|
+
name = item.get("name", item.get("id"))
|
|
185
|
+
data_dict[name] = [item.get(eng, "").strip() for eng in selected_engines]
|
|
186
|
+
df = pl.DataFrame(data_dict)
|
|
187
|
+
else:
|
|
188
|
+
data_dict = {"Operation": [item.get("name", item.get("id")) for item in filtered_items]}
|
|
189
|
+
for eng in selected_engines:
|
|
190
|
+
data_dict[eng] = [item.get(eng, "").strip() for item in filtered_items]
|
|
191
|
+
df = pl.DataFrame(data_dict)
|
|
192
|
+
|
|
193
|
+
return RosieResult(df, filtered_items, selected_engines, gotchas, should_transpose)
|
|
194
|
+
|
|
195
|
+
def search(self, query: str, *engines: str, gotchas: bool = False) -> RosieResult:
|
|
196
|
+
"""Search query matching across name, id, category, and gotchas."""
|
|
197
|
+
q = query.lower()
|
|
198
|
+
matched_ops = []
|
|
199
|
+
for item in self._raw_data:
|
|
200
|
+
fields = [
|
|
201
|
+
item.get("id", ""),
|
|
202
|
+
item.get("name", ""),
|
|
203
|
+
item.get("category", ""),
|
|
204
|
+
str(item.get("gotchas", ""))
|
|
205
|
+
]
|
|
206
|
+
if any(q in f.lower() for f in fields):
|
|
207
|
+
matched_ops.append(item.get("id"))
|
|
208
|
+
return self.show(*engines, ops=matched_ops, gotchas=gotchas)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
helper = SyntaxLookup()
|
|
212
|
+
show = helper.show
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
operations:
|
|
2
|
+
# ==========================================
|
|
3
|
+
# 1. INSPECT (category: "inspect")
|
|
4
|
+
# ==========================================
|
|
5
|
+
- id: read_data
|
|
6
|
+
name: "Read Data (CSV)"
|
|
7
|
+
category: "inspect"
|
|
8
|
+
pandas: "pd.read_csv('file.csv')"
|
|
9
|
+
polars: "pl.read_csv('file.csv')\n# or pl.scan_csv('file.csv') for lazy"
|
|
10
|
+
pyspark: "spark.read.csv('file.csv', header=True, inferSchema=True)"
|
|
11
|
+
gotchas:
|
|
12
|
+
pandas: "Eager only; uses 5-10x memory of raw file size."
|
|
13
|
+
polars: "Use scan_csv() to optimize execution plans before loading."
|
|
14
|
+
pyspark: "inferSchema=True triggers two full reads of the dataset."
|
|
15
|
+
|
|
16
|
+
- id: view_rows
|
|
17
|
+
name: "Preview Rows (head)"
|
|
18
|
+
category: "inspect"
|
|
19
|
+
pandas: "df.head(5)"
|
|
20
|
+
polars: "df.head(5)"
|
|
21
|
+
pyspark: "df.show(5)"
|
|
22
|
+
gotchas:
|
|
23
|
+
pandas: "Returns a new DataFrame slice."
|
|
24
|
+
polars: "Runs without copying memory."
|
|
25
|
+
pyspark: "df.head(5) returns Row objects; use df.show(5) for tabular view."
|
|
26
|
+
|
|
27
|
+
- id: schema_info
|
|
28
|
+
name: "Schema & Types (info)"
|
|
29
|
+
category: "inspect"
|
|
30
|
+
pandas: "df.info()"
|
|
31
|
+
polars: "df.schema\n# or df.glimpse()"
|
|
32
|
+
pyspark: "df.printSchema()"
|
|
33
|
+
gotchas:
|
|
34
|
+
pandas: "df.info() prints directly to stdout; does not return an object."
|
|
35
|
+
polars: "No df.info(). Use df.glimpse() for compact single-screen view."
|
|
36
|
+
pyspark: "df.printSchema() prints tree structure to console."
|
|
37
|
+
|
|
38
|
+
- id: count_nulls
|
|
39
|
+
name: "Count Nulls"
|
|
40
|
+
category: "inspect"
|
|
41
|
+
pandas: "df.isnull().sum()"
|
|
42
|
+
polars: "df.null_count()"
|
|
43
|
+
pyspark: "df.select([F.count(F.when(F.col(c).isNull(), c)).alias(c) for c in df.columns])"
|
|
44
|
+
gotchas:
|
|
45
|
+
pandas: "Calculated column by column eagerly."
|
|
46
|
+
polars: "null_count() returns a 1-row DataFrame multithreaded."
|
|
47
|
+
pyspark: "Spark has no built-in null_count(); requires list comprehension."
|
|
48
|
+
|
|
49
|
+
- id: drop_duplicates
|
|
50
|
+
name: "Deduplication"
|
|
51
|
+
category: "inspect"
|
|
52
|
+
pandas: "df.drop_duplicates(subset=['id'])"
|
|
53
|
+
polars: "df.unique(subset=['id'])"
|
|
54
|
+
pyspark: "df.dropDuplicates(['id'])"
|
|
55
|
+
gotchas:
|
|
56
|
+
pandas: "Defaults to keep='first'."
|
|
57
|
+
polars: "Method is unique(), not drop_duplicates()."
|
|
58
|
+
pyspark: "Triggers global shuffle across nodes."
|
|
59
|
+
|
|
60
|
+
- id: describe_stats
|
|
61
|
+
name: "Summary Statistics"
|
|
62
|
+
category: "inspect"
|
|
63
|
+
pandas: "df.describe()"
|
|
64
|
+
polars: "df.describe()"
|
|
65
|
+
pyspark: "df.describe().show()"
|
|
66
|
+
gotchas:
|
|
67
|
+
pandas: "Includes percentiles 25, 50, 75 by default."
|
|
68
|
+
polars: "Returns a DataFrame with min, max, mean, std, null_count."
|
|
69
|
+
pyspark: "Computes count, mean, stddev, min, max eagerly."
|
|
70
|
+
|
|
71
|
+
# ==========================================
|
|
72
|
+
# 2. CLEAN (category: "clean")
|
|
73
|
+
# ==========================================
|
|
74
|
+
- id: filter_rows
|
|
75
|
+
name: "Filter Rows"
|
|
76
|
+
category: "clean"
|
|
77
|
+
pandas: "df[(df['age'] > 25) & (df['status'] == 'active')]"
|
|
78
|
+
polars: "df.filter((pl.col('age') > 25) & (pl.col('status') == 'active'))"
|
|
79
|
+
pyspark: "df.filter((F.col('age') > 25) & (F.col('status') == 'active'))"
|
|
80
|
+
gotchas:
|
|
81
|
+
pandas: "Missing parentheses around conditions causes bitwise precedence errors."
|
|
82
|
+
polars: "Never use Python and/or; use & and |."
|
|
83
|
+
pyspark: "SQL string syntax also allowed: df.filter('age > 25 AND status = \"active\"')."
|
|
84
|
+
|
|
85
|
+
- id: handle_nulls
|
|
86
|
+
name: "Drop or Fill Nulls"
|
|
87
|
+
category: "clean"
|
|
88
|
+
pandas: "df.dropna(subset=['col'])\ndf['col'].fillna(0)"
|
|
89
|
+
polars: "df.drop_nulls('col')\ndf.with_columns(pl.col('col').fill_null(0))"
|
|
90
|
+
pyspark: "df.dropna(subset=['col'])\ndf.fillna({'col': 0})"
|
|
91
|
+
gotchas:
|
|
92
|
+
pandas: "fillna in place modifies state; returns None if inplace=True."
|
|
93
|
+
polars: "fill_null() also supports strategies: 'forward', 'backward', 'mean'."
|
|
94
|
+
pyspark: "df.fillna() converts numeric targets automatically."
|
|
95
|
+
|
|
96
|
+
- id: cast_types
|
|
97
|
+
name: "Cast Data Types"
|
|
98
|
+
category: "clean"
|
|
99
|
+
pandas: "df['age'] = df['age'].astype('int64')"
|
|
100
|
+
polars: "df = df.with_columns(pl.col('age').cast(pl.Int64))"
|
|
101
|
+
pyspark: "df = df.withColumn('age', F.col('age').cast('long'))"
|
|
102
|
+
gotchas:
|
|
103
|
+
pandas: "Astype fails completely if column contains NaN for integer casts."
|
|
104
|
+
polars: "Use strict=False in cast() to turn invalid parses into nulls without error."
|
|
105
|
+
pyspark: "Invalid casts silently return null in Spark."
|
|
106
|
+
|
|
107
|
+
- id: rename_columns
|
|
108
|
+
name: "Rename Columns"
|
|
109
|
+
category: "clean"
|
|
110
|
+
pandas: "df.rename(columns={'old_name': 'new_name'})"
|
|
111
|
+
polars: "df.rename({'old_name': 'new_name'})"
|
|
112
|
+
pyspark: "df.withColumnRenamed('old_name', 'new_name')"
|
|
113
|
+
gotchas:
|
|
114
|
+
pandas: "Requires columns={} argument explicitly."
|
|
115
|
+
polars: "Renaming is zero-copy metadata update."
|
|
116
|
+
pyspark: "Chain withColumnRenamed multiple times or use select alias."
|
|
117
|
+
|
|
118
|
+
- id: string_cleaning
|
|
119
|
+
name: "String Cleaning (Trim / Lower)"
|
|
120
|
+
category: "clean"
|
|
121
|
+
pandas: "df['name'].str.strip().str.lower()"
|
|
122
|
+
polars: "df.with_columns(pl.col('name').str.strip_chars().str.to_lowercase())"
|
|
123
|
+
pyspark: "df.withColumn('name', F.lower(F.trim(F.col('name'))))"
|
|
124
|
+
gotchas:
|
|
125
|
+
pandas: "Accessor is .str; can be slow on large string series."
|
|
126
|
+
polars: "strip_chars() removes whitespace; strip_prefix() for prefixes."
|
|
127
|
+
pyspark: "Requires functions imported from pyspark.sql.functions as F."
|
|
128
|
+
|
|
129
|
+
# ==========================================
|
|
130
|
+
# 3. TRANSFORM (category: "transform")
|
|
131
|
+
# ==========================================
|
|
132
|
+
- id: add_column
|
|
133
|
+
name: "Add / Mutate Column"
|
|
134
|
+
category: "transform"
|
|
135
|
+
pandas: "df['bonus'] = df['salary'] * 0.1"
|
|
136
|
+
polars: "df.with_columns((pl.col('salary') * 0.1).alias('bonus'))"
|
|
137
|
+
pyspark: "df.withColumn('bonus', F.col('salary') * 0.1)"
|
|
138
|
+
gotchas:
|
|
139
|
+
pandas: "Can throw SettingWithCopyWarning on dataframe slices."
|
|
140
|
+
polars: "Place multiple expressions inside one with_columns() call for concurrency."
|
|
141
|
+
pyspark: "Chaining many withColumn calls degrades Catalyst optimizer plan."
|
|
142
|
+
|
|
143
|
+
- id: groupby_agg
|
|
144
|
+
name: "GroupBy & Aggregate"
|
|
145
|
+
category: "transform"
|
|
146
|
+
pandas: "df.groupby('dept').agg(avg_sal=('salary', 'mean')).reset_index()"
|
|
147
|
+
polars: "df.group_by('dept').agg(pl.col('salary').mean().alias('avg_sal'))"
|
|
148
|
+
pyspark: "df.groupBy('dept').agg(F.mean('salary').alias('avg_sal'))"
|
|
149
|
+
gotchas:
|
|
150
|
+
pandas: "Pandas creates index on group keys; must call reset_index()."
|
|
151
|
+
polars: "Ordering not preserved unless explicitly called with sort()."
|
|
152
|
+
pyspark: "Triggers wide network shuffle."
|
|
153
|
+
|
|
154
|
+
- id: join_tables
|
|
155
|
+
name: "Join Tables"
|
|
156
|
+
category: "transform"
|
|
157
|
+
pandas: "pd.merge(df1, df2, on='id', how='left')"
|
|
158
|
+
polars: "df1.join(df2, on='id', how='left')"
|
|
159
|
+
pyspark: "df1.join(df2, on='id', how='left')"
|
|
160
|
+
gotchas:
|
|
161
|
+
pandas: "Different keys use left_on='a', right_on='b'."
|
|
162
|
+
polars: "Different keys use left_on='a', right_on='b'."
|
|
163
|
+
pyspark: "Duplicate join columns remain in schema unless dropped or joined with array."
|
|
164
|
+
|
|
165
|
+
- id: reshape
|
|
166
|
+
name: "Reshape (Pivot / Melt)"
|
|
167
|
+
category: "transform"
|
|
168
|
+
pandas: "# Pivot wide:\ndf.pivot(index='id', columns='var', values='val')\n# Melt long:\ndf.melt(id_vars=['id'], value_vars=['a', 'b'])"
|
|
169
|
+
polars: "# Pivot wide:\ndf.pivot(on='var', index='id', values='val')\n# Melt long:\ndf.unpivot(index=['id'], on=['a', 'b'])"
|
|
170
|
+
pyspark: "# Pivot wide:\ndf.groupBy('id').pivot('var').sum('val')\n# Melt long:\ndf.selectExpr('id', 'stack(2, \"a\", a, \"b\", b) as (var, val)')"
|
|
171
|
+
gotchas:
|
|
172
|
+
pandas: "Pandas uses melt(); Polars 1.0+ renamed melt() to unpivot()."
|
|
173
|
+
polars: "Pivot requires eager DataFrame (not LazyFrame)."
|
|
174
|
+
pyspark: "Unpivot in Spark uses stack() expression via selectExpr."
|
|
175
|
+
|
|
176
|
+
- id: window_functions
|
|
177
|
+
name: "Window Functions (Rank)"
|
|
178
|
+
category: "transform"
|
|
179
|
+
pandas: "df['rank'] = df.groupby('dept')['salary'].rank(ascending=False)"
|
|
180
|
+
polars: "df.with_columns(pl.col('salary').rank(descending=True).over('dept').alias('rank'))"
|
|
181
|
+
pyspark: "w = Window.partitionBy('dept').orderBy(F.col('salary').desc())\ndf.withColumn('rank', F.rank().over(w))"
|
|
182
|
+
gotchas:
|
|
183
|
+
pandas: "Requires groupby followed by transform or rank."
|
|
184
|
+
polars: ".over() replaces complex joins without memory penalty."
|
|
185
|
+
pyspark: "Omitting partitionBy() loads all data into single node causing OOM."
|
|
186
|
+
|
|
187
|
+
# ==========================================
|
|
188
|
+
# 4. EDA (category: "eda")
|
|
189
|
+
# ==========================================
|
|
190
|
+
- id: univariate_counts
|
|
191
|
+
name: "Value Counts (Univariate)"
|
|
192
|
+
category: "eda"
|
|
193
|
+
pandas: "df['category'].value_counts(normalize=True)"
|
|
194
|
+
polars: "df['category'].value_counts(normalize=True)"
|
|
195
|
+
pyspark: "df.groupBy('category').count().withColumn('pct', F.col('count') / df.count())"
|
|
196
|
+
gotchas:
|
|
197
|
+
pandas: "Direct Series method."
|
|
198
|
+
polars: "Returns a DataFrame with columns [category, count]."
|
|
199
|
+
pyspark: "No native normalize param; calculate count / total."
|
|
200
|
+
|
|
201
|
+
- id: univariate_quantiles
|
|
202
|
+
name: "Quantiles / Percentiles (Univariate)"
|
|
203
|
+
category: "eda"
|
|
204
|
+
pandas: "df['age'].quantile([0.25, 0.5, 0.75])"
|
|
205
|
+
polars: "df.select([pl.col('age').quantile(0.5)])"
|
|
206
|
+
pyspark: "df.approxQuantile('age', [0.25, 0.5, 0.75], 0.01)"
|
|
207
|
+
gotchas:
|
|
208
|
+
pandas: "Calculates exact quantiles in-memory."
|
|
209
|
+
polars: "Supports exact quantile on Column/Series."
|
|
210
|
+
pyspark: "Uses Greenwald-Khanna approx algorithm; 0.01 is relative error."
|
|
211
|
+
|
|
212
|
+
- id: univariate_skew_kurt
|
|
213
|
+
name: "Skewness & Kurtosis (Univariate)"
|
|
214
|
+
category: "eda"
|
|
215
|
+
pandas: "skew = df['val'].skew()\nkurt = df['val'].kurt()"
|
|
216
|
+
polars: "df.select([pl.col('val').skew().alias('skew'), pl.col('val').kurtosis().alias('kurt')])"
|
|
217
|
+
pyspark: "df.select(F.skewness('val').alias('skew'), F.kurtosis('val').alias('kurt'))"
|
|
218
|
+
gotchas:
|
|
219
|
+
pandas: "Pandas calculates unbiased Fisher-Pearson kurtosis."
|
|
220
|
+
polars: "Zero-copy expression execution inside select."
|
|
221
|
+
pyspark: "Distributed computation via built-in functions."
|
|
222
|
+
|
|
223
|
+
- id: bivariate_corr
|
|
224
|
+
name: "Correlation Matrix (Bivariate)"
|
|
225
|
+
category: "eda"
|
|
226
|
+
pandas: "df[['col_a', 'col_b']].corr(method='pearson')"
|
|
227
|
+
polars: "df.select(pl.corr('col_a', 'col_b', method='pearson'))"
|
|
228
|
+
pyspark: "df.stat.corr('col_a', 'col_b', method='pearson')"
|
|
229
|
+
gotchas:
|
|
230
|
+
pandas: "corr() on full dataframe creates N x N matrix."
|
|
231
|
+
polars: "pl.corr() is an expression calculated between two columns."
|
|
232
|
+
pyspark: "stat.corr() works pairwise between two columns only."
|
|
233
|
+
|
|
234
|
+
- id: bivariate_crosstab
|
|
235
|
+
name: "Cross-Tabulation (Bivariate)"
|
|
236
|
+
category: "eda"
|
|
237
|
+
pandas: "pd.crosstab(df['cat_a'], df['cat_b'])"
|
|
238
|
+
polars: "df.pivot(on='cat_b', index='cat_a', values='cat_a', aggregate_function='len')"
|
|
239
|
+
pyspark: "df.stat.crosstab('cat_a', 'cat_b')"
|
|
240
|
+
gotchas:
|
|
241
|
+
pandas: "pd.crosstab is standalone function taking Series."
|
|
242
|
+
polars: "Polars builds cross-tab using pivot with aggregate_function='len'."
|
|
243
|
+
pyspark: "Native stat.crosstab returns a DataFrame."
|
|
244
|
+
|
|
245
|
+
- id: multivariate_cov
|
|
246
|
+
name: "Covariance (Multivariate)"
|
|
247
|
+
category: "eda"
|
|
248
|
+
pandas: "df[['a', 'b', 'c']].cov()"
|
|
249
|
+
polars: "df.select(pl.cov('a', 'b'))"
|
|
250
|
+
pyspark: "df.stat.cov('a', 'b')"
|
|
251
|
+
gotchas:
|
|
252
|
+
pandas: "Calculates full N x N covariance matrix."
|
|
253
|
+
polars: "Expression-based; compute pairs via pl.cov(col1, col2)."
|
|
254
|
+
pyspark: "stat.cov(col1, col2) calculates scalar covariance."
|
|
255
|
+
|
|
256
|
+
# ==========================================
|
|
257
|
+
# 5. EXPORT (category: "export")
|
|
258
|
+
# ==========================================
|
|
259
|
+
- id: save_parquet
|
|
260
|
+
name: "Export Parquet"
|
|
261
|
+
category: "export"
|
|
262
|
+
pandas: "df.to_parquet('output.parquet', compression='snappy')"
|
|
263
|
+
polars: "df.write_parquet('output.parquet', compression='snappy')"
|
|
264
|
+
pyspark: "df.write.mode('overwrite').parquet('output.parquet')"
|
|
265
|
+
gotchas:
|
|
266
|
+
pandas: "Requires pyarrow or fastparquet installed."
|
|
267
|
+
polars: "Native Rust multi-threaded writer with zstd/snappy."
|
|
268
|
+
pyspark: "Writes directory of partition files, not single file."
|
|
269
|
+
|
|
270
|
+
- id: save_csv
|
|
271
|
+
name: "Export CSV"
|
|
272
|
+
category: "export"
|
|
273
|
+
pandas: "df.to_csv('output.csv', index=False)"
|
|
274
|
+
polars: "df.write_csv('output.csv')"
|
|
275
|
+
pyspark: "df.write.mode('overwrite').csv('output.csv', header=True)"
|
|
276
|
+
gotchas:
|
|
277
|
+
pandas: "Always set index=False or it prepends extra unnamed column."
|
|
278
|
+
polars: "Never outputs row indexes."
|
|
279
|
+
pyspark: "Outputs part-* files in folder unless coalesced."
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: rosie-df
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: High-performance DataFrame Rosetta Stone powered by Polars
|
|
5
|
+
Author-email: anandlenin <anandijjina719@gmail.com>
|
|
6
|
+
Requires-Python: >=3.9
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: pyyaml>=6.0
|
|
9
|
+
Requires-Dist: polars>=1.0.0
|
|
10
|
+
|
|
11
|
+
Cheatsheet tool mapping syntax across Pandas, Polars, and PySpark.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
rosie_df/__init__.py
|
|
4
|
+
rosie_df/cli.py
|
|
5
|
+
rosie_df/core.py
|
|
6
|
+
rosie_df.egg-info/PKG-INFO
|
|
7
|
+
rosie_df.egg-info/SOURCES.txt
|
|
8
|
+
rosie_df.egg-info/dependency_links.txt
|
|
9
|
+
rosie_df.egg-info/entry_points.txt
|
|
10
|
+
rosie_df.egg-info/requires.txt
|
|
11
|
+
rosie_df.egg-info/top_level.txt
|
|
12
|
+
rosie_df/data/cheatsheet.yaml
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
rosie_df
|
rosie_df-0.1.0/setup.cfg
ADDED