modulift 0.0.2__tar.gz → 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {modulift-0.0.2 → modulift-0.1.0}/PKG-INFO +16 -8
- modulift-0.1.0/modulift/__init__.py +8 -0
- modulift-0.1.0/modulift/api.py +307 -0
- modulift-0.1.0/modulift/funcs.py +43 -0
- {modulift-0.0.2 → modulift-0.1.0}/modulift.egg-info/PKG-INFO +16 -8
- {modulift-0.0.2 → modulift-0.1.0}/modulift.egg-info/SOURCES.txt +1 -1
- modulift-0.1.0/modulift.egg-info/requires.txt +10 -0
- {modulift-0.0.2 → modulift-0.1.0}/setup.py +5 -2
- modulift-0.0.2/modulift/__init__.py +0 -3
- modulift-0.0.2/modulift/api.py +0 -229
- modulift-0.0.2/modulift/data.py +0 -6
- modulift-0.0.2/modulift.egg-info/requires.txt +0 -7
- {modulift-0.0.2 → modulift-0.1.0}/modulift.egg-info/dependency_links.txt +0 -0
- {modulift-0.0.2 → modulift-0.1.0}/modulift.egg-info/top_level.txt +0 -0
- {modulift-0.0.2 → modulift-0.1.0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: modulift
|
|
3
|
-
Version: 0.0
|
|
3
|
+
Version: 0.1.0
|
|
4
4
|
Summary: A powerful search tool for discovering Python packages based on user queries
|
|
5
5
|
Home-page: https://github.com/okanyenigun/modulift
|
|
6
6
|
Author: Okan Yenigün
|
|
@@ -68,7 +68,7 @@ print(result)
|
|
|
68
68
|
'package': 'numpy',
|
|
69
69
|
'description': 'NumPy is the fundamental package for scientific computing with Python. It provides powerful N-dimensional arrays, sophisticated mathematical functions, tools for integrating C/C++ and Fortran code, and useful linear algebra, Fourier transform, and random number capabilities.',
|
|
70
70
|
'keywords': 'scientific computing, array, linear algebra, Fourier transform, random number generation, data processing, numerical analysis',
|
|
71
|
-
'popularity':
|
|
71
|
+
'popularity': 5
|
|
72
72
|
}
|
|
73
73
|
```
|
|
74
74
|
|
|
@@ -86,7 +86,7 @@ print(result)
|
|
|
86
86
|
'package': 'nullsweep',
|
|
87
87
|
'description': 'NullSweep is a Python library designed for detecting and handling patterns of missing data in pandas DataFrames. It provides a simple API to identify global missing data patterns across the entire dataset, patterns related to specific features within the dataset, and to impute missing values using various strategies.',
|
|
88
88
|
'keywords': 'missing data, data imputation, pandas, pattern detection, feature-specific patterns',
|
|
89
|
-
'popularity':
|
|
89
|
+
'popularity': 4
|
|
90
90
|
}
|
|
91
91
|
```
|
|
92
92
|
|
|
@@ -101,7 +101,7 @@ result = mf.search_by_package_name("numpy", markdown=True)
|
|
|
101
101
|
|
|
102
102
|
- **Description**: NumPy is the fundamental package for scientific computing with Python. It provides powerful N-dimensional arrays, sophisticated mathematical functions, tools for integrating C/C++ and Fortran code, and useful linear algebra, Fourier transform, and random number capabilities.
|
|
103
103
|
- **Keywords**: scientific computing, array, linear algebra, Fourier transform, random number generation, data processing, numerical analysis
|
|
104
|
-
- **Popularity**:
|
|
104
|
+
- **Popularity**: 5
|
|
105
105
|
|
|
106
106
|
### 2. Search by Keywords
|
|
107
107
|
|
|
@@ -145,7 +145,7 @@ Markdown Output
|
|
|
145
145
|
|
|
146
146
|
- **Description**: The SageMaker Python SDK is an open-source library designed for training and deploying machine learning models on Amazon SageMaker. It supports popular deep learning frameworks like Apache MXNet and TensorFlow, as well as Amazon's own scalable algorithms optimized for SageMaker. Users can also deploy custom algorithms in Docker containers, making it versatile for various machine learning tasks.
|
|
147
147
|
- **Keywords**: machine learning, deep learning, model deployment, Amazon SageMaker, data science, AI, Docker
|
|
148
|
-
- **Popularity**:
|
|
148
|
+
- **Popularity**: 5
|
|
149
149
|
|
|
150
150
|
---
|
|
151
151
|
|
|
@@ -153,7 +153,7 @@ Markdown Output
|
|
|
153
153
|
|
|
154
154
|
- **Description**: fastbook is a comprehensive resource for learning deep learning through practical coding examples using the fastai library and PyTorch. It is designed for coders of all backgrounds, providing accessible education on deep learning techniques and applications across various domains, including computer vision and natural language processing.
|
|
155
155
|
- **Keywords**: deep learning, fastai, PyTorch, computer vision, natural language processing, machine learning, data science, Jupyter Notebooks
|
|
156
|
-
- **Popularity**:
|
|
156
|
+
- **Popularity**: 5
|
|
157
157
|
|
|
158
158
|
---
|
|
159
159
|
|
|
@@ -161,7 +161,7 @@ Markdown Output
|
|
|
161
161
|
|
|
162
162
|
- **Description**: ProAiLab is a Python library designed for building and deploying AI models. It provides tools for data preprocessing, model training, and evaluation, making it suitable for both beginners and experienced machine learning practitioners.
|
|
163
163
|
- **Keywords**: machine learning, AI, deep learning, data science, model development
|
|
164
|
-
- **Popularity**:
|
|
164
|
+
- **Popularity**: 4
|
|
165
165
|
|
|
166
166
|
### 3. Full-Text Search in Descriptions
|
|
167
167
|
|
|
@@ -177,7 +177,15 @@ Markdown Output
|
|
|
177
177
|
|
|
178
178
|
- **Description**: The SageMaker Python SDK is an open-source library designed for training and deploying machine learning models on Amazon SageMaker. It supports popular deep learning frameworks like Apache MXNet and TensorFlow, as well as Amazon's own scalable algorithms optimized for SageMaker. Users can also deploy custom algorithms in Docker containers, making it versatile for various machine learning tasks.
|
|
179
179
|
- **Keywords**: machine learning, deep learning, model deployment, Amazon SageMaker, data science, AI, Docker
|
|
180
|
-
- **Popularity**:
|
|
180
|
+
- **Popularity**: 5
|
|
181
|
+
|
|
182
|
+
### 4. Find Similar Packages
|
|
183
|
+
|
|
184
|
+
Use cosine similarity to identify packages similar to a given package name.
|
|
185
|
+
|
|
186
|
+
```python
|
|
187
|
+
similar_packages = mf.find_similar_package("nullsweep")
|
|
188
|
+
```
|
|
181
189
|
|
|
182
190
|
## License
|
|
183
191
|
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
from typing import Literal, List, Dict
|
|
3
|
+
from .funcs import load_data, jaccard_similarity, markdown_print
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def search_by_keywords(
|
|
7
|
+
*args,
|
|
8
|
+
relation: Literal["and", "or"] = "or",
|
|
9
|
+
limit: int = 5,
|
|
10
|
+
method: Literal["exact", "jaccard"] = "exact",
|
|
11
|
+
markdown: bool = False,
|
|
12
|
+
) -> List[Dict]:
|
|
13
|
+
"""
|
|
14
|
+
Search for packages by keywords.
|
|
15
|
+
This function allows you to search for packages based on keywords. You can specify the relation between the keywords
|
|
16
|
+
(AND/OR) and the method of searching (exact match or Jaccard similarity). The results can be limited to a certain number
|
|
17
|
+
and can be printed in Markdown format.
|
|
18
|
+
|
|
19
|
+
Example:
|
|
20
|
+
search_by_keywords("data", "science", relation="and", limit=10, method="exact", markdown=True)
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
relation (Literal["and", "or"], optional): The relation between the keywords, or: any of the keywords must be present, and: all of the keywords must be present. Defaults to "or".
|
|
24
|
+
limit (int, optional): The maximum number of results to return. Defaults to 5.
|
|
25
|
+
method (Literal["exact", "jaccard"], optional): The method of searching, exact: exact match, jaccard: Jaccard similarity. Defaults to "exact".
|
|
26
|
+
markdown (bool, optional): If True, the results will be printed in Markdown format. Defaults to False.
|
|
27
|
+
|
|
28
|
+
Returns:
|
|
29
|
+
List[Dict]: A list of dictionaries containing the package name, description, keywords, and popularity. Example:
|
|
30
|
+
[
|
|
31
|
+
{
|
|
32
|
+
"package": "example-package",
|
|
33
|
+
"description": "An example package for demonstration purposes.",
|
|
34
|
+
"keywords": "example, demo, test",
|
|
35
|
+
"popularity": 4,
|
|
36
|
+
},
|
|
37
|
+
...
|
|
38
|
+
]
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
# Validation
|
|
42
|
+
if not args:
|
|
43
|
+
raise ValueError("At least one keyword must be provided.")
|
|
44
|
+
if relation not in ["and", "or"]:
|
|
45
|
+
raise ValueError("Relation must be 'and' or 'or'.")
|
|
46
|
+
if method not in ["exact", "jaccard"]:
|
|
47
|
+
raise ValueError("Method must be 'exact' or 'jaccard'.")
|
|
48
|
+
if limit <= 0:
|
|
49
|
+
raise ValueError("Limit must be a positive integer.")
|
|
50
|
+
|
|
51
|
+
keywords = [arg.lower() for arg in args]
|
|
52
|
+
|
|
53
|
+
# Load the dataset
|
|
54
|
+
df = load_data()
|
|
55
|
+
|
|
56
|
+
# Filter the dataset based on the keywords
|
|
57
|
+
if method == "exact":
|
|
58
|
+
if relation == "or":
|
|
59
|
+
mask = df["keywords"].apply(lambda cell: any(keyword in str(cell) for keyword in keywords))
|
|
60
|
+
elif relation == "and":
|
|
61
|
+
mask = df["keywords"].apply(lambda cell: all(keyword in str(cell) for keyword in keywords))
|
|
62
|
+
|
|
63
|
+
results = df[mask][["package", "description", "keywords", "popularity"]] \
|
|
64
|
+
.sort_values("popularity", ascending=False) \
|
|
65
|
+
.head(limit) \
|
|
66
|
+
.to_dict(orient="records")
|
|
67
|
+
elif method == "jaccard":
|
|
68
|
+
df["keyword_set"] = df["keywords"].apply(lambda x: set(x.split(", ")))
|
|
69
|
+
jaccard_scores = df["keyword_set"].apply(lambda x: jaccard_similarity(x, keywords))
|
|
70
|
+
jaccard_scores = pd.Series(jaccard_scores).sort_values(ascending=False).head(limit)
|
|
71
|
+
results = [
|
|
72
|
+
{
|
|
73
|
+
"package": df.iloc[i]["package"],
|
|
74
|
+
"description": df.iloc[i]["description"],
|
|
75
|
+
"keywords": df.iloc[i]["keywords"],
|
|
76
|
+
"popularity": df.iloc[i]["popularity"],
|
|
77
|
+
"jaccard_score": jaccard_scores[i],
|
|
78
|
+
}
|
|
79
|
+
for i in jaccard_scores.index
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
if markdown:
|
|
83
|
+
markdown_print(results)
|
|
84
|
+
|
|
85
|
+
return results
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def search_by_package_name(
|
|
89
|
+
package_name: str,
|
|
90
|
+
markdown: bool = False,
|
|
91
|
+
) -> Dict[str, str]:
|
|
92
|
+
"""
|
|
93
|
+
Search for a package by its name.
|
|
94
|
+
This function allows you to search for a package based on its name. If the package is found, its details will be returned.
|
|
95
|
+
|
|
96
|
+
Example:
|
|
97
|
+
search_by_package_name("example-package", markdown=True)
|
|
98
|
+
|
|
99
|
+
Args:
|
|
100
|
+
package_name (str): The name of the package to search for.
|
|
101
|
+
markdown (bool, optional): If True, the results will be printed in Markdown format. Defaults to False.
|
|
102
|
+
|
|
103
|
+
Returns:
|
|
104
|
+
Dict[str, str]: A dictionary containing the package name, description, keywords, and popularity. Example:
|
|
105
|
+
{
|
|
106
|
+
"package": "example-package",
|
|
107
|
+
"description": "An example package for demonstration purposes.",
|
|
108
|
+
"keywords": "example, demo, test",
|
|
109
|
+
"popularity": 4,
|
|
110
|
+
}
|
|
111
|
+
"""
|
|
112
|
+
# Validation
|
|
113
|
+
if not package_name:
|
|
114
|
+
raise ValueError("Package name must be provided.")
|
|
115
|
+
|
|
116
|
+
df = load_data()
|
|
117
|
+
|
|
118
|
+
# Filter the dataset based on the package name
|
|
119
|
+
result_df = df[df["package"] == package_name][["package", "description", "keywords", "popularity"]]
|
|
120
|
+
|
|
121
|
+
if result_df.empty:
|
|
122
|
+
result_df = df[df["package"].str.lower() == package_name.lower()][["package", "description", "keywords", "popularity"]]
|
|
123
|
+
|
|
124
|
+
if result_df.empty:
|
|
125
|
+
raise ValueError(f"No package found with the name '{package_name}'.")
|
|
126
|
+
|
|
127
|
+
result = result_df.iloc[0].to_dict()
|
|
128
|
+
|
|
129
|
+
if markdown:
|
|
130
|
+
markdown_print([result])
|
|
131
|
+
|
|
132
|
+
return result
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def search_by_description(
|
|
136
|
+
description: str,
|
|
137
|
+
limit: int = 5,
|
|
138
|
+
method: Literal["fts", "cosine"]="fts",
|
|
139
|
+
markdown: bool = False,
|
|
140
|
+
**kwargs
|
|
141
|
+
) -> List[Dict]:
|
|
142
|
+
"""
|
|
143
|
+
Search for packages by description.
|
|
144
|
+
This function allows you to search for packages based on their description. You can specify the method of searching
|
|
145
|
+
(full-text search or cosine similarity) and the number of results to return. The results can be printed in Markdown format.
|
|
146
|
+
|
|
147
|
+
Example:
|
|
148
|
+
search_by_description("data analysis", limit=10, method="fts", markdown=True)
|
|
149
|
+
|
|
150
|
+
Args:
|
|
151
|
+
description (str): The description to search for.
|
|
152
|
+
limit (int, optional): The maximum number of results to return. Defaults to 5.
|
|
153
|
+
method (Literal["fts", "cosine"], optional): The method of searching, fts: full-text search, cosine: cosine similarity. Defaults to "fts".
|
|
154
|
+
markdown (bool, optional): _description_. Defaults to False.
|
|
155
|
+
|
|
156
|
+
Returns:
|
|
157
|
+
List[Dict]: A list of dictionaries containing the package name, description, keywords, and popularity. Example:
|
|
158
|
+
[
|
|
159
|
+
{
|
|
160
|
+
"package": "example-package",
|
|
161
|
+
"description": "An example package for demonstration purposes.",
|
|
162
|
+
"keywords": "example, demo, test",
|
|
163
|
+
"popularity": 4,
|
|
164
|
+
},
|
|
165
|
+
...
|
|
166
|
+
]
|
|
167
|
+
"""
|
|
168
|
+
# Validation
|
|
169
|
+
if not description:
|
|
170
|
+
raise ValueError("Description must be provided.")
|
|
171
|
+
if limit <= 0:
|
|
172
|
+
raise ValueError("Limit must be a positive integer.")
|
|
173
|
+
if method not in ["fts", "cosine"]:
|
|
174
|
+
raise ValueError("Method must be 'fts' or 'cosine'.")
|
|
175
|
+
|
|
176
|
+
df = load_data()
|
|
177
|
+
|
|
178
|
+
# Filter the dataset based on the description
|
|
179
|
+
if method == "fts":
|
|
180
|
+
mask = df["description"].str.contains(description, case=False, na=False)
|
|
181
|
+
|
|
182
|
+
results = df[mask][["package", "description", "keywords", "popularity"]] \
|
|
183
|
+
.sort_values("popularity", ascending=False) \
|
|
184
|
+
.head(limit) \
|
|
185
|
+
.to_dict(orient="records")
|
|
186
|
+
elif method == "cosine":
|
|
187
|
+
import numpy as np
|
|
188
|
+
from sklearn.metrics.pairwise import cosine_similarity
|
|
189
|
+
from sentence_transformers import SentenceTransformer
|
|
190
|
+
|
|
191
|
+
embedding_model = kwargs.get("embedding_model", "sentence-transformers/all-MiniLM-L6-v2")
|
|
192
|
+
if embedding_model == "sentence-transformers/all-MiniLM-L6-v2":
|
|
193
|
+
print("Using default embedding model: sentence-transformers/all-MiniLM-L6-v2."
|
|
194
|
+
"If you want to use a different model, please specify it under the 'embedding_model' parameter."
|
|
195
|
+
)
|
|
196
|
+
model = SentenceTransformer(embedding_model)
|
|
197
|
+
|
|
198
|
+
query_embedding = model.encode(description, convert_to_numpy=True)
|
|
199
|
+
embeddings = np.stack(df["embedding"].values)
|
|
200
|
+
cosine_similarities = cosine_similarity([query_embedding], embeddings).flatten()
|
|
201
|
+
cosine_similarities = pd.Series(cosine_similarities).sort_values(ascending=False).head(limit)
|
|
202
|
+
results = [
|
|
203
|
+
{
|
|
204
|
+
"package": df.iloc[i]["package"],
|
|
205
|
+
"description": df.iloc[i]["description"],
|
|
206
|
+
"keywords": df.iloc[i]["keywords"],
|
|
207
|
+
"popularity": df.iloc[i]["popularity"],
|
|
208
|
+
"cosine_similarity": cosine_similarities[i],
|
|
209
|
+
}
|
|
210
|
+
for i in cosine_similarities.index
|
|
211
|
+
]
|
|
212
|
+
|
|
213
|
+
if markdown:
|
|
214
|
+
markdown_print(results)
|
|
215
|
+
|
|
216
|
+
return results
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def find_similar_packages(
|
|
220
|
+
reference_package: str,
|
|
221
|
+
limit: int = 5,
|
|
222
|
+
method: Literal["tf-idf", "cosine", "jaccard"] = "tf-idf",
|
|
223
|
+
markdown: bool = False,
|
|
224
|
+
) -> List[Dict]:
|
|
225
|
+
"""
|
|
226
|
+
Find similar packages based on a reference package.
|
|
227
|
+
This function allows you to find similar packages based on a reference package. You can specify the method of searching
|
|
228
|
+
(TF-IDF, cosine similarity, or Jaccard similarity) and the number of results to return. The results can be printed in Markdown format.
|
|
229
|
+
|
|
230
|
+
Example:
|
|
231
|
+
find_similar_packages("example-package", limit=10, method="tf-idf", markdown=True)
|
|
232
|
+
|
|
233
|
+
Args:
|
|
234
|
+
reference_package (str): The name of the reference package to search for.
|
|
235
|
+
limit (int, optional): The maximum number of results to return. Defaults to 5.
|
|
236
|
+
method (Literal["tf, optional): The method of searching, tf-idf: TF-IDF similarity, cosine: cosine similarity, jaccard: Jaccard similarity. Defaults to "tf-idf".
|
|
237
|
+
markdown (bool, optional): If True, the results will be printed in Markdown format. Defaults to False.
|
|
238
|
+
|
|
239
|
+
Returns:
|
|
240
|
+
List[Dict]: A list of dictionaries containing the package name, description, keywords, and popularity. Example:
|
|
241
|
+
[
|
|
242
|
+
{
|
|
243
|
+
"package": "example-package",
|
|
244
|
+
"description": "An example package for demonstration purposes.",
|
|
245
|
+
"keywords": "example, demo, test",
|
|
246
|
+
"popularity": 4,
|
|
247
|
+
},
|
|
248
|
+
...
|
|
249
|
+
]
|
|
250
|
+
"""
|
|
251
|
+
# Validation
|
|
252
|
+
if not reference_package:
|
|
253
|
+
raise ValueError("Reference package name must be provided.")
|
|
254
|
+
if limit <= 0:
|
|
255
|
+
raise ValueError("Limit must be a positive integer.")
|
|
256
|
+
if method not in ["tf-idf", "cosine", "jaccard"]:
|
|
257
|
+
raise ValueError("Method must be 'tf-idf', 'cosine', or 'jaccard'.")
|
|
258
|
+
|
|
259
|
+
df = load_data()
|
|
260
|
+
|
|
261
|
+
package_row = df.query("package == @reference_package")
|
|
262
|
+
if package_row.empty:
|
|
263
|
+
raise ValueError(f"No package found with the name '{reference_package}'. Ensure the name is correct.")
|
|
264
|
+
|
|
265
|
+
reference_idx = package_row.index[0]
|
|
266
|
+
|
|
267
|
+
if method == "tf-idf":
|
|
268
|
+
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
269
|
+
from sklearn.metrics.pairwise import cosine_similarity
|
|
270
|
+
|
|
271
|
+
tfidf_vectorizer = TfidfVectorizer(stop_words="english")
|
|
272
|
+
combined_text = df["description"] + " " + df["keywords"]
|
|
273
|
+
tfidf_matrix = tfidf_vectorizer.fit_transform(combined_text)
|
|
274
|
+
cosine_similarities = cosine_similarity(tfidf_matrix[reference_idx], tfidf_matrix).flatten()
|
|
275
|
+
candidates = pd.Series(cosine_similarities).sort_values(ascending=False).head(limit+1).index[1:]
|
|
276
|
+
|
|
277
|
+
elif method == "cosine":
|
|
278
|
+
import numpy as np
|
|
279
|
+
from sklearn.metrics.pairwise import cosine_similarity
|
|
280
|
+
|
|
281
|
+
query_embedding = df.iloc[reference_idx]["embedding"]
|
|
282
|
+
embeddings = np.stack(df["embedding"].values)
|
|
283
|
+
cosine_similarities = cosine_similarity([query_embedding], embeddings).flatten()
|
|
284
|
+
candidates = pd.Series(cosine_similarities).sort_values(ascending=False).head(limit+1).index[1:]
|
|
285
|
+
|
|
286
|
+
elif method == "jaccard":
|
|
287
|
+
df["keyword_set"] = df["keywords"].apply(lambda x: set(x.split(", ")))
|
|
288
|
+
query_keywords = df.iloc[reference_idx]["keywords"]
|
|
289
|
+
query_set = set(query_keywords.split(", "))
|
|
290
|
+
df_filtered = df["keyword_set"].apply(lambda x: jaccard_similarity(x, query_set))
|
|
291
|
+
candidates = pd.Series(df_filtered).sort_values(ascending=False).head(limit+1).index[1:]
|
|
292
|
+
|
|
293
|
+
results = [
|
|
294
|
+
{
|
|
295
|
+
"package": df.iloc[i]["package"],
|
|
296
|
+
"description": df.iloc[i]["description"],
|
|
297
|
+
"keywords": df.iloc[i]["keywords"],
|
|
298
|
+
"popularity": int(df.iloc[i]["popularity"]),
|
|
299
|
+
|
|
300
|
+
}
|
|
301
|
+
for i in candidates
|
|
302
|
+
]
|
|
303
|
+
|
|
304
|
+
if markdown:
|
|
305
|
+
markdown_print(results)
|
|
306
|
+
|
|
307
|
+
return results
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
from datasets import load_dataset
|
|
3
|
+
from datasets.exceptions import DatasetNotFoundError, DataFilesNotFoundError
|
|
4
|
+
from typing import Sequence, Any, List, Dict
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def load_data() -> pd.DataFrame:
|
|
8
|
+
"""
|
|
9
|
+
Load the dataset from the Hugging Face Hub.
|
|
10
|
+
"""
|
|
11
|
+
try:
|
|
12
|
+
dataset = load_dataset("yeniguno/pypi")["train"]
|
|
13
|
+
return dataset.to_pandas()
|
|
14
|
+
except DatasetNotFoundError as e:
|
|
15
|
+
print(f"🚫 Dataset not found: {e}")
|
|
16
|
+
except DataFilesNotFoundError as e:
|
|
17
|
+
print(f"🚫 Dataset repo exists but no matching files: {e}")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def jaccard_similarity(set1: Sequence, set2: Sequence) -> float:
|
|
21
|
+
"""
|
|
22
|
+
Calculate the Jaccard similarity between two sets.
|
|
23
|
+
"""
|
|
24
|
+
intersection = len(set1.intersection(set2))
|
|
25
|
+
union = len(set1.union(set2))
|
|
26
|
+
return intersection / union if union != 0 else 0
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def markdown_print(data: List[Dict[str, Any]]) -> None:
|
|
30
|
+
"""
|
|
31
|
+
Print the data in Markdown format.
|
|
32
|
+
"""
|
|
33
|
+
from IPython.display import display, Markdown
|
|
34
|
+
|
|
35
|
+
markdown_text= ""
|
|
36
|
+
|
|
37
|
+
for item in data:
|
|
38
|
+
for key, value in item.items():
|
|
39
|
+
markdown_text += f"**{key.capitalize()}:** {value}\n\n"
|
|
40
|
+
markdown_text += "\n---\n"
|
|
41
|
+
|
|
42
|
+
display(Markdown(markdown_text))
|
|
43
|
+
return
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: modulift
|
|
3
|
-
Version: 0.0
|
|
3
|
+
Version: 0.1.0
|
|
4
4
|
Summary: A powerful search tool for discovering Python packages based on user queries
|
|
5
5
|
Home-page: https://github.com/okanyenigun/modulift
|
|
6
6
|
Author: Okan Yenigün
|
|
@@ -68,7 +68,7 @@ print(result)
|
|
|
68
68
|
'package': 'numpy',
|
|
69
69
|
'description': 'NumPy is the fundamental package for scientific computing with Python. It provides powerful N-dimensional arrays, sophisticated mathematical functions, tools for integrating C/C++ and Fortran code, and useful linear algebra, Fourier transform, and random number capabilities.',
|
|
70
70
|
'keywords': 'scientific computing, array, linear algebra, Fourier transform, random number generation, data processing, numerical analysis',
|
|
71
|
-
'popularity':
|
|
71
|
+
'popularity': 5
|
|
72
72
|
}
|
|
73
73
|
```
|
|
74
74
|
|
|
@@ -86,7 +86,7 @@ print(result)
|
|
|
86
86
|
'package': 'nullsweep',
|
|
87
87
|
'description': 'NullSweep is a Python library designed for detecting and handling patterns of missing data in pandas DataFrames. It provides a simple API to identify global missing data patterns across the entire dataset, patterns related to specific features within the dataset, and to impute missing values using various strategies.',
|
|
88
88
|
'keywords': 'missing data, data imputation, pandas, pattern detection, feature-specific patterns',
|
|
89
|
-
'popularity':
|
|
89
|
+
'popularity': 4
|
|
90
90
|
}
|
|
91
91
|
```
|
|
92
92
|
|
|
@@ -101,7 +101,7 @@ result = mf.search_by_package_name("numpy", markdown=True)
|
|
|
101
101
|
|
|
102
102
|
- **Description**: NumPy is the fundamental package for scientific computing with Python. It provides powerful N-dimensional arrays, sophisticated mathematical functions, tools for integrating C/C++ and Fortran code, and useful linear algebra, Fourier transform, and random number capabilities.
|
|
103
103
|
- **Keywords**: scientific computing, array, linear algebra, Fourier transform, random number generation, data processing, numerical analysis
|
|
104
|
-
- **Popularity**:
|
|
104
|
+
- **Popularity**: 5
|
|
105
105
|
|
|
106
106
|
### 2. Search by Keywords
|
|
107
107
|
|
|
@@ -145,7 +145,7 @@ Markdown Output
|
|
|
145
145
|
|
|
146
146
|
- **Description**: The SageMaker Python SDK is an open-source library designed for training and deploying machine learning models on Amazon SageMaker. It supports popular deep learning frameworks like Apache MXNet and TensorFlow, as well as Amazon's own scalable algorithms optimized for SageMaker. Users can also deploy custom algorithms in Docker containers, making it versatile for various machine learning tasks.
|
|
147
147
|
- **Keywords**: machine learning, deep learning, model deployment, Amazon SageMaker, data science, AI, Docker
|
|
148
|
-
- **Popularity**:
|
|
148
|
+
- **Popularity**: 5
|
|
149
149
|
|
|
150
150
|
---
|
|
151
151
|
|
|
@@ -153,7 +153,7 @@ Markdown Output
|
|
|
153
153
|
|
|
154
154
|
- **Description**: fastbook is a comprehensive resource for learning deep learning through practical coding examples using the fastai library and PyTorch. It is designed for coders of all backgrounds, providing accessible education on deep learning techniques and applications across various domains, including computer vision and natural language processing.
|
|
155
155
|
- **Keywords**: deep learning, fastai, PyTorch, computer vision, natural language processing, machine learning, data science, Jupyter Notebooks
|
|
156
|
-
- **Popularity**:
|
|
156
|
+
- **Popularity**: 5
|
|
157
157
|
|
|
158
158
|
---
|
|
159
159
|
|
|
@@ -161,7 +161,7 @@ Markdown Output
|
|
|
161
161
|
|
|
162
162
|
- **Description**: ProAiLab is a Python library designed for building and deploying AI models. It provides tools for data preprocessing, model training, and evaluation, making it suitable for both beginners and experienced machine learning practitioners.
|
|
163
163
|
- **Keywords**: machine learning, AI, deep learning, data science, model development
|
|
164
|
-
- **Popularity**:
|
|
164
|
+
- **Popularity**: 4
|
|
165
165
|
|
|
166
166
|
### 3. Full-Text Search in Descriptions
|
|
167
167
|
|
|
@@ -177,7 +177,15 @@ Markdown Output
|
|
|
177
177
|
|
|
178
178
|
- **Description**: The SageMaker Python SDK is an open-source library designed for training and deploying machine learning models on Amazon SageMaker. It supports popular deep learning frameworks like Apache MXNet and TensorFlow, as well as Amazon's own scalable algorithms optimized for SageMaker. Users can also deploy custom algorithms in Docker containers, making it versatile for various machine learning tasks.
|
|
179
179
|
- **Keywords**: machine learning, deep learning, model deployment, Amazon SageMaker, data science, AI, Docker
|
|
180
|
-
- **Popularity**:
|
|
180
|
+
- **Popularity**: 5
|
|
181
|
+
|
|
182
|
+
### 4. Find Similar Packages
|
|
183
|
+
|
|
184
|
+
Use cosine similarity to identify packages similar to a given package name.
|
|
185
|
+
|
|
186
|
+
```python
|
|
187
|
+
similar_packages = mf.find_similar_package("nullsweep")
|
|
188
|
+
```
|
|
181
189
|
|
|
182
190
|
## License
|
|
183
191
|
|
|
@@ -7,7 +7,7 @@ with open("Readme.md", "r") as f:
|
|
|
7
7
|
|
|
8
8
|
setup(
|
|
9
9
|
name="modulift",
|
|
10
|
-
version="0.0
|
|
10
|
+
version="0.1.0",
|
|
11
11
|
description="A powerful search tool for discovering Python packages based on user queries",
|
|
12
12
|
packages=find_packages(),
|
|
13
13
|
long_description=long_description,
|
|
@@ -35,7 +35,10 @@ setup(
|
|
|
35
35
|
'Operating System :: OS Independent'
|
|
36
36
|
],
|
|
37
37
|
install_requires=[
|
|
38
|
-
'datasets==3.1.0'
|
|
38
|
+
'datasets==3.1.0',
|
|
39
|
+
'scikit-learn==1.6.1',
|
|
40
|
+
'ipywidgets==8.1.5',
|
|
41
|
+
'sentence-transformers==3.4.1',
|
|
39
42
|
],
|
|
40
43
|
extras_require={
|
|
41
44
|
'dev': ['twine==5.1.1'],
|
modulift-0.0.2/modulift/api.py
DELETED
|
@@ -1,229 +0,0 @@
|
|
|
1
|
-
from IPython.display import display, Markdown
|
|
2
|
-
from typing import List, Optional
|
|
3
|
-
from .data import load_data
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
def search_by_keywords(*args: str, relation: str = "or", limit: Optional[int] = None, markdown: bool = False) -> List[dict]:
|
|
7
|
-
"""
|
|
8
|
-
Search for keywords in the 'keywords' column with a specified relation ('or' or 'and') and optional limit on results.
|
|
9
|
-
|
|
10
|
-
Parameters:
|
|
11
|
-
*args (str): Keywords to search for in the 'keywords' column.
|
|
12
|
-
relation (str, optional): Defines the logical relationship for keyword matching.
|
|
13
|
-
Use "or" for matching any keyword, and "and" for matching all keywords.
|
|
14
|
-
Defaults to "or".
|
|
15
|
-
limit (int, optional): The maximum number of results to return. Defaults to None, which returns all matches.
|
|
16
|
-
markdown (bool, optional): If True, prints the results in markdown format. Defaults to False.
|
|
17
|
-
|
|
18
|
-
Returns:
|
|
19
|
-
List[dict]: A list of dictionaries with 'package', 'description', 'keywords', and 'popularity' fields
|
|
20
|
-
for each row that matches the search criteria, sorted by 'popularity' in the specified order.
|
|
21
|
-
|
|
22
|
-
Notes:
|
|
23
|
-
- The 'popularity' column is sorted in the order ['very popular', 'popular', 'not popular'].
|
|
24
|
-
- Markdown output is only printed and does not affect the returned result structure.
|
|
25
|
-
|
|
26
|
-
Example:
|
|
27
|
-
>>> search_by_keywords("machine learning", "data science", relation="and", limit=5, markdown=True)
|
|
28
|
-
[
|
|
29
|
-
{
|
|
30
|
-
'package': 'example-package',
|
|
31
|
-
'description': 'An example package for machine learning.',
|
|
32
|
-
'keywords': 'machine learning, data science, AI',
|
|
33
|
-
'popularity': 'very popular'
|
|
34
|
-
},
|
|
35
|
-
...
|
|
36
|
-
]
|
|
37
|
-
"""
|
|
38
|
-
# Validate and normalize inputs
|
|
39
|
-
if not args:
|
|
40
|
-
raise ValueError("At least one keyword must be provided for search.")
|
|
41
|
-
|
|
42
|
-
if relation not in {"or", "and"}:
|
|
43
|
-
raise ValueError("The 'relation' parameter must be either 'or' or 'and'.")
|
|
44
|
-
|
|
45
|
-
if limit is not None and (not isinstance(limit, int) or limit <= 0):
|
|
46
|
-
raise ValueError("The 'limit' parameter must be a positive integer if specified.")
|
|
47
|
-
|
|
48
|
-
# Load the DataFrame
|
|
49
|
-
df = load_data()
|
|
50
|
-
|
|
51
|
-
# Convert the input keywords to a list
|
|
52
|
-
keywords = list(args)
|
|
53
|
-
|
|
54
|
-
# Define the boolean mask based on 'or' or 'and' logic
|
|
55
|
-
if relation == "and":
|
|
56
|
-
mask = df['keywords'].apply(lambda cell: all(keyword.lower() in str(cell).lower() for keyword in keywords))
|
|
57
|
-
else: # default to 'or' logic
|
|
58
|
-
mask = df['keywords'].apply(lambda cell: any(keyword.lower() in str(cell).lower() for keyword in keywords))
|
|
59
|
-
|
|
60
|
-
# Filter the DataFrame with matching rows and select required columns
|
|
61
|
-
result_df = df[mask][['package', 'description', 'keywords', 'popularity']]
|
|
62
|
-
|
|
63
|
-
# Sort the DataFrame by 'POPULARITY' based on the defined order
|
|
64
|
-
result_df = result_df.sort_values('popularity', ascending=False)
|
|
65
|
-
|
|
66
|
-
# Apply limit if specified
|
|
67
|
-
if limit is not None:
|
|
68
|
-
result_df = result_df.head(limit)
|
|
69
|
-
|
|
70
|
-
# Convert to a list of dictionaries
|
|
71
|
-
results = result_df.to_dict(orient='records')
|
|
72
|
-
|
|
73
|
-
# Print in markdown format if markdown is True
|
|
74
|
-
if markdown:
|
|
75
|
-
for i, row in enumerate(results, 1):
|
|
76
|
-
|
|
77
|
-
package = row['package'].strip()
|
|
78
|
-
description = row['description'].strip()
|
|
79
|
-
keywords = row['keywords'].strip()
|
|
80
|
-
popularity = str(row['popularity'])
|
|
81
|
-
|
|
82
|
-
markdown_text = f"""
|
|
83
|
-
**{package}**
|
|
84
|
-
|
|
85
|
-
- **Description**: {description}
|
|
86
|
-
- **Keywords**: {keywords}
|
|
87
|
-
- **Popularity**: {popularity}
|
|
88
|
-
|
|
89
|
-
---
|
|
90
|
-
"""
|
|
91
|
-
display(Markdown(markdown_text))
|
|
92
|
-
|
|
93
|
-
return results
|
|
94
|
-
|
|
95
|
-
def search_by_package_name(package_name: str, markdown: bool = False) -> Optional[dict]:
|
|
96
|
-
"""
|
|
97
|
-
Search for an exact package name in the 'package' column of the DataFrame.
|
|
98
|
-
|
|
99
|
-
Parameters:
|
|
100
|
-
package_name (str): The exact package name to search for.
|
|
101
|
-
markdown (bool, optional): If True, prints the result in markdown format. Defaults to False.
|
|
102
|
-
|
|
103
|
-
Returns:
|
|
104
|
-
Optional[dict]: A dictionary with 'package', 'description', 'keywords', and 'popularity' fields if a match is found;
|
|
105
|
-
otherwise, returns None.
|
|
106
|
-
|
|
107
|
-
Example:
|
|
108
|
-
>>> search_by_package_name("example-package", markdown=True)
|
|
109
|
-
{
|
|
110
|
-
'package': 'example-package',
|
|
111
|
-
'description': 'An example package for data processing.',
|
|
112
|
-
'keywords': 'data processing, analysis, example',
|
|
113
|
-
'popularity': 'very popular'
|
|
114
|
-
}
|
|
115
|
-
"""
|
|
116
|
-
if not isinstance(package_name, str) or not package_name.strip():
|
|
117
|
-
raise ValueError("The 'package_name' parameter must be a non-empty string.")
|
|
118
|
-
|
|
119
|
-
# Load the DataFrame
|
|
120
|
-
df = load_data()
|
|
121
|
-
|
|
122
|
-
# Perform exact match search in the 'package' column
|
|
123
|
-
result_df = df[df['package'] == package_name][['package', 'description', 'keywords', 'popularity']]
|
|
124
|
-
|
|
125
|
-
# Check if there is a match
|
|
126
|
-
if not result_df.empty:
|
|
127
|
-
result = result_df.iloc[0].to_dict()
|
|
128
|
-
|
|
129
|
-
# Print in markdown format if markdown is True
|
|
130
|
-
if markdown:
|
|
131
|
-
|
|
132
|
-
package = result['package'].strip()
|
|
133
|
-
description = result['description'].strip()
|
|
134
|
-
keywords = result['keywords'].strip()
|
|
135
|
-
popularity = str(result['popularity'])
|
|
136
|
-
|
|
137
|
-
markdown_text = f"""
|
|
138
|
-
**{package}**
|
|
139
|
-
|
|
140
|
-
- **Description**: {description}
|
|
141
|
-
- **Keywords**: {keywords}
|
|
142
|
-
- **Popularity**: {popularity}
|
|
143
|
-
|
|
144
|
-
---
|
|
145
|
-
"""
|
|
146
|
-
|
|
147
|
-
display(Markdown(markdown_text))
|
|
148
|
-
|
|
149
|
-
return result
|
|
150
|
-
else:
|
|
151
|
-
if markdown:
|
|
152
|
-
print(f"No match found for package name '{package_name}'.")
|
|
153
|
-
return None
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
def search_by_description(text: str, limit: Optional[int] = None, markdown: bool = False) -> List[dict]:
|
|
157
|
-
"""
|
|
158
|
-
Perform a full-text search in the 'description' column for a specific text, with an optional limit and markdown output.
|
|
159
|
-
|
|
160
|
-
Parameters:
|
|
161
|
-
text (str): The text to search for in the 'description' column.
|
|
162
|
-
limit (int, optional): The maximum number of results to return. Defaults to None, which returns all matches.
|
|
163
|
-
markdown (bool, optional): If True, prints the results in markdown format. Defaults to False.
|
|
164
|
-
|
|
165
|
-
Returns:
|
|
166
|
-
List[dict]: A list of dictionaries with 'package', 'description', 'keywords', and 'popularity' fields
|
|
167
|
-
for rows where the 'description' column contains the search text, limited by 'limit' if specified.
|
|
168
|
-
|
|
169
|
-
Example:
|
|
170
|
-
>>> search_by_description("data analysis", limit=5, markdown=True)
|
|
171
|
-
[
|
|
172
|
-
{
|
|
173
|
-
'package': 'example-package',
|
|
174
|
-
'description': 'A package for data analysis and visualization.',
|
|
175
|
-
'keywords': 'data, analysis, visualization',
|
|
176
|
-
'popularity': 'very popular'
|
|
177
|
-
},
|
|
178
|
-
...
|
|
179
|
-
]
|
|
180
|
-
"""
|
|
181
|
-
# Validate 'text' input
|
|
182
|
-
if not isinstance(text, str) or not text.strip():
|
|
183
|
-
raise ValueError("The 'text' parameter must be a non-empty string.")
|
|
184
|
-
|
|
185
|
-
# Validate 'limit' parameter
|
|
186
|
-
if limit is not None and (not isinstance(limit, int) or limit <= 0):
|
|
187
|
-
raise ValueError("The 'limit' parameter must be a positive integer if specified.")
|
|
188
|
-
|
|
189
|
-
# Load the DataFrame
|
|
190
|
-
df = load_data()
|
|
191
|
-
|
|
192
|
-
# Perform case-insensitive full-text search in the 'description' column
|
|
193
|
-
mask = df['description'].str.contains(text, case=False, na=False)
|
|
194
|
-
|
|
195
|
-
# Filter the DataFrame with matching rows and select required columns
|
|
196
|
-
result_df = df[mask][['package', 'description', 'keywords', 'popularity']]
|
|
197
|
-
|
|
198
|
-
# Sort the DataFrame by 'POPULARITY' based on the defined order
|
|
199
|
-
result_df = result_df.sort_values('popularity', ascending=False)
|
|
200
|
-
|
|
201
|
-
# Apply limit if specified
|
|
202
|
-
if limit is not None:
|
|
203
|
-
result_df = result_df.head(limit)
|
|
204
|
-
|
|
205
|
-
# Convert to a list of dictionaries
|
|
206
|
-
results = result_df.to_dict(orient='records')
|
|
207
|
-
|
|
208
|
-
# Print in markdown format if markdown is True
|
|
209
|
-
if markdown:
|
|
210
|
-
for i, row in enumerate(results, 1):
|
|
211
|
-
|
|
212
|
-
package = row['package'].strip()
|
|
213
|
-
description = row['description'].strip()
|
|
214
|
-
keywords = row['keywords'].strip()
|
|
215
|
-
popularity = str(row['popularity'])
|
|
216
|
-
|
|
217
|
-
markdown_text = f"""
|
|
218
|
-
**{package}**
|
|
219
|
-
|
|
220
|
-
- **Description**: {description}
|
|
221
|
-
- **Keywords**: {keywords}
|
|
222
|
-
- **Popularity**: {popularity}
|
|
223
|
-
|
|
224
|
-
---
|
|
225
|
-
"""
|
|
226
|
-
display(Markdown(markdown_text))
|
|
227
|
-
else:
|
|
228
|
-
print(f"No matches found for the text '{text}'.")
|
|
229
|
-
return results
|
modulift-0.0.2/modulift/data.py
DELETED
|
File without changes
|
|
File without changes
|
|
File without changes
|