bootlick 0.2.2__tar.gz → 0.2.3__tar.gz

{bootlick-0.2.2 → bootlick-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bootlick
-Version: 0.2.2
+Version: 0.2.3
 Summary: A package for specific websites
 Author-email: Unknown <unknown@example.com>
 License-Expression: MIT

{bootlick-0.2.2 → bootlick-0.2.3}/bootlick/nse_eod_prices.py

@@ -62,6 +62,149 @@ def get_dynamic_config():
     return get_config()
 
 
+# Canonical columns written to {date}_mf.csv. Process mode and historical files depend on these names.
+_AMFI_NAV_OUTPUT_COLUMNS = [
+    "code",
+    "scheme",
+    "isin_growth_dividend",
+    "isin_reinvestment",
+    "nav",
+    "repurchase_price",
+    "sale_price",
+    "date",
+    "category",
+]
+
+
+def _amfi_colmap(columns):
+    return {str(c).strip().lower(): c for c in columns}
+
+
+def _amfi_resolve_col(colmap, *aliases):
+    for alias in aliases:
+        if alias in colmap:
+            return colmap[alias]
+    return None
+
+
+def _is_amfi_isin(value):
+    """True for AMFI scheme ISINs (INF...), not Plan/Option text like 'INSTITUTIONAL...'."""
+    if not pd.notna(value):
+        return False
+    text = str(value).strip()
+    return len(text) >= 12 and text.startswith("IN") and " " not in text
+
+
+def _join_amfi_scheme(name, plan, option):
+    parts = []
+    for value in (name, plan, option):
+        if pd.notna(value):
+            text = str(value).strip()
+            if text:
+                parts.append(text)
+    return " - ".join(parts) if parts else pd.NA
+
+
+def _canonicalize_amfi_nav_frame(data: pd.DataFrame) -> pd.DataFrame:
+    """Map old (Repurchase/Sale) and new (Plan/Option) AMFI headers to one schema."""
+    colmap = _amfi_colmap(data.columns)
+    code_col = _amfi_resolve_col(colmap, "scheme code", "code")
+    name_col = _amfi_resolve_col(colmap, "scheme name", "nav name", "scheme")
+    isin_growth_col = _amfi_resolve_col(
+        colmap, "isin div payout/isin growth", "isin growth", "isin div payout"
+    )
+    isin_reinv_col = _amfi_resolve_col(colmap, "isin div reinvestment")
+    nav_col = _amfi_resolve_col(colmap, "net asset value", "nav")
+    date_col = _amfi_resolve_col(colmap, "date")
+    required = {
+        "scheme code": code_col,
+        "scheme name": name_col,
+        "isin div payout/isin growth": isin_growth_col,
+        "isin div reinvestment": isin_reinv_col,
+        "net asset value": nav_col,
+        "date": date_col,
+    }
+    missing = [name for name, col in required.items() if col is None]
+    if missing:
+        raise ValueError(
+            f"Unrecognized AMFI NAV header, missing: {missing}; got: {[str(c).strip() for c in data.columns]}"
+        )
+
+    scheme_name = data[name_col]
+    plan_col = _amfi_resolve_col(colmap, "plan")
+    option_col = _amfi_resolve_col(colmap, "option")
+    if plan_col is not None or option_col is not None:
+        plan = data[plan_col] if plan_col is not None else pd.Series(pd.NA, index=data.index)
+        option = data[option_col] if option_col is not None else pd.Series(pd.NA, index=data.index)
+        scheme = [_join_amfi_scheme(n, p, o) for n, p, o in zip(scheme_name, plan, option)]
+    else:
+        scheme = scheme_name
+
+    repurchase_col = _amfi_resolve_col(colmap, "repurchase price")
+    sale_col = _amfi_resolve_col(colmap, "sale price")
+    repurchase = data[repurchase_col] if repurchase_col is not None else pd.Series(pd.NA, index=data.index)
+    sale = data[sale_col] if sale_col is not None else pd.Series(pd.NA, index=data.index)
+    category = data["category"] if "category" in data.columns else pd.Series(pd.NA, index=data.index)
+
+    return pd.DataFrame(
+        {
+            "code": data[code_col],
+            "scheme": scheme,
+            "isin_growth_dividend": data[isin_growth_col],
+            "isin_reinvestment": data[isin_reinv_col],
+            "nav": data[nav_col],
+            "repurchase_price": repurchase,
+            "sale_price": sale,
+            "date": data[date_col],
+            "category": category,
+        }
+    )
+
+
+def _amfi_process_category(value):
+    try:
+        if value == "" or not np.isnan(pd.to_numeric(value, errors="coerce")):
+            return np.nan
+    except ValueError:
+        pass
+    return value
+
+
+def parse_amfi_nav_csv(content: str) -> pd.DataFrame:
+    """Parse AMFI NAV history text (legacy or Plan/Option header) into the saved CSV schema."""
+    data = pd.read_csv(
+        io.StringIO(content),
+        sep=";",
+        engine="python",
+        na_values=["", " "],
+        dtype=str,
+        skip_blank_lines=False,
+        on_bad_lines="warn",
+    )
+    data.columns = [str(c).strip() for c in data.columns]
+    data = data.dropna(how="all")
+    rows_to_remove = (data.isna().sum(axis=1) == len(data.columns) - 1) & data.iloc[:, 0].str.contains(
+        "Mutual Fund", na=False
+    )
+    data = data[~rows_to_remove]
+    data.loc[:, "category"] = data.iloc[:, 0]
+    data["category"] = data["category"].apply(_amfi_process_category)
+    data["category"] = data["category"].ffill()
+    data = data[data.isna().sum(axis=1) < len(data.columns) - 2]
+    data = _canonicalize_amfi_nav_frame(data)
+    for col in data.columns:
+        data[col] = data[col].apply(lambda x: x.strip() if isinstance(x, str) else x)
+    data["date"] = pd.to_datetime(data["date"], format="%d-%b-%Y")
+    data["isin_growth_dividend"] = data["isin_growth_dividend"].apply(
+        lambda x: x if _is_amfi_isin(x) else pd.NA
+    )
+    data["isin_reinvestment"] = data["isin_reinvestment"].apply(
+        lambda x: x if _is_amfi_isin(x) else pd.NA
+    )
+    data = data[data["isin_growth_dividend"].notna() | data["isin_reinvestment"].notna()]
+    return data[_AMFI_NAV_OUTPUT_COLUMNS]
+
+
 def catch(func, *args, handle=lambda e: e, **kwargs):
     try:
         return func(*args, **kwargs)
@@ -788,57 +931,7 @@ def save_mf_data(p: str, session, save_to_rds=True):
                     {"url": url, "preview": first_line[:200]},
                 )
                 return None
-            # AMFI mixes header rows, blank lines, and section titles without delimiters.
-            # Use the Python engine so pandas can handle rows with variable field counts.
-            data = pd.read_csv(
-                io.StringIO(content),
-                sep=";",
-                engine="python",
-                na_values=["", " "],
-                dtype=str,
-                skip_blank_lines=False,
-            )
-            data = data.dropna(how="all")
-            rows_to_remove = (data.isna().sum(axis=1) == len(data.columns) - 1) & data.iloc[:, 0].str.contains(
-                "Mutual Fund", na=False
-            )
-            data = data[~rows_to_remove]
-            # Transpose category to each row
-            data.loc[:, "category"] = data.iloc[:, 0]  # Assuming the category is in the first column
-            data["category"] = data["category"].apply(process_category)
-            data["category"] = data["category"].ffill()  # Forward-fill the NaN values
-
-            # Delete category headers
-            # Keep rows where the number of NaNs is less than the total number of columns minus 2
-            data = data[data.isna().sum(axis=1) < len(data.columns) - 2]
-            # Rename columns
-            data.columns = [
-                "code",
-                "scheme",
-                "isin_growth_dividend",
-                "isin_reinvestment",
-                "nav",
-                "repurchase_price",
-                "sale_price",
-                "date",
-                "category",
-            ]
-
-            # Convert 'date' to datetime format
-            data["date"] = pd.to_datetime(data["date"], format="%d-%b-%Y")
-
-            # Change 'isin' to NaN if it does not start with 'IN'
-            data["isin_growth_dividend"] = data["isin_growth_dividend"].apply(
-                lambda x: x if pd.notna(x) and x.startswith("IN") else pd.NA
-            )
-            data["isin_reinvestment"] = data["isin_reinvestment"].apply(
-                lambda x: x if pd.notna(x) and x.startswith("IN") else pd.NA
-            )
-
-            # Filter rows that have 'isin' to exclude unclaimed dividend info
-            data = data[data["isin_growth_dividend"].notna() | data["isin_reinvestment"].notna()]
-
-            # Save DataFrame to CSV
+            data = parse_amfi_nav_csv(content)
             save_pandas_in_csv_out(data, dest_file, index=True)
 
             get_bootlick_logger().log_info(
@@ -860,16 +953,6 @@ def save_mf_data(p: str, session, save_to_rds=True):
             return None
         return dest_file
 
-    # Custom function to handle the logic
-    def process_category(value):
-        try:
-            # Check if the value can be coerced into a numeric type or is an empty string
-            if value == "" or not np.isnan(pd.to_numeric(value, errors="coerce")):
-                return np.nan  # Replace with NaN
-        except ValueError:
-            pass
-        return value  # Keep the original value if it cannot be coerced
-
     try:
         outfolder = get_dynamic_config().get("daily_prices")
         if outfolder is None:
@@ -907,10 +990,12 @@ def save_mf_data(p: str, session, save_to_rds=True):
                     f"Processing date: {p}\n\n"
                     f"Please ensure the CSV file is downloaded first using download mode before processing.",
                 )
-                return
+                return False
             else:
                 # In download mode, download the file
                 dest_file = download_csv(url, dest_file)
+                if dest_file is None or not file_exists_and_valid(dest_file, min_size=1000):
+                    return False
         if not save_to_rds or dest_file is None or not file_exists_and_valid(dest_file, min_size=1000):
             return
         if file_exists_and_valid(dest_file):
@@ -1021,6 +1106,7 @@ def save_mf_data(p: str, session, save_to_rds=True):
             f"Unable to process mutual fund data for {p}",
             f"{traceback.format_exc()}",
         )
+        return False
 
 
 def save_option_data_old(p: str, session, save_to_rds: bool = True):

{bootlick-0.2.2 → bootlick-0.2.3}/bootlick.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bootlick
-Version: 0.2.2
+Version: 0.2.3
 Summary: A package for specific websites
 Author-email: Unknown <unknown@example.com>
 License-Expression: MIT

{bootlick-0.2.2 → bootlick-0.2.3}/pyproject.toml

@@ -27,7 +27,7 @@ include-package-data = true
 
 [project]
 name = "bootlick"
-version = "0.2.2"
+version = "0.2.3"
 description = "A package for specific websites"
 readme = "README.md"
 license = "MIT"