dd-parser-cleaner 0.1.0__tar.gz

dd_parser_cleaner-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+.venv/
+uv.lock
+__pycache__/
+*.pyc
+.pytest_cache/
+.DS_Store
+dist/
+dd_analysis_results/
+dd_cleaner_results/
+documents/
+*.csv
+parser_run.log

dd_parser_cleaner-0.1.0/.python-version

@@ -0,0 +1 @@
+3.13

dd_parser_cleaner-0.1.0/.vscode/settings.json

@@ -0,0 +1,13 @@
+{
+  "python.analysis.extraPaths": [
+    "${workspaceFolder}/src"
+  ],
+  "python.analysis.packageIndexDepths": [
+    {
+      "name": "dd_parser",
+      "depth": 5
+    }
+  ],
+  "python.analysis.importFormat": "absolute",
+  "python.analysis.typeCheckingMode": "basic"
+}

dd_parser_cleaner-0.1.0/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 2.4
+Name: dd-parser-cleaner
+Version: 0.1.0
+Summary: A private, local LLM-powered data dictionary parser and entity mapper with automated cleaning.
+Requires-Python: >=3.10
+Requires-Dist: ollama>=0.2.0
+Requires-Dist: pandas>=2.2.0
+Requires-Dist: pydantic>=2.6.0
+Requires-Dist: pypdf>=4.1.0
+Requires-Dist: pyyaml>=6.0.1

dd_parser_cleaner-0.1.0/config.yaml

@@ -0,0 +1,22 @@
+# operational_settings
+batch_size: 10
+files:
+  - "sba_dd.csv"
+
+# llm_settings
+model_name: "llama3.2"
+temperature: 0.0
+system_prompt: "You are a precise data engineering assistant. Respond strictly in JSON."
+csv_target_column_index: 0
+
+# =====================================================================
+# Pipeline Ingestion/Execution Directory Targets
+# =====================================================================
+# dd_parser module outputs
+dd_parser_output_dir: "dd_analysis_results"      # Sub-dir where the blueprint lands
+output_filename: "sba_analysis_results.csv"      # Name of the handshake blueprint
+
+# dd_cleaner module outputs
+raw_dataset_file: "sba_loans_raw.csv"            # Sourced from the root working_dir
+dd_cleaner_output_dir: "dd_cleaner_results"      # Dedicated sub-dir for cleaned data
+clean_output_filename: "sba_loans_clean.csv"     # Final production dataset filename

dd_parser_cleaner-0.1.0/gemini/stash.md

@@ -0,0 +1,309 @@
+## 📑 Session Stash: Unified Project State & KMDS Document Reporting
+
+## 📌 Project State Summary
+
+* Workspace Title: `dd-parser-cleaner`
+* Active Platform Integration: Fully aligned with the `kmds-data-helper` ecosystem [15]. Ingests and processes data dictionary properties inside the `data_dictionary/` workspace [15], maps target source payloads out of the `data/` workspace [15], and drops clean, readable Markdown summaries directly into the `documents/` workspace [15].
+* Pipeline Handshake Status: Fully functional. The inference engine locks down a secure `# DD-PARSER-SIGNATURE` comment header row at the top of the mapping CSV, which the data cleaner validates before execution.
+* Execution Safety: Resolves case-variant header anomalies dynamically at runtime using a lowercase field map, ensuring type-casting rules apply flawlessly to mismatched dataset schemas.
+
+---
+
+## 📂 Active Unified Workspace Layout
+
+```text
+/home/rajiv/programming/dd_parser/     # Workspace Directory
+├── pyproject.toml                     # Distribution and entry point registry
+├── config.yaml                        # Centralized execution parameter file
+└── src/
+    ├── dd_parser/                     # LLM Inference and Heuristic Engine
+    │   ├── __init__.py
+    │   ├── cli.py
+    │   ├── core.py                    # Generates blueprint matrix + dd_parsing_summary.md
+    │   └── models.py                  # Pydantic schema validation contract
+    └── dd_cleaner/                    # Case-Insensitive Transformation Engine
+        ├── __init__.py
+        ├── cli.py
+        └── engine.py                  # Generates clean data payload + data_cleaning_summary.md
+```
+
+---
+
+## 📄 Core Code Matrix Updates
+
+## 1. Upgraded Project Settings (`pyproject.toml`)
+
+```toml
+[project]
+name = "dd-parser-cleaner"
+version = "0.1.0"
+description = "A private, local LLM-powered data dictionary parser and entity mapper with automated cleaning."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "pandas>=2.2.0",
+    "pydantic>=2.6.0",
+    "pypdf>=4.1.0",
+    "ollama>=0.2.0",
+    "pyyaml>=6.0.1",
+]
+
+[project.scripts]
+classify-entities = "dd_parser.cli:main"
+clean-dataset     = "dd_cleaner.cli:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dd_parser", "src/dd_cleaner"]
+```
+
+## 2. Runtime Execution Configuration (`config.yaml`)
+
+```yaml
+# operational_settings
+batch_size: 10
+files:
+  - "sba_dd.csv"
+
+# llm_settings
+model_name: "llama3.2"
+temperature: 0.0
+system_prompt: "You are a precise data engineering assistant. Respond strictly in JSON."
+csv_target_column_index: 0
+
+# =====================================================================
+# Pipeline Ingestion/Execution Directory Targets
+# =====================================================================
+# Document Report Analytics Target Location
+documents_dir: "documents"                     
+
+# dd_parser module outputs
+dd_parser_output_dir: "dd_analysis_results"    
+output_filename: "sba_analysis_results.csv"    
+
+# dd_cleaner module outputs
+raw_dataset_file: "sba_loans_raw.csv"          
+dd_cleaner_output_dir: "dd_cleaner_results"    
+clean_output_filename: "sba_loans_clean.csv"   
+```
+
+## 3. Reporting Parser Engine (`src/dd_parser/core.py`)
+
+```python
+importos
+importjson
+importyaml
+importtime
+importlogging
+importpandasas pd
+frompypdfimportPdfReader
+importollama
+fromtypingimportList, Dict, Any
+fromdd_parser.modelsimportAttributeAnalysis, BatchAnalysisResponse
+
+logger = logging.getLogger("dd_parser")
+
+classLocalEntityClassifier:
+    def__init__(self):
+        self.working_dir: str = ""
+        self.config: Dict[Any, Any] = {}
+
+    defset_working_config(self, working_dir: str, config_path: str):
+        abs_config_path = os.path.abspath(config_path)
+        ifnot os.path.exists(abs_config_path):
+            raise FileNotFoundError(f"Configuration file not found at: {abs_config_path}")
+          
+        with open(abs_config_path, 'r') asf:
+            self.config = yaml.safe_load(f)
+          
+        ifnot os.path.isdir(working_dir):
+            raise FileNotFoundError(f"Target data directory not found: {os.path.abspath(working_dir)}")
+        self.working_dir = os.path.abspath(working_dir)
+      
+        raw_output_dir = self.config.get('dd_parser_output_dir', 'dd_analysis_results')
+        abs_output_dir = os.path.isabs(raw_output_dir) and raw_output_dir or os.path.abspath(os.path.join(self.working_dir, raw_output_dir))
+        os.makedirs(abs_output_dir, exist_ok=True)
+      
+        log_file_path = os.path.join(abs_output_dir, "parser_run.log")
+        file_handler = logging.FileHandler(log_file_path, encoding='utf-8')
+        file_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S"))
+        logger.addHandler(file_handler)
+
+        logger.info("Context Initialized with Hybrid Processing Configuration.")
+
+    defextract_attributes(self, file_path: str, csv_idx: int = 0) -> List[str]:
+        _, ext = os.path.splitext(file_path)
+        ext = ext.lower()
+        if ext == '.csv':
+            return pd.read_csv(file_path).iloc[:, csv_idx].dropna().astype(str).tolist()
+        elif ext == '.pdf':
+            return [line.strip() forpagein PdfReader(file_path).pages forlinein page.extract_text().split('\n') if line.strip()]
+        elif ext in ['.md', '.markdown']:
+            with open(file_path, 'r', encoding='utf-8') asf:
+                return [line.strip() forlinein f if line.strip()]
+        raise ValueError(f"Unsupported format: {ext}")
+
+    defanalyze_batch(self, attributes: List[str]) -> List[AttributeAnalysis]:
+        prompt = f"""
+        Analyze the following data dictionary attributes. 
+      
+        ### EXAMPLES OF EXCELLENT PERFORMANCE
+        Input: ["BorrCity", "BankStreet", "GrossApproval", "SoldSecMrktInd"]
+        Output Schema Map:
+        {{
+            "analysis": [
+                {{"attribute_name": "BorrCity", "provisional_entity": "Borrower", "is_geographical": true, "related_entity": "Borrower", "provisional_python_type": "str"}},
+                {{"attribute_name": "BankStreet", "provisional_entity": "Bank", "is_geographical": true, "related_entity": "Bank", "provisional_python_type": "str"}},
+                {{"attribute_name": "GrossApproval", "provisional_entity": "Loan", "is_geographical": false, "related_entity": null, "provisional_python_type": "float"}},
+                {{"attribute_name": "SoldSecMrktInd", "provisional_entity": "Loan", "is_geographical": false, "related_entity": null, "provisional_python_type": "bool"}}
+            ]
+        }}
+
+        ### CURRENT EXECUTION BATCH
+        Attributes to process: {json.dumps(attributes)}
+        """
+        response = ollama.chat(
+            model=self.config.get('model_name', 'llama3.2'),
+            messages=[
+                {"role": "system", "content": self.config.get('system_prompt', 'You are a precise data engineering assistant. Respond strictly in JSON.')},
+                {"role": "user", "content": prompt}
+            ],
+            options={"temperature": 0.0},
+            format=BatchAnalysisResponse.model_json_schema()
+        )
+        return BatchAnalysisResponse(**json.loads(response['message']['content'])).analysis
+
+    defpost_process_cleaner(self, analysis_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        cleaned_records = []
+        foritemin analysis_list:
+            data = item.copy()
+            attr = data['attribute_name']
+      
+            if attr.startswith('Borr'):
+                data['provisional_entity'] = 'Borrower'
+                if data['is_geographical']: data['related_entity'] = 'Borrower'
+            elif attr.startswith('Bank'):
+                data['provisional_entity'] = 'Bank'
+                if attr in ['BankStreet', 'BankCity', 'BankState', 'BankZip']:
+                    data['is_geographical'] = True
+                    data['related_entity'] = 'Bank'
+            elif attr.startswith('Project'):
+                data['provisional_entity'] = 'Project'
+                if attr in ['ProjectCounty', 'ProjectState']:
+                    data['is_geographical'] = True
+                    data['related_entity'] = 'Project'
+            elif'Approval'in attr or'Disbursement'in attr or attr in ['Program', 'Subprogram']:
+                data['provisional_entity'] = 'Loan'
+            elif attr.startswith('SBA'):
+                data['provisional_entity'] = 'SBA'
+                if data['is_geographical']: data['related_entity'] = 'SBA'
+              
+            if attr.endswith('Ind') or'Indicator'in attr:
+                data['provisional_python_type'] = 'bool'
+              
+            ifnot data['is_geographical']:
+                data['related_entity'] = ""
+            cleaned_records.append(data)
+        return cleaned_records
+
+    defgenerate_parsing_markdown_summary(self, final_results: List[Dict[str, Any]], base_project_dir: str):
+        df = pd.DataFrame(final_results)
+        doc_dir_name = self.config.get('documents_dir', 'documents')
+        abs_doc_dir = os.path.abspath(os.path.join(base_project_dir, doc_dir_name))
+        os.makedirs(abs_doc_dir, exist_ok=True)
+      
+        report_path = os.path.join(abs_doc_dir, "dd_parsing_summary.md")
+        entity_counts = df['provisional_entity'].value_counts()
+        total_attributes = len(df)
+      
+        with open(report_path, 'w', encoding='utf-8') asf:
+            f.write("# 📑 KMDS Data Helper: Data Dictionary Parsing Summary\n\n")
+            f.write(f"**Total Tracked Attributes:** {total_attributes}  \n")
+            f.write(f"**Unique Detected Entities:** {len(entity_counts)}\n\n")
+          
+            f.write("## 🏗️ Entity Size & Distribution Profile\n")
+            f.write("| Detected Entity Node | Number of Attributes (Size) |\n")
+            f.write("| :--- | :--- |\n")
+            forent, countin entity_counts.items():
+                f.write(f"| {ent} | {count} |\n")
+            f.write("\n")
+          
+            f.write("## 🎛️ Attribute Structural Categories\n")
+            categorical_df = df[df['provisional_python_type'] == 'bool']
+            numerical_df = df[df['provisional_python_type'].isin(['int', 'float'])]
+            semantic_df = df[~df['provisional_python_type'].isin(['bool', 'int', 'float'])]
+          
+            f.write(f"### 📊 Categorical Fields (Total: {len(categorical_df)})\n")
+            for_, rowin categorical_df.iterrows():
+                f.write(f"- `{row['attribute_name']}` ({row['provisional_python_type']}) $\rightarrow$ Node: **{row['provisional_entity']}**\n")
+            f.write("\n")
+          
+            f.write(f"### 🔢 Numerical Fields (Total: {len(numerical_df)})\n")
+            for_, rowin numerical_df.iterrows():
+                f.write(f"- `{row['attribute_name']}` ({row['provisional_python_type']}) $\rightarrow$ Node: **{row['provisional_entity']}**\n")
+            f.write("\n")
+          
+            f.write(f"### 🧠 Semantic Attributes Grouped By Parent Class (Total: {len(semantic_df)})\n")
+            grouped_semantic = semantic_df.groupby('provisional_entity')
+            forent_group, group_dfin grouped_semantic:
+                f.write(f"#### Entity Category: `{ent_group}`\n")
+                for_, rowin group_df.iterrows():
+                    geo_suffix = row['is_geographical'] andf" [GEO Linked: {row['related_entity']}]"or""
+                    f.write(f"  - `{row['attribute_name']}` ({row['provisional_python_type']}){geo_suffix}\n")
+                f.write("\n")
+
+    defprocess(self):
+        files_to_process = self.config.get('files', [])
+        ifnot files_to_process: return
+
+        raw_output_dir = self.config.get('dd_parser_output_dir', 'dd_analysis_results')
+        abs_output_dir = os.path.isabs(raw_output_dir) and raw_output_dir or os.path.abspath(os.path.join(self.working_dir, raw_output_dir))
+      
+        batch_size, csv_col_idx = self.config.get('batch_size', 10), self.config.get('csv_target_column_index', 0)
+        config_filename = self.config.get('output_filename')
+        base_project_dir = os.path.abspath(os.path.join(self.working_dir, ".."))
+
+        forfilepathin files_to_process:
+            input_file_path = os.path.isabs(filepath) and filepath or os.path.abspath(os.path.join(self.working_dir, filepath))
+            ifnot os.path.exists(input_file_path): continue
+              
+            filename = os.path.basename(input_file_path)
+            try:
+                raw_attributes = self.extract_attributes(input_file_path, csv_col_idx)
+            except Exception ase: continue
+
+            final_results = []
+            foriin range(0, len(raw_attributes), batch_size):
+                batch = raw_attributes[i:i+batch_size]
+                try:
+                    batch_output = self.analyze_batch(batch)
+                    batch_dicts = [item.model_dump() foritemin batch_output]
+                    cleaned_batch = self.post_process_cleaner(batch_dicts)
+                    final_results.extend(cleaned_batch)
+                except Exception ase: pass
+
+            if final_results:
+                out_filename = (config_filename and len(files_to_process) == 1) and config_filename orf"mapped_{os.path.splitext(filename)}.csv"
+                output_csv_path = os.path.join(abs_output_dir, out_filename)
+              
+                preamble = f"# DD-PARSER-SIGNATURE: PROCESSED-BY-{self.config.get('model_name', 'llama3.2').upper()}\n"
+                with open(output_csv_path, 'w', encoding='utf-8') asf:
+                    f.write(preamble)
+                  
+                pd.DataFrame(final_results).to_csv(output_csv_path, mode='a', index=False)
+                self.generate_parsing_markdown_summary(final_results, base_project_dir)
+```
+
+---
+
+## 🚀 Ready for Next Sprint
+
+When you initiate your next tracking session, we will pick up directly with the data cleaner logic to add:
+
+* Geographic Scrubbing Routines: Processing columns tagged with `is_geographical: true` to enforce title casing (e.g. `"Colorado Springs"`) and length-padding string masks for postal indices [15] (e.g., zero-padding ZIP codes to a strict length of 5 digits).
+* Missing Value Custom Strategy Options: Designing explicit rules to safely substitute or isolate null cells based on attribute mappings.
+
+Let me know whenever you are ready to kick off the geographic data cleaning extensions!

dd_parser_cleaner-0.1.0/pyproject.toml

@@ -0,0 +1,25 @@
+[project]
+name = "dd-parser-cleaner"   # This renames the overall workspace project
+version = "0.1.0"
+description = "A private, local LLM-powered data dictionary parser and entity mapper with automated cleaning."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "pandas>=2.2.0",
+    "pydantic>=2.6.0",
+    "pypdf>=4.1.0",
+    "ollama>=0.2.0",
+    "pyyaml>=6.0.1",
+]
+
+[project.scripts]
+classify-entities = "dd_parser.cli:main"  # Reverted back to your original folder string
+clean-dataset     = "dd_cleaner.cli:main"   # Remains completely stable
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+# ─── ADD THIS BLOC TO RESOLVE THE WHEEL SELECTION BUG ───
+[tool.hatch.build.targets.wheel]
+packages = ["src/dd_parser", "src/dd_cleaner"]

dd_parser_cleaner-0.1.0/src/dd_cleaner/cli.py

@@ -0,0 +1,35 @@
+import argparse
+import logging
+import sys
+
+def main():
+    # 1. Initialize clean, standardized console log streaming
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S"
+    )
+
+    parser = argparse.ArgumentParser(
+        description="Run the data cleaner pipeline driven by the verified dd-parser blueprint matrix map."
+    )
+    parser.add_argument("--working-dir", required=True, help="Path to raw data directories containing target payloads.")
+    parser.add_argument("--config", required=True, help="Path to config.yaml file containing script targets.")
+    args = parser.parse_args()
+
+    # Import engine inside main to allow logging configs to bind cleanly first
+    from dd_cleaner.engine import DataCleanerEngine
+
+    try:
+        # 2. Fire up the pipeline runner
+        cleaner = DataCleanerEngine()
+        cleaner.set_working_config(working_dir=args.working_dir, config_path=args.config)
+        cleaner.clean_dataset()
+        
+    except Exception as e:
+        logger = logging.getLogger("dd_cleaner_main")
+        logger.error(f"🛑 Critical Pipeline Failure: {str(e)}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

dd_parser_cleaner-0.1.0/src/dd_cleaner/engine.py

@@ -0,0 +1,160 @@
+import os
+import logging
+import yaml
+import pandas as pd
+from typing import Dict, Any
+
+logger = logging.getLogger("dd_cleaner")
+
+class DataCleanerEngine:
+    def __init__(self):
+        self.working_dir: str = ""
+        self.config: Dict[Any, Any] = {}
+
+    def set_working_config(self, working_dir: str, config_path: str):
+        """Loads the shared config and binds the root working directory context."""
+        abs_config_path = os.path.abspath(config_path)
+        if not os.path.exists(abs_config_path):
+            raise FileNotFoundError(f"Configuration file not found at: {abs_config_path}")
+        with open(abs_config_path, 'r') as f:
+            self.config = yaml.safe_load(f)
+        if not os.path.isdir(working_dir):
+            raise FileNotFoundError(f"Target data directory not found: {os.path.abspath(working_dir)}")
+        self.working_dir = os.path.abspath(working_dir)
+        logger.info("Cleaner Context Initialized.")
+
+    def verify_and_load_blueprint(self) -> pd.DataFrame:
+        """Handshakes with the dd_parser output subdirectory to grab the metadata map."""
+        parser_dir = self.config.get('dd_parser_output_dir', 'dd_analysis_results')
+        blueprint_name = self.config.get('output_filename', 'sba_analysis_results.csv')
+        blueprint_path = os.path.isabs(parser_dir) and os.path.join(parser_dir, blueprint_name) or os.path.abspath(os.path.join(self.working_dir, parser_dir, blueprint_name))
+        
+        if not os.path.exists(blueprint_path):
+            raise FileNotFoundError(f"Missing parsing matrix blueprint file at: {blueprint_path}")
+            
+        with open(blueprint_path, 'r', encoding='utf-8') as f:
+            first_line = f.readline()
+            if not first_line.startswith("# DD-PARSER-SIGNATURE"):
+                raise ValueError(f"Rejected: File at {blueprint_path} does not originate from dd-parser pipeline!")
+                
+        return pd.read_csv(blueprint_path, comment='#')
+
+    def generate_cleaning_markdown_summary(self, data_df: pd.DataFrame, base_project_dir: str):
+        """Compiles clean data type breakdowns and null metrics to the documents/ workspace."""
+        doc_dir_name = self.config.get('documents_dir', 'documents')
+        abs_doc_dir = os.path.abspath(os.path.join(base_project_dir, doc_dir_name))
+        os.makedirs(abs_doc_dir, exist_ok=True)
+        
+        report_path = os.path.join(abs_doc_dir, "data_cleaning_summary.md")
+        type_summary = data_df.dtypes.value_counts()
+        null_counts = data_df.isnull().sum()
+        total_rows = len(data_df)
+        
+        with open(report_path, 'w', encoding='utf-8') as f:
+            f.write("# 🧼 KMDS Data Helper: Data Cleaning Summary Report\n\n")
+            f.write("## 📊 Converted Data Types Summary\n")
+            for dtype_name, count in type_summary.items():
+                f.write(f"| {dtype_name} | {count} |\n")
+            f.write("\n## 🗃️ Missing Value Counts\n")
+            for col in data_df.columns:
+                f.write(f"| `{col}` | {null_counts[col]} | {(total_rows - null_counts[col])/total_rows*100:.2f}% |\n")
+                
+        logger.info(f"Generated clean dataset validation profile saved to: {report_path}")
+
+    def execute_numeric_imputer(self, series: pd.Series) -> pd.Series:
+        """
+        Isolated strategy vector for missing continuous numerical data items.
+        Swap out this inner logic to upgrade from median to kNN/Iterative models later.
+        """
+        fill_value = series.median()
+        return series.fillna(fill_value)
+
+    def execute_poc_feature_prep(self, df: pd.DataFrame, blueprint_df: pd.DataFrame) -> pd.DataFrame:
+        """Applies basic missing value strategies and builds address strings for geocoding."""
+        prep_df = df.copy()
+        raw_cols_lower = {col.lower(): col for col in prep_df.columns}
+        
+        # 1. Extract and compile Geo attributes on a per-entity basis
+        geo_blueprint = blueprint_df[blueprint_df['is_geographical'] == True]
+        entity_geo_groups = geo_blueprint.groupby('provisional_entity')
+        
+        for entity_name, group in entity_geo_groups:
+            geo_cols = []
+            for _, row in group.iterrows():
+                attr_lower = row['attribute_name'].lower()
+                if attr_lower in raw_cols_lower:
+                    geo_cols.append(raw_cols_lower[attr_lower])
+            
+            if geo_cols:
+                logger.info(f" -> Consolidating geo attributes for entity: '{entity_name}'")
+                prep_df[f"{entity_name.lower()}_geo_search_string"] = prep_df[geo_cols].fillna("").astype(str).agg(", ".join, axis=1)
+        
+        # 2. Variable Strategy Loop driven by Schema Typing definitions
+        for _, row in blueprint_df.iterrows():
+            attr_lower = row['attribute_name'].lower()
+            if attr_lower not in raw_cols_lower:
+                continue
+            col_name = raw_cols_lower[attr_lower]
+            t_type = row['provisional_python_type']
+            
+            # Numeric Strategy: Route directly through decoupled method
+            if t_type in ['int', 'float']:
+                if prep_df[col_name].isnull().any():
+                    prep_df[col_name] = self.execute_numeric_imputer(prep_df[col_name])
+                    
+            # Categorical Strategy: Explicitly flag missing indices
+            elif t_type == 'str':
+                prep_df[col_name] = prep_df[col_name].replace(["nan", "None", ""], None).fillna("MISSING")
+                
+        return prep_df
+
+    def clean_dataset(self):
+        blueprint_df = self.verify_and_load_blueprint()
+        raw_file = self.config.get('raw_dataset_file', 'sba_loans_raw.csv')
+        base_project_dir = os.path.abspath(os.path.join(self.working_dir, ".."))
+        raw_path = os.path.isabs(raw_file) and raw_file or os.path.abspath(os.path.join(base_project_dir, "data", raw_file))
+        
+        data_df = pd.read_csv(raw_path)
+        raw_columns_lower = {col.lower(): col for col in data_df.columns}
+        
+        # [STAGE 1] Perform baseline type conversions
+        for _, row in blueprint_df.iterrows():
+            blueprint_attr = row['attribute_name']
+            target_type = row['provisional_python_type']
+            attr_lower = blueprint_attr.lower()
+            if attr_lower not in raw_columns_lower: continue
+            col_name = raw_columns_lower[attr_lower]
+            
+            try:
+                if target_type == 'bool':
+                    if data_df[col_name].dtype == object:
+                        data_df[col_name] = data_df[col_name].astype(str).str.upper().str.strip().isin(['TRUE', '1', 'Y', 'YES', 'T'])
+                    else:
+                        data_df[col_name] = data_df[col_name].fillna(False).astype(bool)
+                elif target_type == 'int':
+                    data_df[col_name] = pd.to_numeric(data_df[col_name], errors='coerce')
+                elif target_type == 'float':
+                    data_df[col_name] = pd.to_numeric(data_df[col_name], errors='coerce')
+                elif target_type in ['datetime.date', 'datetime.datetime']:
+                    data_df[col_name] = pd.to_datetime(data_df[col_name], errors='coerce')
+                else:
+                    data_df[col_name] = data_df[col_name].astype(str).str.strip()
+            except Exception: pass
+
+        # [STAGE 2] Write standard output and generate report
+        cleaner_dir = self.config.get('dd_cleaner_output_dir', 'dd_cleaner_results')
+        clean_filename = self.config.get('clean_output_filename', 'sba_loans_clean.csv')
+        abs_dest_dir = os.path.isabs(cleaner_dir) and cleaner_dir or os.path.abspath(os.path.join(base_project_dir, "data", cleaner_dir))
+        os.makedirs(abs_dest_dir, exist_ok=True)
+        
+        data_df.to_csv(os.path.join(abs_dest_dir, clean_filename), index=False)
+        self.generate_cleaning_markdown_summary(data_df, base_project_dir)
+        
+        # [STAGE 3] Run PoC feature prep AFTER generating the report
+        logger.info("Executing PoC Missing Value Strategies and Geo-string preparation...")
+        poc_ready_df = self.execute_poc_feature_prep(data_df, blueprint_df)
+        
+        # Save the finalized feature-selection dataset
+        poc_output_path = os.path.join(abs_dest_dir, "feature_selection_ready.csv")
+        poc_ready_df.to_csv(poc_output_path, index=False)
+        logger.info(f"🎉 Hand-off Complete! Modeling matrix saved to: {poc_output_path}")

dd_parser_cleaner-0.1.0/src/dd_parser/__init__.py

@@ -0,0 +1,9 @@
+# src/dd_parser/__init__.py
+
+from dd_parser.core import LocalEntityClassifier
+from dd_parser.models import AttributeAnalysis
+
+__all__ = [
+    "LocalEntityClassifier",
+    "AttributeAnalysis",
+]

dd_parser_cleaner-0.1.0/src/dd_parser/cli.py

@@ -0,0 +1,25 @@
+import argparse
+import logging
+
+def main():
+    # Setup initial clean console output streaming
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S"
+    )
+
+    parser = argparse.ArgumentParser(description="Run the private local LLM Data Dictionary Parser.")
+    parser.add_argument("--working-dir", required=True, help="Path to raw source files.")
+    parser.add_argument("--config", required=True, help="Path to config.yaml file.")
+    args = parser.parse_args()
+
+    # Import inside main to let logging configure cleanly first
+    from dd_parser.core import LocalEntityClassifier
+
+    classifier = LocalEntityClassifier()
+    classifier.set_working_config(working_dir=args.working_dir, config_path=args.config)
+    classifier.process()
+
+if __name__ == "__main__":
+    main()

dd_parser_cleaner-0.1.0/src/dd_parser/core.py

@@ -0,0 +1,224 @@
+import os
+import json
+import yaml
+import time
+import logging
+import pandas as pd
+from pypdf import PdfReader
+import ollama
+from typing import List, Dict, Any
+from dd_parser.models import AttributeAnalysis, BatchAnalysisResponse
+
+logger = logging.getLogger("dd_parser")
+
+class LocalEntityClassifier:
+    def __init__(self):
+        self.working_dir: str = ""
+        self.config: Dict[Any, Any] = {}
+
+    def set_working_config(self, working_dir: str, config_path: str):
+        abs_config_path = os.path.abspath(config_path)
+        if not os.path.exists(abs_config_path):
+            raise FileNotFoundError(f"Configuration file not found at: {abs_config_path}")
+            
+        with open(abs_config_path, 'r') as f:
+            self.config = yaml.safe_load(f)
+            
+        if not os.path.isdir(working_dir):
+            raise FileNotFoundError(f"Target data directory not found: {os.path.abspath(working_dir)}")
+        self.working_dir = os.path.abspath(working_dir)
+        
+        raw_output_dir = self.config.get('dd_parser_output_dir', 'dd_analysis_results')
+        abs_output_dir = os.path.isabs(raw_output_dir) and raw_output_dir or os.path.abspath(os.path.join(self.working_dir, raw_output_dir))
+        os.makedirs(abs_output_dir, exist_ok=True)
+        
+        log_file_path = os.path.join(abs_output_dir, "parser_run.log")
+        file_handler = logging.FileHandler(log_file_path, encoding='utf-8')
+        file_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S"))
+        logger.addHandler(file_handler)
+
+        logger.info("Context Initialized with Hybrid Processing Configuration.")
+        logger.info(f"Loaded Config: {abs_config_path} | Tracking Log: {log_file_path}")
+
+    def extract_attributes(self, file_path: str, csv_idx: int = 0) -> List[str]:
+        _, ext = os.path.splitext(file_path)
+        ext = ext.lower()
+        if ext == '.csv':
+            return pd.read_csv(file_path).iloc[:, csv_idx].dropna().astype(str).tolist()
+        elif ext == '.pdf':
+            return [line.strip() for page in PdfReader(file_path).pages for line in page.extract_text().split('\n') if line.strip()]
+        elif ext in ['.md', '.markdown']:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                return [line.strip() for line in f if line.strip()]
+        raise ValueError(f"Unsupported format: {ext}")
+
+    def analyze_batch(self, attributes: List[str]) -> List[AttributeAnalysis]:
+        prompt = f"""
+        Analyze the following data dictionary attributes. 
+        
+        ### EXAMPLES OF EXCELLENT PERFORMANCE
+        Input: ["BorrCity", "BankStreet", "GrossApproval", "SoldSecMrktInd"]
+        Output Schema Map:
+        {{
+            "analysis": [
+                {{"attribute_name": "BorrCity", "provisional_entity": "Borrower", "is_geographical": true, "related_entity": "Borrower", "provisional_python_type": "str"}},
+                {{"attribute_name": "BankStreet", "provisional_entity": "Bank", "is_geographical": true, "related_entity": "Bank", "provisional_python_type": "str"}},
+                {{"attribute_name": "GrossApproval", "provisional_entity": "Loan", "is_geographical": false, "related_entity": null, "provisional_python_type": "float"}},
+                {{"attribute_name": "SoldSecMrktInd", "provisional_entity": "Loan", "is_geographical": false, "related_entity": null, "provisional_python_type": "bool"}}
+            ]
+        }}
+
+        ### CURRENT EXECUTION BATCH
+        Attributes to process: {json.dumps(attributes)}
+        """
+        
+        response = ollama.chat(
+            model=self.config.get('model_name', 'llama3.2'),
+            messages=[
+                {"role": "system", "content": self.config.get('system_prompt', 'You are a precise data engineering assistant. Respond strictly in JSON.')},
+                {"role": "user", "content": prompt}
+            ],
+            options={"temperature": self.config.get('temperature', 0.0)},
+            format=BatchAnalysisResponse.model_json_schema()
+        )
+        return BatchAnalysisResponse(**json.loads(response['message']['content'])).analysis
+
+    def post_process_cleaner(self, analysis_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        cleaned_records = []
+        for item in analysis_list:
+            data = item.copy()
+            attr = data['attribute_name']
+        
+            if attr.startswith('Borr'):
+                data['provisional_entity'] = 'Borrower'
+                if data['is_geographical']: data['related_entity'] = 'Borrower'
+            elif attr.startswith('Bank'):
+                data['provisional_entity'] = 'Bank'
+                if attr in ['BankStreet', 'BankCity', 'BankState', 'BankZip']:
+                    data['is_geographical'] = True
+                    data['related_entity'] = 'Bank'
+            elif attr.startswith('Project'):
+                data['provisional_entity'] = 'Project'
+                if attr in ['ProjectCounty', 'ProjectState']:
+                    data['is_geographical'] = True
+                    data['related_entity'] = 'Project'
+            elif 'Approval' in attr or 'Disbursement' in attr or attr in ['Program', 'Subprogram']:
+                data['provisional_entity'] = 'Loan'
+            elif attr.startswith('SBA'):
+                data['provisional_entity'] = 'SBA'
+                if data['is_geographical']: data['related_entity'] = 'SBA'
+                
+            if attr.endswith('Ind') or 'Indicator' in attr:
+                data['provisional_python_type'] = 'bool'
+                
+            if not data['is_geographical']:
+                data['related_entity'] = ""
+            
+            cleaned_records.append(data)
+        return cleaned_records
+
+    def generate_parsing_markdown_summary(self, final_results: List[Dict[str, Any]], base_project_dir: str):
+        """Compiles structural data profiles and metadata metrics as a clean Markdown specification file."""
+        df = pd.DataFrame(final_results)
+        doc_dir_name = self.config.get('documents_dir', 'documents')
+        abs_doc_dir = os.path.abspath(os.path.join(base_project_dir, doc_dir_name))
+        os.makedirs(abs_doc_dir, exist_ok=True)
+        
+        report_path = os.path.join(abs_doc_dir, "dd_parsing_summary.md")
+        entity_counts = df['provisional_entity'].value_counts()
+        total_attributes = len(df)
+        
+        with open(report_path, 'w', encoding='utf-8') as f:
+            f.write("# 📑 KMDS Data Helper: Data Dictionary Parsing Summary\n\n")
+            f.write(f"**Total Tracked Attributes:** {total_attributes}  \n")
+            f.write(f"**Unique Detected Entities:** {len(entity_counts)}\n\n")
+            
+            f.write("## 🏗️ Entity Size & Distribution Profile\n")
+            f.write("| Detected Entity Node | Number of Attributes (Size) |\n")
+            f.write("| :--- | :--- |\n")
+            for ent, count in entity_counts.items():
+                f.write(f"| {ent} | {count} |\n")
+            f.write("\n")
+            
+            f.write("## 🎛️ Attribute Structural Categories\n")
+            categorical_df = df[df['provisional_python_type'] == 'bool']
+            numerical_df = df[df['provisional_python_type'].isin(['int', 'float'])]
+            semantic_df = df[~df['provisional_python_type'].isin(['bool', 'int', 'float'])]
+            
+            f.write(f"### 📊 Categorical Fields (Total: {len(categorical_df)})\n")
+            for _, row in categorical_df.iterrows():
+                f.write(f"- `{row['attribute_name']}` ({row['provisional_python_type']}) $\rightarrow$ Node: **{row['provisional_entity']}**\n")
+            f.write("\n")
+            
+            f.write(f"### 🔢 Numerical Fields (Total: {len(numerical_df)})\n")
+            for _, row in numerical_df.iterrows():
+                f.write(f"- `{row['attribute_name']}` ({row['provisional_python_type']}) $\rightarrow$ Node: **{row['provisional_entity']}**\n")
+            f.write("\n")
+            
+            f.write(f"### 🧠 Semantic Attributes Grouped By Parent Class (Total: {len(semantic_df)})\n")
+            grouped_semantic = semantic_df.groupby('provisional_entity')
+            for ent_group, group_df in grouped_semantic:
+                f.write(f"#### Entity Category: `{ent_group}`\n")
+                for _, row in group_df.iterrows():
+                    geo_suffix = row['is_geographical'] and f" [GEO Linked: {row['related_entity']}]" or ""
+                    f.write(f"  - `{row['attribute_name']}` ({row['provisional_python_type']}){geo_suffix}\n")
+                f.write("\n")
+                
+        logger.info(f"Generated parser metadata documentation report saved to: {report_path}")
+
+    def process(self):
+        files_to_process = self.config.get('files', [])
+        if not files_to_process: return
+
+        raw_output_dir = self.config.get('dd_parser_output_dir', 'dd_analysis_results')
+        abs_output_dir = os.path.isabs(raw_output_dir) and raw_output_dir or os.path.abspath(os.path.join(self.working_dir, raw_output_dir))
+        
+        batch_size, csv_col_idx = self.config.get('batch_size', 10), self.config.get('csv_target_column_index', 0)
+        config_filename = self.config.get('output_filename')
+
+        # Extract base project root for kmds-data-helper structure alignment
+        base_project_dir = os.path.abspath(os.path.join(self.working_dir, ".."))
+
+        for filepath in files_to_process:
+            input_file_path = os.path.isabs(filepath) and filepath or os.path.abspath(os.path.join(self.working_dir, filepath))
+            if not os.path.exists(input_file_path): continue
+                
+            filename = os.path.basename(input_file_path)
+            try:
+                raw_attributes = self.extract_attributes(input_file_path, csv_col_idx)
+                logger.info(f"Extracted {len(raw_attributes)} attributes from {filename}")
+            except Exception as e:
+                logger.exception(f"Read failure on {filename}: {e}")
+                continue
+
+            final_results = []
+            for i in range(0, len(raw_attributes), batch_size):
+                batch = raw_attributes[i:i+batch_size]
+                start_time = time.perf_counter()
+                try:
+                    batch_output = self.analyze_batch(batch)
+                    batch_dicts = [item.model_dump() for item in batch_output]
+                    cleaned_batch = self.post_process_cleaner(batch_dicts)
+                    final_results.extend(cleaned_batch)
+                    logger.info(f" Batch run {i} verified in {time.perf_counter() - start_time:.2f}s")
+                except Exception as e:
+                    logger.error(f" Batch crash at element index {i}: {e}")
+
+            if final_results:
+                if config_filename and len(files_to_process) == 1:
+                    out_filename = config_filename
+                else:
+                    base_name, _ = os.path.splitext(filename)
+                    out_filename = f"mapped_{base_name}.csv"
+                
+                output_csv_path = os.path.join(abs_output_dir, out_filename)
+                
+                preamble = f"# DD-PARSER-SIGNATURE: PROCESSED-BY-{self.config.get('model_name', 'llama3.2').upper()}\n"
+                with open(output_csv_path, 'w', encoding='utf-8') as f:
+                    f.write(preamble)
+                    
+                pd.DataFrame(final_results).to_csv(output_csv_path, mode='a', index=False)
+                logger.info(f"Finished tracking {filename}. Saved verified matrix map to: {output_csv_path}")
+                
+                # Dynamic Markdown Generation Task Trigger
+                self.generate_parsing_markdown_summary(final_results, base_project_dir)

dd_parser_cleaner-0.1.0/src/dd_parser/models.py

@@ -0,0 +1,21 @@
+from pydantic import BaseModel, Field
+from typing import List, Optional, Literal
+
+class AttributeAnalysis(BaseModel):
+    attribute_name: str
+    provisional_entity: str = Field(
+        description="The primary business entity this attribute belongs to (e.g., Customer, Product, Transaction)."
+    )
+    is_geographical: bool = Field(
+        description="True if the attribute represents a physical location, address, coordinate, country, or region."
+    )
+    related_entity: Optional[str] = Field(
+        None, 
+        description="If geographical, which entity does this location bind to? (e.g., 'Customer' for 'shipping_state')."
+    )
+    provisional_python_type: Literal["str", "int", "float", "datetime.date", "datetime.datetime", "bool"] = Field(
+        description="Semantic Python data type based on semantics."
+    )
+
+class BatchAnalysisResponse(BaseModel):
+    analysis: List[AttributeAnalysis]

dd_parser_cleaner-0.1.0/test_client.py

@@ -0,0 +1,27 @@
+import os
+from dd_parser import LocalEntityClassifier
+
+def run_local_test():
+    # Folder containing your dictionary files
+    target_data_dir = "/home/rajiv/programming/kmds_descriptive_analytics/kmds_sba_loans/data_dictionary"
+
+    # Path to the config file sitting in your current workspace
+    config_name = "config.yaml" 
+    
+    print("=== Starting dd_parser Integration Test ===")
+    
+    try:
+        classifier = LocalEntityClassifier()
+        
+        # FIXED: Pass config_path as the second keyword argument matching our refactored core engine
+        classifier.set_working_config(working_dir=target_data_dir, config_path=config_name)
+        
+        print("\n🚀 Dispatched extraction and Ollama micro-batching pipelines...")
+        classifier.process()
+        print("\n=== Test Finished Successfully ===")
+        
+    except Exception as e:
+        print(f"\n❌ Pipeline execution failed with exception: {e}")
+
+if __name__ == "__main__":
+    run_local_test()