xelytics-core 0.1.0__tar.gz

xelytics_core-0.1.0/PKG-INFO

@@ -0,0 +1,104 @@
+Metadata-Version: 2.4
+Name: xelytics-core
+Version: 0.1.0
+Summary: Pure analytics engine for statistical analysis and insight generation
+Author: Xelytics Team
+License: MIT
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: pandas>=2.1.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: scipy>=1.11.0
+Requires-Dist: scikit-learn>=1.3.0
+Requires-Dist: statsmodels>=0.14.0
+Requires-Dist: pingouin>=0.5.3
+Requires-Dist: plotly>=5.17.0
+Provides-Extra: llm
+Requires-Dist: openai>=1.6.0; extra == "llm"
+Requires-Dist: groq>=0.4.0; extra == "llm"
+Requires-Dist: httpx>=0.25.0; extra == "llm"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
+Requires-Dist: black>=23.11.0; extra == "dev"
+Requires-Dist: mypy>=1.7.0; extra == "dev"
+
+# Xelytics-Core
+
+**Pure analytics engine for statistical analysis and insight generation.**
+
+## Installation
+
+```bash
+pip install -e .
+```
+
+## Quick Start
+
+```python
+from xelytics import analyze, AnalysisConfig
+import pandas as pd
+
+# Load your data
+df = pd.read_csv("data.csv")
+
+# Run automated analysis
+result = analyze(df, mode="automated")
+
+# Access results
+print(f"Analyzed {result.metadata.row_count} rows")
+print(f"Found {len(result.statistics)} statistical tests")
+print(f"Generated {len(result.visualizations)} visualizations")
+print(f"Produced {len(result.insights)} insights")
+
+# Export to JSON
+json_output = result.to_json()
+```
+
+## API Contract
+
+```python
+from xelytics import analyze, AnalysisConfig, AnalysisResult
+
+result = analyze(
+    data=df,
+    mode="automated",  # or "semi-automated"
+    config=AnalysisConfig(
+        significance_level=0.05,
+        enable_llm_insights=True,
+        max_visualizations=10,
+    )
+)
+```
+
+## Output Schema
+
+```python
+AnalysisResult(
+    summary=DatasetSummary(...),
+    statistics=[StatisticalTestResult(...), ...],
+    visualizations=[VisualizationSpec(...), ...],
+    insights=[Insight(...), ...],
+    metadata=RunMetadata(...),
+)
+```
+
+## Design Principles
+
+1. **Pure analytics engine** - No HTTP, no database, no auth
+2. **Deterministic** - Same input = same output
+3. **LLM is optional** - Rule-based insights work without LLM
+4. **Type-safe** - All inputs/outputs are typed dataclasses
+
+## License
+
+MIT

xelytics_core-0.1.0/README.md

@@ -0,0 +1,70 @@
+# Xelytics-Core
+
+**Pure analytics engine for statistical analysis and insight generation.**
+
+## Installation
+
+```bash
+pip install -e .
+```
+
+## Quick Start
+
+```python
+from xelytics import analyze, AnalysisConfig
+import pandas as pd
+
+# Load your data
+df = pd.read_csv("data.csv")
+
+# Run automated analysis
+result = analyze(df, mode="automated")
+
+# Access results
+print(f"Analyzed {result.metadata.row_count} rows")
+print(f"Found {len(result.statistics)} statistical tests")
+print(f"Generated {len(result.visualizations)} visualizations")
+print(f"Produced {len(result.insights)} insights")
+
+# Export to JSON
+json_output = result.to_json()
+```
+
+## API Contract
+
+```python
+from xelytics import analyze, AnalysisConfig, AnalysisResult
+
+result = analyze(
+    data=df,
+    mode="automated",  # or "semi-automated"
+    config=AnalysisConfig(
+        significance_level=0.05,
+        enable_llm_insights=True,
+        max_visualizations=10,
+    )
+)
+```
+
+## Output Schema
+
+```python
+AnalysisResult(
+    summary=DatasetSummary(...),
+    statistics=[StatisticalTestResult(...), ...],
+    visualizations=[VisualizationSpec(...), ...],
+    insights=[Insight(...), ...],
+    metadata=RunMetadata(...),
+)
+```
+
+## Design Principles
+
+1. **Pure analytics engine** - No HTTP, no database, no auth
+2. **Deterministic** - Same input = same output
+3. **LLM is optional** - Rule-based insights work without LLM
+4. **Type-safe** - All inputs/outputs are typed dataclasses
+
+## License
+
+MIT

xelytics_core-0.1.0/pyproject.toml

@@ -0,0 +1,64 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "xelytics-core"
+version = "0.1.0"
+description = "Pure analytics engine for statistical analysis and insight generation"
+readme = "README.md"
+license = {text = "MIT"}
+requires-python = ">=3.9"
+authors = [
+    {name = "Xelytics Team"}
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+
+dependencies = [
+    "pandas>=2.1.0",
+    "numpy>=1.24.0",
+    "scipy>=1.11.0",
+    "scikit-learn>=1.3.0",
+    "statsmodels>=0.14.0",
+    "pingouin>=0.5.3",
+    "plotly>=5.17.0",
+]
+
+[project.optional-dependencies]
+llm = [
+    "openai>=1.6.0",
+    "groq>=0.4.0",
+    "httpx>=0.25.0",
+]
+dev = [
+    "pytest>=7.4.0",
+    "pytest-asyncio>=0.21.0",
+    "pytest-cov>=4.1.0",
+    "black>=23.11.0",
+    "mypy>=1.7.0",
+]
+
+[project.scripts]
+xelytics = "xelytics.cli.main:main"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["xelytics*"]
+
+[tool.black]
+line-length = 100
+target-version = ['py39', 'py310', 'py311', 'py312']
+
+[tool.mypy]
+python_version = "3.9"
+warn_return_any = true
+warn_unused_configs = true

xelytics_core-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

xelytics_core-0.1.0/tests/test_core.py

@@ -0,0 +1,167 @@
+"""Unit tests for core modules.
+
+Tests ingestion, profiling, and feature detection.
+"""
+
+import pytest
+import pandas as pd
+import numpy as np
+
+from xelytics.core.ingestion import DataIngestion, IngestionResult
+from xelytics.core.profiler import DataProfiler, ProfileResult
+from xelytics.core.features import FeatureDetector, FeatureDetectionResult
+
+
+class TestDataIngestion:
+    """Tests for DataIngestion module."""
+    
+    def test_ingest_valid_dataframe(self, sample_mixed_df):
+        """Test ingestion of valid DataFrame."""
+        ingestion = DataIngestion()
+        result = ingestion.ingest(sample_mixed_df)
+        
+        assert isinstance(result, IngestionResult)
+        assert result.row_count == 100
+        assert result.column_count == 5
+        assert len(result.column_dtypes) == 5
+    
+    def test_ingest_empty_dataframe_raises(self):
+        """Test that empty DataFrame raises ValueError."""
+        ingestion = DataIngestion()
+        with pytest.raises(ValueError, match="cannot be empty"):
+            ingestion.ingest(pd.DataFrame())
+    
+    def test_ingest_invalid_type_raises(self):
+        """Test that non-DataFrame raises error."""
+        ingestion = DataIngestion()
+        with pytest.raises(ValueError, match="must be a pandas DataFrame"):
+            ingestion.ingest([1, 2, 3])  # type: ignore
+    
+    def test_type_normalization(self):
+        """Test automatic type normalization."""
+        ingestion = DataIngestion()
+        df = pd.DataFrame({
+            'numeric_str': ['1', '2', '3', '4', '5'],
+            'date_str': ['2024-01-01', '2024-01-02', '2024-01-03', '2024-01-04', '2024-01-05'],
+        })
+        result = ingestion.ingest(df)
+        
+        # Should normalize numeric strings to proper types
+        assert result.row_count == 5
+
+
+class TestDataProfiler:
+    """Tests for DataProfiler module."""
+    
+    def test_profile_numeric_columns(self, sample_numeric_df):
+        """Test profiling numeric columns."""
+        profiler = DataProfiler()
+        result = profiler.profile(sample_numeric_df)
+        
+        assert isinstance(result, ProfileResult)
+        assert len(result.column_profiles) == 4
+        
+        # Check numeric statistics present
+        sales_profile = next(p for p in result.column_profiles if p.column_name == 'sales')
+        assert sales_profile.mean is not None
+        assert sales_profile.std is not None
+        assert sales_profile.min is not None
+        assert sales_profile.max is not None
+    
+    def test_profile_categorical_columns(self, sample_categorical_df):
+        """Test profiling categorical columns."""
+        profiler = DataProfiler()
+        result = profiler.profile(sample_categorical_df)
+        
+        assert len(result.column_profiles) == 4
+        
+        # Check categorical statistics present
+        region_profile = next(p for p in result.column_profiles if p.column_name == 'region')
+        assert region_profile.unique_count == 4
+        assert region_profile.data_type == 'categorical'
+    
+    def test_profile_missing_values(self, sample_with_missing_df):
+        """Test profiling DataFrame with missing values."""
+        profiler = DataProfiler()
+        result = profiler.profile(sample_with_missing_df)
+        
+        assert result.total_missing_cells > 0
+        
+        # Check missing value detection
+        value1_profile = next(p for p in result.column_profiles if p.column_name == 'value1')
+        assert value1_profile.missing_count == 10
+
+
+class TestFeatureDetector:
+    """Tests for FeatureDetector module."""
+    
+    def test_detect_numeric_columns(self, sample_numeric_df):
+        """Test detection of numeric columns."""
+        detector = FeatureDetector()
+        result = detector.detect(sample_numeric_df)
+        
+        assert isinstance(result, FeatureDetectionResult)
+        assert len(result.numeric_columns) == 4
+        assert 'sales' in result.numeric_columns
+    
+    def test_detect_categorical_columns(self, sample_categorical_df):
+        """Test detection of categorical columns."""
+        detector = FeatureDetector()
+        result = detector.detect(sample_categorical_df)
+        
+        assert len(result.categorical_columns) == 4
+        assert 'region' in result.categorical_columns
+    
+    def test_detect_datetime_columns(self, sample_mixed_df):
+        """Test detection of datetime columns."""
+        detector = FeatureDetector()
+        result = detector.detect(sample_mixed_df)
+        
+        assert 'date' in result.datetime_columns
+    
+    def test_detect_groupable_columns(self, sample_mixed_df):
+        """Test detection of groupable columns."""
+        detector = FeatureDetector()
+        result = detector.detect(sample_mixed_df)
+        
+        # Should detect region and category as groupable
+        assert len(result.groupable_columns) >= 2
+    
+    def test_no_name_heuristics(self):
+        """Test that feature detection uses data only, not column names.
+        
+        Per plan constraint: Feature detection must rely only on data.
+        """
+        detector = FeatureDetector()
+        
+        # Create DataFrame with misleading column names
+        df = pd.DataFrame({
+            'id_column': ['A', 'B', 'C'] * 10,  # Named like ID but low cardinality
+            'date_field': [1.5, 2.5, 3.5] * 10,  # Named like date but numeric
+            'target_var': np.random.choice(['X', 'Y'], 30),  # Named like target but categorical
+        })
+        
+        result = detector.detect(df)
+        
+        # 'id_column' should NOT be classified as identifier (low cardinality)
+        assert 'id_column' not in result.identifier_columns
+        assert 'id_column' in result.categorical_columns
+        
+        # 'date_field' should NOT be classified as datetime (it's numeric)
+        assert 'date_field' not in result.datetime_columns
+        assert 'date_field' in result.numeric_columns
+    
+    def test_deterministic_detection(self, sample_mixed_df, assert_deterministic):
+        """Test that feature detection is deterministic."""
+        detector = FeatureDetector()
+        
+        def detect_wrapper():
+            return detector.detect(sample_mixed_df)
+        
+        # Run 5 times and assert same output
+        results = []
+        for _ in range(5):
+            result = detector.detect(sample_mixed_df)
+            results.append(tuple(sorted(result.column_roles.items())))
+        
+        assert len(set(results)) == 1, "Feature detection is not deterministic"

xelytics_core-0.1.0/tests/test_golden.py

@@ -0,0 +1,178 @@
+"""Golden output and determinism tests.
+
+Tests to ensure:
+1. Same input → same output (determinism)
+2. Output matches expected golden values (regression)
+"""
+
+import pytest
+import pandas as pd
+import numpy as np
+import json
+import hashlib
+
+from xelytics import analyze, AnalysisConfig
+
+
+class TestDeterminism:
+    """Tests that same input produces same output."""
+    
+    def test_analyze_deterministic(self, golden_dataset):
+        """Test that analyze() produces identical output across runs."""
+        config = AnalysisConfig(
+            mode="automated",
+            enable_llm_insights=False,  # Disable LLM for determinism
+        )
+        
+        hashes = []
+        for _ in range(3):
+            result = analyze(golden_dataset, mode="automated", config=config)
+            # Hash the core output
+            output = {
+                "row_count": result.summary.row_count,
+                "column_count": result.summary.column_count,
+                "tests_executed": result.metadata.tests_executed,
+                "numeric_columns": sorted(result.summary.numeric_columns),
+                "categorical_columns": sorted(result.summary.categorical_columns),
+            }
+            hash_val = hashlib.md5(json.dumps(output, sort_keys=True).encode()).hexdigest()
+            hashes.append(hash_val)
+        
+        assert len(set(hashes)) == 1, f"Non-deterministic output: {hashes}"
+    
+    def test_statistical_results_deterministic(self, golden_dataset):
+        """Test that statistical test results are deterministic."""
+        config = AnalysisConfig(mode="automated", enable_llm_insights=False)
+        
+        p_values_runs = []
+        for _ in range(3):
+            result = analyze(golden_dataset, mode="automated", config=config)
+            p_values = [r.p_value for r in result.statistics]
+            p_values_runs.append(tuple(p_values))
+        
+        assert len(set(p_values_runs)) == 1, "P-values differ across runs"
+    
+    def test_insight_generation_deterministic(self, golden_dataset):
+        """Test that insight generation is deterministic."""
+        config = AnalysisConfig(mode="automated", enable_llm_insights=False)
+        
+        insight_titles_runs = []
+        for _ in range(3):
+            result = analyze(golden_dataset, mode="automated", config=config)
+            titles = sorted([i.title for i in result.insights])
+            insight_titles_runs.append(tuple(titles))
+        
+        assert len(set(insight_titles_runs)) == 1, "Insights differ across runs"
+
+
+class TestGoldenOutput:
+    """Golden output regression tests.
+    
+    These tests verify that key outputs match expected values.
+    If the expected values change, investigate why before updating.
+    """
+    
+    def test_golden_dataset_row_count(self, golden_dataset):
+        """Test expected row count."""
+        result = analyze(golden_dataset, mode="automated")
+        assert result.summary.row_count == 10
+    
+    def test_golden_dataset_column_count(self, golden_dataset):
+        """Test expected column count."""
+        result = analyze(golden_dataset, mode="automated")
+        assert result.summary.column_count == 4
+    
+    def test_golden_dataset_numeric_columns(self, golden_dataset):
+        """Test expected numeric column detection."""
+        result = analyze(golden_dataset, mode="automated")
+        
+        expected_numeric = ['age', 'income']
+        assert sorted(result.summary.numeric_columns) == sorted(expected_numeric)
+    
+    def test_golden_dataset_categorical_columns(self, golden_dataset):
+        """Test expected categorical column detection."""
+        result = analyze(golden_dataset, mode="automated")
+        
+        expected_categorical = ['education', 'region']
+        assert sorted(result.summary.categorical_columns) == sorted(expected_categorical)
+    
+    def test_golden_dataset_no_missing(self, golden_dataset):
+        """Test that golden dataset has no missing values."""
+        result = analyze(golden_dataset, mode="automated")
+        assert result.summary.total_missing_cells == 0
+    
+    def test_json_serialization_roundtrip(self, golden_dataset):
+        """Test that JSON serialization is lossless."""
+        from xelytics.schemas.outputs import AnalysisResult
+        
+        result = analyze(golden_dataset, mode="automated")
+        
+        # Serialize to JSON
+        json_str = result.to_json()
+        
+        # Deserialize
+        restored = AnalysisResult.from_json(json_str)
+        
+        # Compare key fields
+        assert restored.summary.row_count == result.summary.row_count
+        assert restored.summary.column_count == result.summary.column_count
+        assert len(restored.statistics) == len(result.statistics)
+        assert len(restored.insights) == len(result.insights)
+
+
+class TestBackwardCompatibility:
+    """Backward compatibility tests.
+    
+    Ensures schema changes don't break existing integrations.
+    """
+    
+    def test_analysis_result_has_required_fields(self, sample_mixed_df):
+        """Test that AnalysisResult has all required fields."""
+        result = analyze(sample_mixed_df, mode="automated")
+        
+        # Required top-level fields (per API_CONTRACT.md)
+        assert hasattr(result, 'summary')
+        assert hasattr(result, 'statistics')
+        assert hasattr(result, 'visualizations')
+        assert hasattr(result, 'insights')
+        assert hasattr(result, 'metadata')
+    
+    def test_dataset_summary_fields(self, sample_mixed_df):
+        """Test that DatasetSummary has expected fields."""
+        result = analyze(sample_mixed_df, mode="automated")
+        summary = result.summary
+        
+        assert hasattr(summary, 'row_count')
+        assert hasattr(summary, 'column_count')
+        assert hasattr(summary, 'numeric_columns')
+        assert hasattr(summary, 'categorical_columns')
+        assert hasattr(summary, 'column_profiles')
+    
+    def test_statistical_result_fields(self, sample_mixed_df):
+        """Test that StatisticalTestResult has expected fields."""
+        result = analyze(sample_mixed_df, mode="automated")
+        
+        if result.statistics:
+            stat = result.statistics[0]
+            assert hasattr(stat, 'test_name')
+            assert hasattr(stat, 'test_type')
+            assert hasattr(stat, 'statistic')
+            assert hasattr(stat, 'p_value')
+            assert hasattr(stat, 'significant')
+            assert hasattr(stat, 'interpretation')
+    
+    def test_json_output_structure(self, sample_mixed_df):
+        """Test that JSON output has expected structure."""
+        result = analyze(sample_mixed_df, mode="automated")
+        output = result.to_dict()
+        
+        # Top-level keys
+        assert 'summary' in output
+        assert 'statistics' in output
+        assert 'visualizations' in output
+        assert 'insights' in output
+        assert 'metadata' in output
+        
+        # Summary keys
+        assert 'row_count' in output['summary']
+        assert 'column_count' in output['summary']