m2datakit 0.1.97__py3-none-any.whl

m2datakit/__init__.py

@@ -0,0 +1,16 @@
+# m2datakit/__init__.py
+
+# Convenience imports available at package import time
+import glob  # noqa: F401
+import json  # noqa: F401
+import os  # noqa: F401
+import re  # noqa: F401
+
+from dotenv import load_dotenv  # noqa: F401
+
+from . import core, tasks
+
+__all__ = [
+    "core",
+    "tasks",
+]

m2datakit/core/__init__.py

@@ -0,0 +1,13 @@
+from . import pipeline  # Expose pipeline cleanly
+from . import log
+from .config import settings
+from .helpers import summarize_common_metadata
+from .utils import get_package_version
+
+__all__ = [
+    "log",
+    "pipeline",
+    "settings",
+    "get_package_version",
+    "summarize_common_metadata",
+]

m2datakit/core/codebook.py

@@ -0,0 +1,108 @@
+import pandas as pd
+from docx import Document
+from fpdf import FPDF
+
+
+def make_codebook(df, output_format='markdown', filename='codebook', custom_descriptions=None):
+    """
+    Generate a skimr-style codebook from a pandas DataFrame and export it.
+    
+    Parameters:
+    - df: pandas DataFrame
+    - output_format: 'markdown', 'docx', or 'pdf'
+    - filename: base name for output file
+    - custom_descriptions: dict of {column_name: description} overrides
+    """
+
+    def describe_variable(col, series):
+        if pd.api.types.is_numeric_dtype(series):
+            return {
+                'n': len(series),
+                'missing': series.isna().sum(),
+                'mean': round(series.mean(), 2),
+                'std': round(series.std(), 2),
+                'min': series.min(),
+                'max': series.max()
+            }
+        elif pd.api.types.is_categorical_dtype(series) or series.dtype == object:
+            top = series.mode().iloc[0] if not series.mode().empty else ''
+            return {
+                'n': len(series),
+                'missing': series.isna().sum(),
+                'unique': series.nunique(),
+                'top': top,
+                'freq': series.value_counts().iloc[0] if not series.value_counts().empty else ''
+            }
+        else:
+            return {
+                'n': len(series),
+                'missing': series.isna().sum(),
+                'example': str(series.dropna().iloc[0]) if not series.dropna().empty else ''
+            }
+
+    rows = []
+    for col in df.columns:
+        desc = custom_descriptions.get(col) if custom_descriptions else None
+        if not desc:
+            desc = col.replace('_', ' ').capitalize()
+        summary = describe_variable(col, df[col])
+        summary['variable'] = col
+        summary['type'] = str(df[col].dtype)
+        summary['description'] = desc
+        rows.append(summary)
+
+    codebook_df = pd.DataFrame(rows)
+
+    # Reorder columns
+    cols_order = ['variable', 'type', 'description'] + [col for col in codebook_df.columns if col not in ['variable', 'type', 'description']]
+    codebook_df = codebook_df[cols_order]
+
+    # MARKDOWN
+    if output_format == 'markdown':
+        with open(f'{filename}.md', 'w') as f:
+            f.write('# Codebook\n\n')
+            for _, row in codebook_df.iterrows():
+                f.write(f"### {row['variable']} ({row['type']})\n")
+                f.write(f"**Description**: {row['description']}\n\n")
+                for key, val in row.items():
+                    if key not in ['variable', 'type', 'description']:
+                        f.write(f"- {key}: {val}\n")
+                f.write('\n')
+
+    # DOCX
+    elif output_format == 'docx':
+        doc = Document()
+        doc.add_heading('Codebook', level=1)
+        for _, row in codebook_df.iterrows():
+            doc.add_heading(f"{row['variable']} ({row['type']})", level=2)
+            doc.add_paragraph(f"Description: {row['description']}")
+            for key, val in row.items():
+                if key not in ['variable', 'type', 'description']:
+                    doc.add_paragraph(f"{key}: {val}")
+        doc.save(f"{filename}.docx")
+
+    # PDF
+    elif output_format == 'pdf':
+        pdf = FPDF()
+        pdf.add_page()
+        pdf.set_font("Arial", size=10)
+        pdf.set_auto_page_break(auto=True, margin=15)
+        pdf.multi_cell(0, 10, txt="Codebook", align='L')
+        for _, row in codebook_df.iterrows():
+            pdf.set_font("Arial", style='B', size=10)
+            pdf.cell(0, 10, f"{row['variable']} ({row['type']})", ln=True)
+            pdf.set_font("Arial", size=10)
+            pdf.multi_cell(0, 10, f"Description: {row['description']}")
+            for key, val in row.items():
+                if key not in ['variable', 'type', 'description']:
+                    line = f"{key}: {val}"
+                    pdf.multi_cell(0, 10, line)
+            pdf.ln(5)
+        pdf.output(f"{filename}.pdf")
+
+    else:
+        raise ValueError("Format must be one of: 'markdown', 'docx', 'pdf'")
+
+    print(f"Codebook saved to {filename}.{output_format}")
+
+

m2datakit/core/config.py

@@ -0,0 +1,86 @@
+import os
+import re
+from importlib.metadata import version
+from pathlib import Path
+from typing import List
+
+from pydantic import BaseModel, Field
+
+
+class Settings(BaseModel):
+    PACKAGE_VERSION: str = Field(default_factory=lambda: version("m2datakit"))
+
+    # ABSTRACT ALL IDS BY PROVIDER
+    DEDUP_IDS_METRICWIRE: List[str] = [
+        "userId",
+        "submissionSessionId",
+        "activityId",
+    ]
+    DEDUP_IDS_UAS: List[str] = [
+        "userId",
+        "submissionSessionId",
+        "activityId",
+    ]
+
+    DEDUP_IDS_MONGODB: List[str] = []
+    DEDUP_IDS_QUALTRICS: List[str] = [
+        "index",
+        "ResponseId",
+        "M2C2_ASSESSMENT_ORDER",
+        "M2C2_AUTO_ADVANCE",
+        "M2C2_LANGUAGE",
+    ]
+
+    # def unnest_trial_level_data(df: pd.DataFrame, drop_duplicates=True, column_order: List[str] = None) -> pd.DataFrame:
+    #     column_order = column_order or ["participant_id", "session_id", "group", "wave", "activity_id", "study_id", "document_uuid"]
+    #     trial_df = trial_df.drop_duplicates(subset=["activity_uuid", "session_uuid", "trial_begin_iso8601_timestamp"])
+
+    STANDARD_GROUPING_FOR_AGGREGATION_LEGACY: List[str] = [
+        "study_uid",
+        "user_uid",
+        "uuid",
+        "activity_name",
+    ]
+
+    STANDARD_GROUPING_FOR_AGGREGATION: List[str] = [
+        "study_uid",
+        "user_uid",
+        "session_uuid",
+        "activity_name",
+    ]
+
+    STANDARD_GROUPING_FOR_AGGREGATION_QUALTRICS: List[str] = ["ResponseId"]
+
+    STANDARD_GROUPING_FOR_AGGREGATION_METRICWIRE: List[str] = [
+        "userId",
+        "submissionSessionId",
+        "activityId",
+    ]
+    STANDARD_GROUPING_FOR_AGGREGATION_UAS: List[str] = [
+        "userId",
+        "submissionSessionId",
+        "activityId",
+    ]
+
+    @property
+    def QUALTRICS_TRIAL_DATA_REGEX(self):
+        return re.compile(r"(M2C2_ASSESSMENT_\d+)_TRIAL_DATA_(\d+)")
+
+    @property
+    def DEFAULT_FUNC_MAP_SCORING(self):
+        from .map import DEFAULT_FUNC_MAP_SCORING
+
+        return DEFAULT_FUNC_MAP_SCORING
+
+    # DEFAULTS FOR UI
+    DEFAULT_PLOT_COLOR: str = "steelblue"
+    DEFAULT_PLOT_DPI: int = 150
+
+
+settings = Settings()
+
+# === Logging Configuration (opt-in) ===
+LOGGING_ENABLED = os.getenv("M2DATAKIT_ENABLE_LOGGING", "0").lower() in {"1", "true", "yes"}
+LOGFIRE_ENABLED = os.getenv("M2DATAKIT_ENABLE_LOGFIRE", "0").lower() in {"1", "true", "yes"}
+LOG_DIR = Path(os.getenv("M2DATAKIT_LOG_DIR", str(Path.cwd() / "logs")))
+LOG_FILE = LOG_DIR / "events.jsonl"

m2datakit/core/export.py

@@ -0,0 +1,68 @@
+def export_dataframe(df, file_name, format=".csv", table_name="my_table", **kwargs):
+    """
+    Exports a Pandas DataFrame to a specified file format, including raw SQL `INSERT` statements.
+
+    Parameters:
+        df (pd.DataFrame): The DataFrame to export.
+        file_name (str): The file name (without extension) to export the DataFrame to.
+        format (str): The file format (e.g., '.csv', '.json', '.xlsx', '.sql', '.parquet', etc.).
+        table_name (str): Table name for SQL `INSERT` statements (used only when format='.sql').
+        **kwargs: Additional keyword arguments for Pandas export functions.
+
+    Returns:
+        str: The full file name of the exported file.
+    """
+    try:
+        file_name_with_extension = f"{file_name}{format}"
+
+        # Export logic for supported formats
+        if format == ".csv":
+            df.to_csv(file_name_with_extension, index=False, **kwargs)
+        elif format == ".json":
+            df.to_json(file_name_with_extension, orient="records", **kwargs)
+        elif format == ".xlsx":
+            df.to_excel(file_name_with_extension, index=False, **kwargs)
+        elif format == ".parquet":
+            df.to_parquet(file_name_with_extension, index=False, **kwargs)
+        elif format == ".html":
+            df.to_html(file_name_with_extension, index=False, **kwargs)
+        elif format == ".pkl":
+            df.to_pickle(file_name_with_extension, **kwargs)
+        elif format == ".txt":
+            df.to_csv(file_name_with_extension, index=False, sep="\t", **kwargs)
+        else:
+            raise ValueError(f"Unsupported file format: {format}")
+
+        return file_name_with_extension
+
+    except Exception as e:
+        raise RuntimeError(f"Export failed: {e}") from e
+
+import json
+from datetime import date
+
+
+def export_jsonld_metadata(df, filename="dataset_metadata.json"):
+    today = date.today().isoformat()
+    metadata = {
+        "@context": "https://schema.org/",
+        "@type": "Dataset",
+        "name": "M2C2Kit Dataset",
+        "description": "This dataset contains M2C2kit data processed via the DataKit Python package.",
+        "creator": {"@type": "Person", "name": "Nelson Roque", "affiliation": "M2C2"},
+        "dateCreated": today,
+        "variableMeasured": [],
+    }
+
+    for col in df.columns:
+        metadata["variableMeasured"].append(
+            {
+                "@type": "PropertyValue",
+                "name": col,
+                "description": f"Auto-description for {col}",
+                "value": str(df[col].dtype),
+            }
+        )
+
+    with open(filename, "w") as f:
+        json.dump(metadata, f, indent=2)