CBACABLab 0.0.1__py3-none-any.whl

CBACABLab/LikertPrep.py

@@ -0,0 +1,102 @@
+import pandas as pd
+from .LoadRawData import LoadSettings
+from .LoadSQLData import LoadQuestionsWide, LoadSurveyWithSensorDataWide
+
+
+def LikertToK(responses, scale, settings=None):
+    """
+    Convert Likert responses to 0-based k values using a scale defined in settings.json.
+
+    Args:
+        responses (pandas.Series): Raw responses (text or numeric).
+        scale (str or dict): Name of a scale under "LikertScales" in settings.json, or a mapping.
+        settings (dict): Optional pre-loaded settings.
+
+    Returns:
+        tuple: (k as nullable Int64 Series, bound)
+
+    Raises:
+        KeyError: If the scale name is unknown.
+        ValueError: If a non-missing response is not in the scale.
+    """
+    if isinstance(scale, str):
+        scales = (settings or LoadSettings()).get("LikertScales", {})
+        if scale not in scales:
+            raise KeyError(f"Unknown Likert scale '{scale}'. Available: {sorted(scales)}")
+        scale = scales[scale]
+    mapping = {str(key).strip().lower(): value for key, value in scale.items()}
+    normalized = responses.astype("string").str.strip().str.lower()
+    # Numeric responses stored as floats (e.g. "4.0") should match "4"
+    normalized = normalized.str.replace(r"^(-?\d+)\.0+$", r"\1", regex=True)
+    k = normalized.map(mapping)
+    unknown = responses[normalized.notna() & k.isna()].unique()
+    if len(unknown):
+        raise ValueError(f"Responses not in scale: {list(unknown)}")    
+    k = k.dropna().astype('int64')
+    return k, max(mapping.values()) # bound
+
+
+def AddLikertColumns(data, scales, question_column="question", response_column="response", settings=None):
+    """
+    Add k and bound columns to panel data, using a Likert scale chosen per question.
+
+    Args:
+        data (pandas.DataFrame): Data with a question column and a response column, such as the output of CreatePanelData.
+        scales (str or dict): A scale name applied to every question, or a dict of
+            {question: scale name or {label: k} mapping} giving each question its own scale.
+        question_column (str, optional): Column holding the question.
+        response_column (str, optional): Column holding the Likert response.
+        settings (dict, optional): Pre-loaded settings.
+
+    Returns:
+        pandas.DataFrame: A copy of data with nullable Int64 columns k and bound.
+
+    Raises:
+        KeyError: If a question has no scale, or a scale name is unknown.
+        ValueError: If a response is not in its question's scale.
+    """
+    settings = settings or LoadSettings()
+    result = data.copy()
+    questions = result[question_column].dropna().unique()
+    if isinstance(scales, str):
+        scales = {question: scales for question in questions}
+    missing = [question for question in questions if question not in scales]
+    if missing:
+        raise KeyError(f"No Likert scale given for questions: {missing}")
+
+    for question in questions:
+        mask = result[question_column] == question
+        try:
+            k, bound = LikertToK(result.loc[mask, response_column], scales[question], settings)
+        except ValueError as error:
+            raise ValueError(f"Question '{question}': {error}") from error
+        result.loc[mask, "k"] = k
+        result.loc[mask, "bound"] = bound
+    result = result.astype({'k': 'int64', 'bound': 'int64'})
+    return result
+
+
+
+def CreatePanelData(question_level_columns, respondent_level_columns, tagFilter=None, instrumentProperties=None, databaseName="data.db"):
+    """
+    Create a panel dataset with exactly one row per respondent and question-level question. 
+
+    Question-level data comes from LoadSurveyWithSensorDataWide: question, response, and one column per
+    sensor measure and tag (pupil_<tag>, duration_<tag>, <property>_<tag>). When no tags are specified, a tag is not included in the column names.
+    Respondent-level questions are added as one response column each.
+
+    Args:
+        question_level_columns (list): List of question-level columns to include.
+        respondent_level_columns (list): List of respondent-level columns to include.
+        tagFilter (list, tuple, or set, optional): Only include these tags from the sensor data.
+        instrumentProperties (list, tuple, or set, optional): Instrument properties to include from the sensor data.
+        databaseName (str, optional): Name of the SQLite database file.
+
+    Returns:
+        pandas.DataFrame: Merged panel dataset.
+    """
+
+    respondent_data = LoadQuestionsWide(respondent_level_columns, databaseName=databaseName)
+    question_data = LoadSurveyWithSensorDataWide(question_level_columns, databaseName=databaseName, tagFilter=tagFilter, instrumentProperties=instrumentProperties)
+    merged_data = pd.merge(question_data, respondent_data, on="respondent_id", how="left")
+    return merged_data

CBACABLab/LoadRawData.py

@@ -0,0 +1,251 @@
+import pandas as pd
+import warnings
+warnings.filterwarnings("ignore", category=UserWarning, module="openpyxl")
+import sqlite3
+import os
+import json
+
+def LoadSettings(path=None):
+    """Load module settings from settings.json (or the given JSON file)."""
+    if path is None:
+        path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "settings.json")
+    with open(path, encoding="utf-8") as f:
+        return json.load(f)
+
+InstrumentColumns = LoadSettings()["InstrumentColumns"]
+
+def insertRow(con, row):
+    sql = "INSERT OR REPLACE INTO FullSurvey(respondent_id, question, response) VALUES (?, ?, ?)"
+    cur = con.cursor()
+    cur.execute(sql, row)
+    con.commit()
+    cur.close()
+
+
+def AutoDetermineQuestionIdtoQuestionNameMapping(surveyName="Survey.xlsx", surveyCSVName="Survey.csv"):
+    """
+    Automatically determine the mapping from question IDs to question names from a Qualtrics survey Excel file and CSV file.
+
+    Args:
+        surveyName (str): The path to the Qualtrics Excel file.
+        surveyCSVName (str): The path to the Qualtrics CSV file.
+
+    Returns:
+        dict: A dictionary mapping question IDs to question names.
+    """
+    qualtricsSurvey = pd.read_excel(surveyName, skiprows=[1])
+    qualtricsSurvey.rename(columns=lambda column: column.strip() if isinstance(column, str) else column, inplace=True)
+    qualtricsSurveyCSV = pd.read_csv(surveyCSVName)
+    qualtricsSurveyCSV.rename(columns=lambda column: column.strip() if isinstance(column, str) else column, inplace=True)
+    questionIdtoQuestionName = {}
+    for column_index, question_id in enumerate(qualtricsSurveyCSV.columns):
+        if isinstance(question_id, str) and question_id.startswith("QID"):
+            if column_index < len(qualtricsSurvey.columns):
+                questionIdtoQuestionName[question_id] = qualtricsSurvey.columns[column_index]
+    return questionIdtoQuestionName
+
+def LoadSurveyData(databaseName="data.db", surveyName="Survey.xlsx"):
+    """
+    Load Qualtrics survey responses from an Excel file and insert them into a SQLite database.
+
+    The FullSurvey table is dropped and recreated. The second row of the export (the question text row)
+    is skipped, and rows without a numeric iMotionsRespondent value are ignored, as are respondents 99 and 999.
+    Every column after the UserLanguage column is treated as a question. Blank responses are not stored.
+
+    Args:
+        databaseName (str; optional): The name of the SQLite database file.
+        surveyName (str; optional): The path to the Qualtrics Excel file.
+
+    Returns:
+        None
+
+    Raises:
+        ValueError: If the UserLanguage column cannot be found.
+    """
+    con = sqlite3.connect(databaseName)
+    cur = con.cursor()
+    cur.execute("DROP TABLE IF EXISTS FullSurvey")
+    cur.execute("CREATE TABLE FullSurvey (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, respondent_id INTEGER NOT NULL, question TEXT NOT NULL, response TEXT NOT NULL, UNIQUE(respondent_id, question) ON CONFLICT REPLACE)")
+    con.commit()
+    cur.close()
+    qualtricsSurvey = pd.read_excel(surveyName, skiprows=[1])
+    qualtricsSurvey.rename(columns=lambda column: column.strip() if isinstance(column, str) else column, inplace=True)
+
+    qualtricsSurvey["iMotionsRespondent"] = pd.to_numeric(
+        qualtricsSurvey["iMotionsRespondent"], errors="coerce"
+    )
+    qualtricsSurvey = qualtricsSurvey.loc[qualtricsSurvey["iMotionsRespondent"].notna()].copy()
+
+    marker_index = next(
+        (
+            index
+            for index, column in enumerate(qualtricsSurvey.columns)
+            if "".join(str(column).casefold().split()) == "userlanguage"
+        ),
+        None,
+    )
+    if marker_index is None:
+        raise ValueError("Could not find 'User Language' or 'UserLanguage' column")
+    
+    for _, row in qualtricsSurvey.iterrows():
+        respondent_id = int(row["iMotionsRespondent"])
+        if respondent_id in (99, 999):
+            continue
+        for question in qualtricsSurvey.columns[marker_index + 1:]:
+            response = row[question]
+            if pd.isna(response):
+                continue
+            response = str(response)
+            if response.strip():
+                insertRow(con, (respondent_id, str(question), response))
+
+
+def insertSensorRow(con, row):
+    cur = con.cursor()
+    cur.execute("INSERT INTO SensorData (respondent_id, question, pupil, duration, tag) VALUES (?, ?, ?, ?, ?)", row)
+    con.commit()
+    cur.close()
+
+def insertFaceRow(con, row):
+    cur = con.cursor()
+    cur.execute("INSERT INTO InstrumentData (respondent_id, question, property, value, tag) VALUES (?, ?, ?, ?, ?)", row)
+    con.commit()
+    cur.close()
+
+def ExtractData(con, df, respondentid, boundedboxes=None, tag="default", questionIdtoQuestionName=None):
+    df = df.rename(columns=lambda column: column.strip() if isinstance(column, str) else column)
+
+    if boundedboxes is not None:
+        if {'Interpolated Gaze X', 'Interpolated Gaze Y'}.issubset(df.columns):
+            gaze_x, gaze_y = 'Interpolated Gaze X', 'Interpolated Gaze Y'
+        elif {'Gaze X', 'Gaze Y'}.issubset(df.columns):
+            gaze_x, gaze_y = 'Gaze X', 'Gaze Y'
+        else:
+            raise ValueError("Missing gaze columns: expected Interpolated Gaze X and Y or Gaze X and Y")
+
+        master_mask = pd.Series(False, index=df.index)
+        for x_min, x_max, y_min, y_max in boundedboxes:
+            bounds = (x_min, x_max, y_min, y_max)
+            if any(not isinstance(v, (int, float)) or isinstance(v, bool) for v in bounds):
+                raise ValueError(f"Bounding box values must be numbers: {bounds}")
+            if x_min >= x_max or y_min >= y_max:
+                raise ValueError(f"Invalid bounding box: ({x_min}, {x_max}, {y_min}, {y_max})")
+            current_mask = (
+                (df[gaze_x] >= x_min) & (df[gaze_x] <= x_max) & 
+                (df[gaze_y] >= y_min) & (df[gaze_y] <= y_max)
+            )
+            master_mask |= current_mask
+        filtered_df = df[master_mask].copy()
+    else:
+        filtered_df = df.copy()
+    
+
+    listOfQuestions = (
+        df.loc[
+            df['MarkerName'].notna()
+            & (df['MarkerName'].str.strip() != '')
+            & (~df['MarkerName'].str.endswith('_end', na=False)),
+            ['Timestamp', 'MarkerName']
+        ]
+        .drop_duplicates(subset=['MarkerName'], keep='first')
+    )
+    listOfQuestionEnds = (
+        df.loc[
+            df['MarkerName'].notna()
+            & (df['MarkerName'].str.strip() != '')
+            & (df['MarkerName'].str.endswith('_end', na=False)),
+            ['Timestamp', 'MarkerName']
+        ]
+        .drop_duplicates(subset=['MarkerName'], keep='last')
+    )
+
+    starts = listOfQuestions.copy()
+    starts['question'] = starts['MarkerName'].str.strip()
+
+    ends = listOfQuestionEnds.copy()
+    ends['question'] = ends['MarkerName'].str.strip().str.replace(r'_end$', '', regex=True)
+
+    listOfQuestions = (
+        starts.merge(
+            ends[['Timestamp', 'question']],
+            on='question',
+            how='inner',
+            suffixes=('_start', '_end')
+        )
+        .rename(columns={'Timestamp_start': 'start_time', 'Timestamp_end': 'end_time'})
+        [['question', 'start_time', 'end_time']]
+        .sort_values('start_time')
+        .reset_index(drop=True)
+    )
+    for _, q in listOfQuestions.iterrows():
+        question = q['question']
+        if questionIdtoQuestionName and question in questionIdtoQuestionName:
+            question_name = questionIdtoQuestionName[question]
+        else:
+            question_name = str(question).strip()
+        start_time = q['start_time']
+        end_time = q['end_time']
+
+        df2 = filtered_df.loc[
+            (filtered_df['Timestamp'] >= start_time) & (filtered_df['Timestamp'] <= end_time),
+            ['ET_PupilLeft', 'ET_PupilRight']
+        ].copy()
+
+        df3 = filtered_df.loc[
+            (filtered_df['Timestamp'] >= start_time) & (filtered_df['Timestamp'] <= end_time),
+            ['Fixation Duration']
+        ].copy()
+        for col in InstrumentColumns:
+            if col in filtered_df.columns:
+                dfTemp = filtered_df.loc[
+                    (filtered_df['Timestamp'] >= start_time) & (filtered_df['Timestamp'] <= end_time),
+                    col
+                ].copy()
+                featureMean = dfTemp.mean()
+                if not pd.isna(featureMean):
+                    row = (respondentid, question_name, col, featureMean, tag)
+                    insertFaceRow(con, row)
+
+        df2['ET_PupilLeft'] = pd.to_numeric(df2['ET_PupilLeft'], errors='coerce')
+        df2['ET_PupilRight'] = pd.to_numeric(df2['ET_PupilRight'], errors='coerce')
+        df2 = df2.dropna(subset=['ET_PupilLeft', 'ET_PupilRight'])
+        pupil = (df2['ET_PupilLeft'].mean() + df2['ET_PupilRight'].mean()) / 2
+        duration = df3['Fixation Duration'].mean() 
+        row = (respondentid, question_name, float(pupil), float(duration), tag)
+        insertSensorRow(con, row)
+
+def LoadSensorData(databaseName = "data.db", sensorFolder = "./sensorData/", boundedboxes = None, tag = "default", rebuild = False, low_memory = False, questionIdtoQuestionName = None):
+    """
+    Load sensor data from a folder and insert it into a SQLite database.
+
+    Args:
+        databaseName (str; optional): The name of the SQLite database file.
+        sensorFolder (str; optional): The path to the folder containing the sensor data.
+        boundedboxes (list; optional): A list of bounded boxes to filter the data.
+        tag (str; optional): A tag to associate with the data.
+        rebuild (bool; optional): Whether to rebuild the database.
+        low_memory (bool; optional): Whether to use low memory mode when reading CSV files.
+
+    Returns:
+        None
+    """
+    con = sqlite3.connect(databaseName)
+    cur = con.cursor()
+    if rebuild:
+        cur.execute("DROP TABLE IF EXISTS SensorData")
+        cur.execute("DROP TABLE IF EXISTS InstrumentData")
+    cur.execute("CREATE TABLE IF NOT EXISTS SensorData (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, respondent_id INTEGER NOT NULL, question TEXT NOT NULL, pupil REAL, duration REAL, tag TEXT, UNIQUE(respondent_id, question, tag) ON CONFLICT REPLACE)")
+    cur.execute("CREATE TABLE IF NOT EXISTS InstrumentData (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, respondent_id INTEGER NOT NULL, question TEXT NOT NULL, property TEXT NOT NULL, value REAL, tag TEXT, UNIQUE(respondent_id, question, property, tag) ON CONFLICT REPLACE)")
+    con.commit()
+    cur.close()
+    folder = sensorFolder
+    if not os.path.isdir(folder):
+        raise FileNotFoundError(f"Sensor folder does not exist: {folder}")
+    
+    f = os.listdir(folder)
+    for file in f:
+        filename, file_extension = os.path.splitext(file)
+        if file_extension == '.csv':
+            _, respondentid = filename.split('_')
+            df = pd.read_csv(folder+file, comment='#', low_memory=low_memory)
+            ExtractData(con, df, int(respondentid), boundedboxes, tag, questionIdtoQuestionName)

CBACABLab/LoadSQLData.py

@@ -0,0 +1,311 @@
+import re
+import sqlite3
+import pandas as pd
+from .LoadRawData import LoadSettings
+
+def _base_question(q):
+    # Strip a trailing _<digits> only if something remains before it
+    return re.sub(r"^(.+)_\d+$", r"\1", q)
+
+def ListInstrumentColumns(databaseName="data.db"):
+    """
+    Return a list of all distinct instrument columns in the InstrumentData table.
+
+    Args:
+        databaseName (str or sqlite3.Connection; optional): The path to the SQLite database file or an open connection.
+
+    Returns:
+        list: The instrument columns, sorted alphabetically.
+    """
+    owns_connection = not isinstance(databaseName, sqlite3.Connection)
+    con = sqlite3.connect(databaseName) if owns_connection else databaseName
+    try:
+        df = pd.read_sql_query(
+            "SELECT DISTINCT property FROM InstrumentData ORDER BY property", con
+        )
+    finally:
+        if owns_connection:
+            con.close()
+    return df["property"].tolist()
+
+def ListLikertScales(settings=None, show_values=False):
+    """
+    Return a list of all Likert scales defined in settings.json.
+
+    Args:
+        settings (dict): Optional pre-loaded settings.
+        show_values (bool): If True, return a list of (scale_name, scale_values) tuples.
+    """
+    scales = (settings or LoadSettings()).get("LikertScales", {})
+    if show_values:
+        return sorted(scales.items())
+    return sorted(scales.keys())
+
+def ListQuestions(databaseName="data.db"):
+    """
+    Return a list of all distinct questions in the FullSurvey table.
+
+    Args:
+        databaseName (str or sqlite3.Connection; optional): The path to the SQLite database file or an open connection.
+
+    Returns:
+        list: The questions, sorted alphabetically.
+    """
+    owns_connection = not isinstance(databaseName, sqlite3.Connection)
+    con = sqlite3.connect(databaseName) if owns_connection else databaseName
+    try:
+        df = pd.read_sql_query(
+            "SELECT DISTINCT question FROM FullSurvey ORDER BY question", con
+        )
+    finally:
+        if owns_connection:
+            con.close()
+    return df["question"].tolist()
+
+def LoadQuestionsWide(questions, databaseName="data.db"):
+    """
+    Load survey responses in wide format, with one row per respondent and one column per question.
+
+    Pivots the long-format FullSurvey table using one LEFT JOIN per question. Missing answers
+    become NaN. Columns whose non-missing values all parse as numbers are converted to numeric dtypes.
+
+    Args:
+        questions (list): The questions to include as columns. Duplicates are ignored.
+        databaseName (str or sqlite3.Connection; optional): The path to the SQLite database file or an open connection.
+
+    Returns:
+        pandas.DataFrame: A respondent_id column followed by one column per question.
+
+    Raises:
+        ValueError: If questions is empty.
+    """
+    questions = list(dict.fromkeys(questions))
+    if not questions:
+        raise ValueError("questions must contain at least one question")
+
+    selects = ["r.respondent_id AS respondent_id"]
+    joins = []
+    for i in range(len(questions)):
+        selects.append(f"q{i}.response AS q{i}")
+        joins.append(
+            f"LEFT JOIN FullSurvey q{i} "
+            f"ON q{i}.respondent_id = r.respondent_id AND q{i}.question = ?"
+        )
+    sql = (
+        f"SELECT {', '.join(selects)} "
+        "FROM (SELECT DISTINCT respondent_id FROM FullSurvey) r "
+        f"{' '.join(joins)} "
+        "ORDER BY r.respondent_id"
+    )
+
+    owns_connection = not isinstance(databaseName, sqlite3.Connection)
+    con = sqlite3.connect(databaseName) if owns_connection else databaseName
+    try:
+        df = pd.read_sql_query(sql, con, params=questions)
+    finally:
+        if owns_connection:
+            con.close()
+
+    df.columns = ["respondent_id"] + questions
+    for column in questions:
+        converted = pd.to_numeric(df[column], errors="coerce")
+        if converted.notna().sum() == df[column].notna().sum():
+            df[column] = converted
+    return df
+
+
+
+
+def LoadSurveyWithSensorDataWide(questions, databaseName="data.db", tagFilter=None, instrumentProperties=None):
+    """
+    Join survey responses to SensorData and InstrumentData in wide format, with one row per respondent and question.
+
+    The survey is joined to SensorData and InstrumentData on respondent_id and question. Each tag
+    gets its own columns: pupil_<tag> and duration_<tag> for SensorData, and <property>_<tag>
+    for InstrumentData. When no tags are specified, a tag is not included in the column names.
+    Missing sensor or instrument data becomes NaN.
+
+    Args:
+        questions (list): The questions to include. Duplicates are ignored.
+        tagFilter (str, optional): Filter for specific tags in the sensor data.
+        instrumentProperties (list, optional): List of instrument properties to include from the instrument data.
+        databaseName (str or sqlite3.Connection; optional): The path to the SQLite database file or an open connection.
+
+    Returns:
+        pandas.DataFrame: One row per (respondent_id, question) with the survey response
+            and the sensor and instrument columns described above.
+
+    Raises:
+        ValueError: If questions is empty.
+    """
+    questions = list(dict.fromkeys(questions))
+    if not questions:
+        raise ValueError("questions must contain at least one question")
+    marks = ", ".join("?" * len(questions))
+    baseQuestions = list(dict.fromkeys(_base_question(q) for q in questions))
+    baseMarks = ", ".join("?" * len(baseQuestions))
+
+    if tagFilter is None:
+        tagFilter = ["default",]
+
+    if not isinstance(tagFilter, (list, tuple, set)):
+        raise ValueError("tagFilter must be a list, tuple, or set")
+    tagFilter = set(tagFilter)
+    if not tagFilter:
+        raise ValueError("tagFilter must contain at least one tag")
+    tagCondition = f"AND tag IN ({', '.join('?' * len(tagFilter))})"
+    tagParams = list(tagFilter)
+
+    if instrumentProperties is not None:
+        if not isinstance(instrumentProperties, (list, tuple, set)):
+            raise ValueError("instrumentProperties must be a list, tuple, or set")
+        instrumentProperties = set(instrumentProperties)
+        if not instrumentProperties:
+            raise ValueError("instrumentProperties must contain at least one property")
+        propertyCondition = f"AND property IN ({', '.join('?' * len(instrumentProperties))})"
+        propertyParams = list(instrumentProperties)
+    else:
+        propertyCondition = ""
+        propertyParams = []
+
+
+    owns_connection = not isinstance(databaseName, sqlite3.Connection)
+    con = sqlite3.connect(databaseName) if owns_connection else databaseName
+    try:
+        survey = pd.read_sql_query(
+            "SELECT respondent_id, question, response FROM FullSurvey "
+            f"WHERE question IN ({marks})", con, params=questions)
+        sensor = pd.read_sql_query(
+            "SELECT respondent_id, question, tag, pupil, duration FROM SensorData "
+            f"WHERE question IN ({baseMarks}) {tagCondition}", con, params=baseQuestions + tagParams)
+        face = pd.read_sql_query(
+            "SELECT respondent_id, question, tag, property, value FROM InstrumentData "
+            f"WHERE question IN ({baseMarks}) {tagCondition} {propertyCondition}", con, params=baseQuestions + tagParams + propertyParams)
+    finally:
+        if owns_connection:
+            con.close()
+
+    key = ["respondent_id", "question"]
+    mkey = ["respondent_id", "base"]
+
+    def widen(df, columns, values):
+        if df.empty:
+            return df[key].copy().rename(columns={"question": "base"})
+        wide = df.pivot_table(index=key, columns=columns, values=values, aggfunc="mean")
+
+        def column_name(measure, tag):
+            return measure if tag == "default" else f"{measure}_{tag}"
+
+        if isinstance(wide.columns, pd.MultiIndex) and len(values) == 1:
+            wide.columns = [column_name(c[-2], c[-1]) for c in wide.columns]
+        else:
+            wide.columns = [column_name(measure, tag) for measure, tag in wide.columns]
+        return wide.reset_index().rename(columns={"question": "base"})
+
+    sensor = widen(sensor.melt(id_vars=key + ["tag"], value_vars=["pupil", "duration"],
+                               var_name="measure", value_name="value"),
+                   ["measure", "tag"], ["value"])
+    face = widen(face.rename(columns={"property": "measure"}),
+                 ["measure", "tag"], ["value"])
+
+    survey["base"] = survey["question"].map(_base_question)
+    result = (survey.merge(sensor, on=mkey, how="left")
+                    .merge(face, on=mkey, how="left")
+                    .drop(columns="base"))
+    result["question"] = pd.Categorical(result["question"], categories=questions, ordered=True)
+    result = result.sort_values(key).reset_index(drop=True)
+    result["question"] = result["question"].astype(str)
+
+    for column in result.columns:
+        if column in key:
+            continue
+        converted = pd.to_numeric(result[column], errors="coerce")
+        if converted.notna().sum() == result[column].notna().sum():
+            result[column] = converted
+
+    return result
+
+def LoadSurveyWithSensorDataLong(questions, databaseName="data.db", tagFilter=None, instrumentProperties=None):
+    """
+    Join survey responses to SensorData and InstrumentData in long format, with one row per tag and instrument property.
+
+    The survey is joined to SensorData on respondent_id and question, and then to InstrumentData on
+    respondent_id, question and tag. Missing sensor or instrument data becomes NaN.
+
+    Args:
+        questions (list): The questions to include. Duplicates are ignored.
+        databaseName (str or sqlite3.Connection; optional): The path to the SQLite database file or an open connection.
+        tagFilter (list, tuple, or set; optional): If provided, only include rows with a tag in this set.
+        instrumentProperties (list; optional): A list of instrument properties to include.
+
+    Returns:
+        pandas.DataFrame: One row per (respondent_id, question, tag, property) with the columns
+            response, pupil, duration, property, and value.
+
+    Raises:
+        ValueError: If questions is empty.
+    """
+    questions = list(dict.fromkeys(questions))
+    if not questions:
+        raise ValueError("questions must contain at least one question")
+    marks = ", ".join("?" * len(questions))
+    baseQuestions = list(dict.fromkeys(_base_question(q) for q in questions))
+    baseMarks = ", ".join("?" * len(baseQuestions))
+
+    if tagFilter is None:
+        tagFilter = ["default",]
+
+    if not isinstance(tagFilter, (list, tuple, set)):
+        raise ValueError("tagFilter must be a list, tuple, or set")
+    tagFilter = set(tagFilter)
+    if not tagFilter:
+        raise ValueError("tagFilter must contain at least one tag")
+    tagCondition = f"AND tag IN ({', '.join('?' * len(tagFilter))})"
+    tagParams = list(tagFilter)
+
+
+    if instrumentProperties is not None:
+        if not isinstance(instrumentProperties, (list, tuple, set)):
+            raise ValueError("instrumentProperties must be a list, tuple, or set")
+        instrumentProperties = set(instrumentProperties)
+        if not instrumentProperties:
+            raise ValueError("instrumentProperties must contain at least one property")
+        propertyCondition = f"AND property IN ({', '.join('?' * len(instrumentProperties))})"
+        propertyParams = list(instrumentProperties)
+    else:
+        propertyCondition = ""
+        propertyParams = []
+
+    owns_connection = not isinstance(databaseName, sqlite3.Connection)
+    con = sqlite3.connect(databaseName) if owns_connection else databaseName
+    try:
+        survey = pd.read_sql_query(
+            "SELECT respondent_id, question, response FROM FullSurvey "
+            f"WHERE question IN ({marks})", con, params=questions)
+        sensor = pd.read_sql_query(
+            "SELECT respondent_id, question, tag, pupil, duration FROM SensorData "
+            f"WHERE question IN ({baseMarks}) {tagCondition}", con, params=baseQuestions + tagParams)
+        face = pd.read_sql_query(
+            "SELECT respondent_id, question, tag, property, value FROM InstrumentData "
+            f"WHERE question IN ({baseMarks}) {tagCondition} {propertyCondition}", con,
+            params=baseQuestions + tagParams + propertyParams)
+    finally:
+        if owns_connection:
+            con.close()
+
+    survey["base"] = survey["question"].map(_base_question)
+    sensor = sensor.rename(columns={"question": "base"})
+    face = face.rename(columns={"question": "base"})
+    result = survey.merge(sensor, on=["respondent_id", "base"], how="left")
+    result = result.merge(face, on=["respondent_id", "base", "tag"], how="left")
+    result = result.drop(columns="base")
+    result["question"] = pd.Categorical(result["question"], categories=questions, ordered=True)
+    result = result.sort_values(["respondent_id", "question", "tag"]).reset_index(drop=True)
+    result["question"] = result["question"].astype(str)
+    for column in result.columns:
+        if column in ["respondent_id", "question", "tag"]:
+            continue
+        converted = pd.to_numeric(result[column], errors="coerce")
+        if converted.notna().sum() == result[column].notna().sum():
+            result[column] = converted
+    return result