{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dbamman/anlp25/blob/main/6.tests/ParametricTest.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eecvY5ezp0wb"
   },
   "source": [
    "This notebook explores a simple hypothesis test checking whether the accuracy of a trained model for binary classification is meaningfully different from a majority class baseline.  We test this making a parametric assumption: we assume that the binary correct/incorrect results follow a binomial distribution (and approximate the binomial with a normal distribution)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "wlggnu6sp0wf"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "from collections import Counter\n",
    "from math import sqrt\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy import sparse\n",
    "from scipy.stats import norm\n",
    "from sklearn import linear_model, preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WJ1fXpp9qJhH"
   },
   "source": [
    "Download the Convote dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "a5EERsnLqC30"
   },
   "outputs": [],
   "source": [
    "# get Convote data\n",
    "!wget --no-check-certificate https://raw.githubusercontent.com/dbamman/anlp25/refs/heads/main/data/convote/train.tsv -O convote_train.tsv\n",
    "!wget --no-check-certificate https://raw.githubusercontent.com/dbamman/anlp25/refs/heads/main/data/convote/dev.tsv -O convote_dev.tsv\n",
    "!wget --no-check-certificate https://raw.githubusercontent.com/dbamman/anlp25/refs/heads/main/data/convote/test.tsv -O convote_test.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WbhcE652p0wh"
   },
   "outputs": [],
   "source": [
    "def read_data(filename):\n",
    "    df = pd.read_csv(filename, names=[\"label\", \"text\"], sep=\"\\t\")\n",
    "\n",
    "    return df.text.to_list(), df.label.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "yiI9m7s9p0wi"
   },
   "outputs": [],
   "source": [
    "x_train, y_train = read_data(\"convote_train.tsv\")\n",
    "x_dev, y_dev = read_data(\"convote_dev.tsv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0ldeIi_Ip0wi"
   },
   "outputs": [],
   "source": [
    "def majority_class(y_train, y_dev):\n",
    "    label_counts = Counter(y_train)\n",
    "    majority = label_counts.most_common(1)[0][0]\n",
    "\n",
    "    correct = 0.\n",
    "    for label in y_dev:\n",
    "        if label == majority:\n",
    "            correct += 1\n",
    "\n",
    "    print(\"%s\\t%.3f\" % (majority, correct/len(y_dev)))\n",
    "    return correct / len(y_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ooOU4CqSp0wj"
   },
   "outputs": [],
   "source": [
    "# Here's a sample dictionary we can create by inspecting the output of the Mann-Whitney test (in 2.compare/)\n",
    "\n",
    "dem_dictionary = set([\"republican\",\"cut\", \"opposition\"])\n",
    "repub_dictionary = set([\"growth\",\"economy\"])\n",
    "\n",
    "def political_dictionary_feature(tokens):\n",
    "    feats = {}\n",
    "    for word in tokens:\n",
    "        if word in dem_dictionary:\n",
    "            feats[\"word_in_dem_dictionary\"] = 1\n",
    "        if word in repub_dictionary:\n",
    "            feats[\"word_in_repub_dictionary\"] = 1\n",
    "    return feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lRW4BbxEp0wj"
   },
   "outputs": [],
   "source": [
    "def unigram_feature(tokens):\n",
    "    feats = {}\n",
    "    for word in tokens:\n",
    "        feats[\"UNIGRAM_%s\" % word] = 1\n",
    "    return feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Cqj7nqiKp0wk"
   },
   "outputs": [],
   "source": [
    "def build_features(x_train, feature_functions):\n",
    "    data = []\n",
    "    for doc in x_train:\n",
    "        feats = {}\n",
    "        tokens = doc.split(\" \")\n",
    "\n",
    "        for function in feature_functions:\n",
    "            feats.update(function(tokens))\n",
    "\n",
    "        data.append(feats)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xysSq9v0p0wk"
   },
   "outputs": [],
   "source": [
    "# This helper function converts a dictionary of feature names to unique numerical ids\n",
    "def create_vocab(data):\n",
    "    feature_vocab = {}\n",
    "    idx = 0\n",
    "    for doc in data:\n",
    "        for feat in doc:\n",
    "            if feat not in feature_vocab:\n",
    "                feature_vocab[feat] = idx\n",
    "                idx += 1\n",
    "\n",
    "    return feature_vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uNpeVKHJp0wk"
   },
   "outputs": [],
   "source": [
    "# This helper function converts a dictionary of feature names to a sparse representation\n",
    "# that we can fit in a scikit-learn model.  This is important because almost all feature\n",
    "# values will be 0 for most documents (note: why?), and we don't want to save them all in\n",
    "# memory.\n",
    "\n",
    "def features_to_ids(data, feature_vocab):\n",
    "    new_data = sparse.lil_matrix((len(data), len(feature_vocab)))\n",
    "    for idx,doc in enumerate(data):\n",
    "        for f in doc:\n",
    "            if f in feature_vocab:\n",
    "                new_data[idx, feature_vocab[f]] = doc[f]\n",
    "    return new_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nwlbtFPhp0wl"
   },
   "outputs": [],
   "source": [
    "# This function trains a model and returns the predicted and true labels for test data\n",
    "def evaluate(x_train, x_dev, y_train, y_dev, feature_functions):\n",
    "    x_train_feat = build_features(x_train, feature_functions)\n",
    "    x_dev_feat = build_features(x_dev, feature_functions)\n",
    "\n",
    "    # just create vocabulary from features in *training* data\n",
    "    feature_vocab = create_vocab(x_train_feat)\n",
    "\n",
    "    x_train_ids = features_to_ids(x_train_feat, feature_vocab)\n",
    "    x_dev_ids = features_to_ids(x_dev_feat, feature_vocab)\n",
    "\n",
    "    logreg = linear_model.LogisticRegression(C=1.0, solver='lbfgs', penalty='l2', max_iter=10000)\n",
    "    logreg.fit(x_train_ids, y_train)\n",
    "    predictions = logreg.predict(x_dev_ids)\n",
    "    return (predictions, y_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mjF1Vn8Hp0wl"
   },
   "outputs": [],
   "source": [
    "baseline = majority_class(y_train, y_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XNP_3CzXp0wl"
   },
   "outputs": [],
   "source": [
    "def binomial_test(predictions, targets, baseline, significance_level=0.95):\n",
    "    correct = [int(prediction == target) for prediction, target in zip(predictions, targets)]\n",
    "\n",
    "    success_rate = np.mean(correct)\n",
    "\n",
    "    # two-tailed test\n",
    "    critical_value = (1 - significance_level) / 2\n",
    "    # ppf finds z such that p(X < z) = critical_value\n",
    "    z_alpha = -1 * norm.ppf(critical_value)\n",
    "    print(\"Critical value: %.3f\\tz_alpha: %.3f\" % (critical_value, z_alpha))\n",
    "\n",
    "    # the standard error is the square root of (the variance/sample size)\n",
    "    # the variance for a binomial test is p*(1-p)\n",
    "    standard_error = sqrt((success_rate * (1-success_rate)) / len(correct))\n",
    "\n",
    "    Z = (success_rate - baseline) / standard_error\n",
    "    lower = success_rate - z_alpha * standard_error\n",
    "    upper = success_rate + z_alpha * standard_error\n",
    "    pval = norm.cdf(-abs(Z))\n",
    "\n",
    "    print (\"Accuracy: %.3f, n = %s\" % (success_rate, len(correct)))\n",
    "    print(\"%s%% Confidence interval: [%.3f,%.3f]\" % (significance_level*100, lower, upper))\n",
    "\n",
    "    print(\"Z score: %.3f\" % Z)\n",
    "    print(\"p-value: %.5f\" % pval)\n",
    "\n",
    "    print (\"Critical region corresponding to z_alpha=[%.3f,%.3f]: [%.3f, %.3f]\" % (-z_alpha, z_alpha, baseline-z_alpha*standard_error, baseline+z_alpha*standard_error))\n",
    "    print (\"Can we reject null that %.3f is different from %.3f at %s significance level? %s\" % (success_rate, baseline, significance_level*100, \"Yes\" if Z < -z_alpha or Z > z_alpha else \"No\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "TvNTs2DGp0wl"
   },
   "outputs": [],
   "source": [
    "features = [political_dictionary_feature]\n",
    "predictions, targets = evaluate(x_train, x_dev, y_train, y_dev, features)\n",
    "binomial_test(predictions, targets, baseline, significance_level=.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6_kmP1TLp0wm"
   },
   "outputs": [],
   "source": [
    "features = [unigram_feature]\n",
    "predictions, targets = evaluate(x_train, x_dev, y_train, y_dev, features)\n",
    "binomial_test(predictions, targets, baseline, significance_level=.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-8bt2oV5p0wm"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}