{
 "nbformat": 4,
 "nbformat_minor": 5,
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "language_info": {
   "name": "python"
  },
  "accelerator": "GPU",
  "gpuClass": "standard"
 },
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ballot Guide Lab v2 — conversational voter questions\n",
    "\n",
    "Runs 15 revised cases in three conditions using the pinned Qwen3.5-2B base and existing trained LoRA adapter. **No retraining and no paid inference endpoint.** Choose a free T4 GPU.\n",
    "\n",
    "The original v1 questions and results remain archived. This notebook creates `results/colab_lora_v2_voterquestions`. It generates real outputs and a blinded rating packet; a fresh independent AI judge and a team member must score those outputs afterward.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare pinned environment and v2 sources"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "from pathlib import Path\n",
    "from urllib.request import urlopen\n",
    "import json, hashlib, zipfile, sys, subprocess, os\n",
    "ROOT = Path('/content/ballot-v2-experiment/ballot-guide-lab')\n",
    "PROJECT_URL = 'https://www.utkarshruhela.com/votereducation/files/prototype-v2-voter-questions.zip'\n",
    "PROJECT_SHA256 = '54d5a7754e13b7b19ff952ccd4c2b772094dfbe9b728af9ff2a6d77eb04a3533'\n",
    "CASE_SHA256 = '97ad117af35b5e56589435ab25cf7591da724c06db983d7374aef966bdc38180'\n",
    "archive = Path('/content/ballot-v2-project.zip')\n",
    "archive.write_bytes(urlopen(PROJECT_URL, timeout=90).read())\n",
    "assert hashlib.sha256(archive.read_bytes()).hexdigest() == PROJECT_SHA256, 'Project archive changed.'\n",
    "destination = ROOT.parent\n",
    "with zipfile.ZipFile(archive) as z:\n",
    "    for name in z.namelist():\n",
    "        assert (destination/name).resolve().is_relative_to(destination.resolve()), 'Unsafe archive path.'\n",
    "    z.extractall(destination)\n",
    "assert hashlib.sha256((ROOT/'pilot/cases.json').read_bytes()).hexdigest() == CASE_SHA256\n",
    "sys.path.insert(0,str(ROOT)); sys.path.insert(0,str(ROOT/'training'))\n",
    "os.environ['UNSLOTH_COMPILE_DISABLE']='1'\n",
    "os.environ['GRADIO_ANALYTICS_ENABLED']='False'\n",
    "subprocess.check_call([sys.executable,'-m','pip','install','--quiet','--upgrade','uv'])\n",
    "import numpy, PIL\n",
    "deps=['torch==2.8.0','triton==3.4.0',f'numpy=={numpy.__version__}',f'pillow=={PIL.__version__}',\n",
    "      'torchvision==0.23.0','torchaudio==2.8.0','bitsandbytes==0.50.2','xformers==0.0.32.post2',\n",
    "      'unsloth_zoo[base] @ git+https://github.com/unslothai/unsloth-zoo@f787f63d3411c00633d1768f617348ee6867ed37',\n",
    "      'unsloth[base] @ git+https://github.com/unslothai/unsloth@7d2bd0118422c93840ea524303de679a1dc916cd']\n",
    "subprocess.check_call(['uv','pip','install','--system','--quiet',*deps])\n",
    "subprocess.check_call(['uv','pip','install','--system','--quiet','--no-deps','torchcodec==0.7.0','trl==0.22.2','torchao==0.18.0'])\n",
    "subprocess.check_call(['uv','pip','install','--system','--quiet','transformers==5.2.0','pypdf==6.10.0',\n",
    "                       'tokenizers==0.22.2','datasets==4.8.5','peft==0.21.1','huggingface-hub==1.33.0'])\n",
    "subprocess.check_call(['unsloth','install-kernels'])\n",
    "print('V2_SETUP_COMPLETE',ROOT, 'cases=15; model training is not required.', flush=True)\n"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reload the saved adapter on the pinned base"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "from unsloth import FastLanguageModel\n",
    "import torch, sys, os, hashlib, json\n",
    "from huggingface_hub import snapshot_download\n",
    "from peft import PeftModel\n",
    "from colab_pipeline import MODEL_ID, MODEL_REVISION, load_base, generation_backend\n",
    "from sources import CONFIG, index_sources, packet_for\n",
    "assert torch.cuda.is_available(), 'Select a T4 GPU runtime.'\n",
    "torch._dynamo.config.disable=True\n",
    "ADAPTER_REPO='utkarshruhela/ballot-guide-ca2024-qwen35-2b-lora'\n",
    "ADAPTER_REVISION='d7c826f350b2031c5041dcef0035e1cf62a64061'\n",
    "ADAPTER_SHA256='53bf107ba041bc131421f01e478913ba647c6c0b354b5cec5d348e99c4171861'\n",
    "adapter_dir=Path(snapshot_download(repo_id=ADAPTER_REPO,revision=ADAPTER_REVISION,\n",
    "                     allow_patterns=['adapter_config.json','adapter_model.safetensors'],token=False))\n",
    "assert hashlib.sha256((adapter_dir/'adapter_model.safetensors').read_bytes()).hexdigest()==ADAPTER_SHA256\n",
    "adapter_config=json.loads((adapter_dir/'adapter_config.json').read_text())\n",
    "assert adapter_config['base_model_name_or_path']==MODEL_ID\n",
    "assert adapter_config.get('revision') in {None,MODEL_REVISION}\n",
    "model,tokenizer=load_base()\n",
    "model=PeftModel.from_pretrained(model,str(adapter_dir),is_trainable=False)\n",
    "FastLanguageModel.for_inference(model)\n",
    "assert model.get_layer_status() and all(layer.enabled for layer in model.get_layer_status())\n",
    "records=index_sources()\n",
    "assert hashlib.sha256((ROOT/'data/raw/california-2024-voter-guide.pdf').read_bytes()).hexdigest()==CONFIG['source_sha256']\n",
    "print('V2_MODEL_READY',torch.cuda.get_device_name(0),'saved adapter verified; no retraining.',flush=True)\n"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate, verify and export all 45 new outputs"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {},
   "source": [
    "from datetime import datetime, timezone\n",
    "from collections import Counter\n",
    "import importlib.metadata, platform, time, zipfile, json, hashlib, subprocess\n",
    "from IPython.display import display, HTML, FileLink\n",
    "import html\n",
    "run_dir=ROOT/'results/colab_lora_v2_voterquestions'\n",
    "run_dir.mkdir(parents=True,exist_ok=True)\n",
    "cases=json.loads((ROOT/'pilot/cases.json').read_text())\n",
    "assert len(cases)==15 and sum(c['difficult'] for c in cases)==5\n",
    "packets={c['id']:packet_for(c['proposition'],records,c['question']) for c in cases}\n",
    "manifest={'run_name':'colab_lora_v2_voterquestions','test_version':'v2-voterquestions',\n",
    "          'started_at':datetime.now(timezone.utc).isoformat(),'base_model':MODEL_ID,'base_revision':MODEL_REVISION,\n",
    "          'adapter_repo':ADAPTER_REPO,'adapter_revision':ADAPTER_REVISION,'adapter_sha256':ADAPTER_SHA256,\n",
    "          'cases_sha256':hashlib.sha256((ROOT/'pilot/cases.json').read_bytes()).hexdigest(),\n",
    "          'source_pdf_sha256':CONFIG['source_sha256'],'gpu':torch.cuda.get_device_name(0),\n",
    "          'decoder':'greedy Transformers; no JSON grammar','max_tokens':CONFIG['inference']['max_tokens'],\n",
    "          'seed':894,'max_length':3072,'enable_thinking':False,'retrained':False,\n",
    "          'untuned_method':'PEFT disable_adapter context; every layer verified disabled',\n",
    "          'prototype_prompt_sha256':hashlib.sha256((ROOT/'prompts/prototype.txt').read_bytes()).hexdigest(),\n",
    "          'baseline_prompt_sha256':hashlib.sha256((ROOT/'prompts/baseline.txt').read_bytes()).hexdigest(),\n",
    "          'parameter_elements_by_dtype':{str(d):sum(p.numel() for p in model.parameters() if p.dtype==d) for d in {p.dtype for p in model.parameters()}},\n",
    "          'packages':{n:importlib.metadata.version(n) for n in ['torch','transformers','unsloth','unsloth_zoo','peft','huggingface-hub','pypdf']}}\n",
    "mfile=run_dir/'runtime.json'\n",
    "if mfile.exists():\n",
    "    old=json.loads(mfile.read_text())\n",
    "    for key in ['cases_sha256','adapter_sha256','base_revision','prototype_prompt_sha256','baseline_prompt_sha256']:\n",
    "        assert old[key]==manifest[key], 'Experiment changed; use a new output directory.'\n",
    "    manifest=old\n",
    "else:mfile.write_text(json.dumps(manifest,indent=2))\n",
    "output=run_dir/'outputs.jsonl'\n",
    "prior=[json.loads(l) for l in output.read_text().splitlines() if l.strip()] if output.exists() else []\n",
    "done={(r['model_label'],r['condition'],r['case_id']) for r in prior}\n",
    "backend=generation_backend(model,tokenizer)\n",
    "started=time.perf_counter()\n",
    "for label,conditions in [('untuned',['baseline','prototype']),('finetuned',['prototype'])]:\n",
    "    from contextlib import nullcontext\n",
    "    with model.disable_adapter() if label=='untuned' else nullcontext():\n",
    "        assert all(layer.enabled==(label=='finetuned') for layer in model.get_layer_status())\n",
    "        for case in cases:\n",
    "            for condition in conditions:\n",
    "                key=(label,condition,case['id'])\n",
    "                if key in done:continue\n",
    "                torch.manual_seed(894)\n",
    "                result=backend.answer(case['question'],case['proposition'],condition,records)\n",
    "                result.update(model_label=label,case_id=case['id'],difficult=case['difficult'],\n",
    "                              generated_at=datetime.now(timezone.utc).isoformat(),retrieved_sources=packets[case['id']],\n",
    "                              decoder='greedy Transformers; no JSON grammar',test_version='v2-voterquestions')\n",
    "                with output.open('a') as f:f.write(json.dumps(result,ensure_ascii=False)+'\\n')\n",
    "                done.add(key)\n",
    "                print('V2_PROGRESS',len(done),'/45',label,condition,case['id'],result['status'],result['latency_seconds'],'s',flush=True)\n",
    "rows=[json.loads(l) for l in output.read_text().splitlines() if l.strip()]\n",
    "assert len(rows)==45\n",
    "for case in cases:\n",
    "    group=[r for r in rows if r['case_id']==case['id']]\n",
    "    assert len(group)==3\n",
    "    assert all(r['retrieved_sources']==group[0]['retrieved_sources'] for r in group)\n",
    "    paired=[r for r in group if r['condition']=='prototype']\n",
    "    assert paired[0]['prompt_sha256']==paired[1]['prompt_sha256'] and paired[0]['source_ids']==paired[1]['source_ids']\n",
    "summary={'outputs':len(rows),'case_count':15,'difficult_cases':5,'generation_seconds':round(time.perf_counter()-started,2),\n",
    "         'adapter_layers_enabled_after_run':all(s.enabled for s in model.get_layer_status()),\n",
    "         'paired_prompts_and_source_packets_identical':True,'scores_complete':False,'human_ratings_complete':False,\n",
    "         'groups':{f'{label}_{condition}':{'outputs':len(g),'technical_check_passes':sum(not r['validation_errors'] for r in g) if condition=='prototype' else None} for label,condition in [('untuned','baseline'),('untuned','prototype'),('finetuned','prototype')] for g in [[r for r in rows if r['model_label']==label and r['condition']==condition]]}}\n",
    "(run_dir/'technical_summary.json').write_text(json.dumps(summary,indent=2))\n",
    "subprocess.check_call([sys.executable,str(ROOT/'evaluate.py'),'prepare','--run-dir',str(run_dir)])\n",
    "(run_dir/'test_cases.json').write_text(json.dumps(cases,indent=2))\n",
    "package=Path('/content/ballot-guide-v2-results.zip')\n",
    "with zipfile.ZipFile(package,'w',zipfile.ZIP_DEFLATED) as z:\n",
    "    for p in run_dir.rglob('*'):\n",
    "        if p.is_file():z.write(p,str(p.relative_to(ROOT)))\n",
    "    for rel in ['pilot/cases.json','pilot/rubric.json','prompts/prototype.txt','prompts/baseline.txt','prompts/judge.txt','evaluation/agent_task.txt','config.json','data/source_manifest.json']:\n",
    "        z.write(ROOT/rel,rel)\n",
    "print('V2_GENERATION_COMPLETE',json.dumps(summary),flush=True)\n",
    "print('V2_RESULTS_SHA256',hashlib.sha256(package.read_bytes()).hexdigest(),flush=True)\n",
    "display(FileLink(str(package)))\n"
   ],
   "execution_count": null,
   "outputs": []
  }
 ]
}