{ "cells": [ { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "import yaml\n", "import csv\n", "import os\n", "import json\n", "from scripts.run_pipeline import get_expanded_pipeline_configs" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Collecting results" ] }, { "cell_type": "code", "execution_count": 34, "metadata": {}, "outputs": [], "source": [ "base_save_dir = ''\n", "behaviors_path = './data/behavior_datasets/harmbench_behaviors_text_test.csv'\n", "pipeline_config_path = './configs/pipeline_configs/run_pipeline.yaml'\n", "model_configs = './configs/model_configs/models.yaml'\n", "max_num_tokens = 512 # Change this if using a different number of tokens in completions\n", "\n", "proper_method_names = ['AutoDAN', 'AutoPrompt', 'DirectRequest', 'GCG-Multi', 'GCG-Transfer', 'FewShot', 'GBDA', 'GCG', 'GCG_custom_targets',\n", " 'HumanJailbreaks', 'PAIR', 'PAP-top5', 'PEZ', 'TAP', 'TAP-Transfer', 'UAT', 'ZeroShot']\n", "\n", "models = ['llama2_7b', 'llama2_13b', 'llama2_70b', 'vicuna_7b_v1_5', 'vicuna_13b_v1_5', 'baichuan2_7b', 'baichuan2_13b', 'qwen_7b_chat', 'qwen_14b_chat', 'qwen_72b_chat',\n", " 'koala_7b', 'koala_13b', 'orca_2_7b', 'orca_2_13b', 'solar_10_7b_instruct', 'mistral_7b_v2', 'mixtral_8x7b', 'openchat_3_5_1210', 'starling_7b', 'zephyr_7b',\n", " 'zephyr_7b_robust6_data2', 'gpt-3.5-turbo-1106', 'gpt-3.5-turbo-0613',\n", " 'gpt-4-1106-preview', 'gpt-4-0613', 'gpt-4-vision-preview', 'claude-instant-1', 'claude-2.1', 'claude-2', 'gemini', 'mistral-medium']" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [], "source": [ "with open(pipeline_config_path, 'r') as f:\n", " pipeline_config = yaml.safe_load(f)\n", "\n", "with open(model_configs, 'r') as f:\n", " model_configs = yaml.safe_load(f)\n", "\n", "with open(behaviors_path, 'r') as f:\n", " reader = csv.DictReader(f)\n", " behaviors = {row['BehaviorID']: row for row in reader}\n", "behavior_ids = list(behaviors.keys())" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [], "source": [ "expanded_pipeline_configs = get_expanded_pipeline_configs(pipeline_config, model_configs, proper_method_names, models)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "all_results = {}\n", "\n", "for proper_method_name, expanded_pipeline_config in expanded_pipeline_configs.items():\n", "\n", " method = expanded_pipeline_config['class_name']\n", " method_path = os.path.join(base_save_dir, method)\n", " # check that method folder exists\n", " if not os.path.exists(method_path):\n", " print(f'could not find {method_path}; skipping')\n", " continue\n", " \n", " if method not in all_results:\n", " all_results[method] = {}\n", " available_experiments = os.listdir(method_path)\n", " expected_experiments = []\n", " for experiment_name in expanded_pipeline_config['model_to_experiment'].values():\n", " expected_experiments.append(experiment_name)\n", " available_experiments = list(set(available_experiments).intersection(expected_experiments))\n", "\n", " for experiment_name in available_experiments:\n", " experiment_path = os.path.join(method_path, experiment_name)\n", " all_results[method][experiment_name] = {}\n", " test_cases_path = os.path.join(experiment_path, 'test_cases', 'test_cases.json')\n", "\n", " # check that test_cases.json exists\n", " if not os.path.exists(test_cases_path):\n", " status = 'missing test_cases.json'\n", " all_results[method][experiment_name]['status'] = status\n", " print(f'missing test cases {test_cases_path}; skipping')\n", " continue\n", " \n", " # check that all behaviors are in test_cases.json\n", " with open(test_cases_path) as f:\n", " test_cases = json.load(f)\n", " missing_behaviors = []\n", " for behavior_id in behavior_ids:\n", " if behavior_id not in test_cases:\n", " missing_behaviors.append(behavior_id)\n", " if len(missing_behaviors) > 0:\n", " status = 'missing behaviors'\n", " all_results[method][experiment_name]['status'] = status\n", " all_results[method][experiment_name]['missing_behaviors'] = missing_behaviors\n", " print(f'missing behaviors in {test_cases_path}')\n", " # print(f'missing behaviors in {test_cases_path}; skipping')\n", " # continue\n", "\n", " # check that results dir is present\n", " if max_num_tokens != 512:\n", " results_dir = os.path.join(experiment_path, 'results', f'{max_num_tokens}_tokens')\n", " else:\n", " results_dir = os.path.join(experiment_path, 'results')\n", " \n", " if not os.path.exists(results_dir):\n", " status = 'missing results dir'\n", " all_results[method][experiment_name]['status'] = status\n", " print(f'missing results {results_dir}; skipping')\n", " continue\n", " \n", " # test cases and results are present, so we can set the experiment status to finished\n", " # we also check the status of each model's results below\n", " if 'status' not in all_results[method][experiment_name]:\n", " all_results[method][experiment_name]['status'] = 'finished'\n", "\n", " # check that results have been computed and populate results if so\n", " all_results[method][experiment_name]['results'] = {}\n", " results_jsons = os.listdir(results_dir)\n", " model_names = [x[:-5] for x in results_jsons if x[-5:] == '.json']\n", " \n", " for model_name in model_names:\n", " all_results[method][experiment_name]['results'][model_name] = {}\n", " with open(os.path.join(results_dir, f'{model_name}.json')) as f:\n", " results = json.load(f)\n", " missing_behaviors = []\n", " for behavior_id in behavior_ids:\n", " if behavior_id not in results:\n", " missing_behaviors.append(behavior_id)\n", " if len(missing_behaviors) > 0:\n", " status = 'missing behaviors'\n", " all_results[method][experiment_name]['results'][model_name]['status'] = status\n", " all_results[method][experiment_name]['results'][model_name]['missing_behaviors'] = missing_behaviors\n", " # continue\n", "\n", " all_results[method][experiment_name]['results'][model_name]['status'] = 'finished'\n", " all_results[method][experiment_name]['results'][model_name] = results" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "missing_behaviors = []\n", "for method in ['PAIR']:\n", " for experiment in ['mixtral_8x7b']:\n", " if all_results[method][experiment]['status'] != 'finished':\n", " missing_behaviors.extend(all_results[method][experiment]['missing_behaviors'])\n", "\n", "print(','.join(list(set(missing_behaviors))))\n", "print(len(missing_behaviors))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for x in all_results['AutoPrompt']['llama2_70b']['missing_behaviors']:\n", " print(behavior_ids.index(x), x)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Printing results (for tables)" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "==================================================\n", "Functional Behavior Category: all\n", "\n", "\n", "\n", "GCG\tGCG-Multi\tGCG-Transfer\tPEZ\tGBDA\tUAT\tAutoPrompt\tFewShot\tZeroShot\tPAIR\tTAP\tTAP-Transfer\tAutoDAN\tPAP-top5\tHumanJailbreaks\tDirectRequest\n", "31.874999999999996\t21.08782880071333\t19.261581330546846\t1.8124999999999998\t1.25\t4.375\t16.5625\t5.0\t2.1875\t9.375\t9.0625\t7.836990595611286\t0.0\t2.6959247648902824\t0.6896551724137931\t0.625\n", "30.312499999999996\t11.353711790393014\t16.579588993382096\t1.875\t2.375\t1.5625\t17.8125\t6.875000000000001\t2.875\t14.6875\t14.0625\t8.150470219435736\t0.9375\t3.5736677115987465\t1.8181818181818181\t3.125\n", "39.0625\t10.9375\t21.839080459770116\t3.125\t2.1875\t4.38871473354232\t21.5625\t7.836990595611286\t2.9375\t14.374999999999998\t13.750000000000002\t15.673981191222571\t2.8125\t4.326018808777429\t2.3824451410658307\t3.125\n", "65.9375\t60.872274143302185\t60.675722744688265\t19.0625\t19.0625\t18.4375\t56.56250000000001\t43.4375\t26.8125\t53.75\t51.724137931034484\t60.18808777429467\t66.25\t19.184952978056426\t38.93416927899687\t23.75\n", "65.625\t60.56118472330475\t55.172413793103445\t16.4375\t14.5625\t14.374999999999998\t43.75\t32.1875\t23.0\t50.31250000000001\t53.60501567398119\t64.89028213166145\t65.9375\t20.12539184952978\t40.50156739811912\t20.0\n", "62.18749999999999\t40.45761830473219\t46.08150470219436\t31.874999999999996\t28.9375\t28.749999999999996\t47.1875\t27.187499999999996\t27.875\t38.125\t51.724137931034484\t59.56112852664577\t53.43750000000001\t19.122257053291534\t27.836990595611283\t18.4375\n", "61.5625\t52.34375\t44.932079414838036\t28.375\t26.5625\t50.31250000000001\t54.37499999999999\t38.4375\t25.75\t52.81249999999999\t54.54545454545454\t63.63636363636363\t60.18808777429467\t21.88087774294671\t31.724137931034484\t19.375\n", "59.4855305466238\t52.31249999999999\t37.89620341344479\t12.812499999999998\t12.5\t10.0\t49.21135646687697\t31.25\t15.937499999999998\t49.6875\t53.125\t57.993730407523515\t47.5\t13.04075235109718\t24.26332288401254\t13.125\n", "62.5\t53.93750000000001\t38.87147335423197\t11.1875\t12.0\t10.0\t45.57377049180328\t28.125\t16.6875\t45.3125\t48.125\t55.4858934169279\t51.87500000000001\t13.60501567398119\t29.529780564263326\t17.1875\n", "exp not started\texp not started\t36.60745384883316\t13.750000000000002\t14.0625\t11.25\texp not started\t32.1875\t18.375\t46.5625\t50.0\t56.42633228840125\t41.25\t21.379310344827587\t38.18181818181819\t17.1875\n", "60.0\t54.55962587685113\t52.0027864855451\t41.75\t51.1875\t49.6875\t54.37499999999999\t41.875\t43.0625\t49.6875\t58.75\t57.36677115987461\t54.0625\t19.247648902821314\t26.771159874608152\t38.125\n", "62.18749999999999\t57.07165109034268\t57.36677115987461\t46.1875\t52.4375\t52.81249999999999\t59.375\t38.4375\t37.0625\t52.5\t58.75\t59.87460815047022\t66.25\t16.489028213166144\t31.661442006269592\t27.187499999999996\n", "45.625\t39.12704598597038\t59.66562173458725\t37.8125\t37.8125\t39.6875\t35.625\t46.875\t41.125\t57.49999999999999\t57.8125\t60.50156739811913\t70.9375\t18.30721003134796\t39.122257053291534\t38.75\n", "50.625\t30.334075723830733\t51.79380006966214\t36.25\t34.5625\t35.0\t32.5\t50.625\t42.3125\t55.625\t60.9375\t63.94984326018809\t69.375\t20.0\t42.44514106583072\t45.0\n", "56.56250000000001\t61.33630289532294\t58.620689655172406\t54.87499999999999\t54.0\t53.125\t54.0625\t57.49999999999999\t55.0625\t56.25\t66.875\t66.4576802507837\t71.875\t31.03448275862069\t60.438871473354226\t60.0\n", "69.0625\t64.140625\t64.68129571577848\t50.74999999999999\t52.4375\t53.125\t61.875\t49.6875\t40.8125\t53.43750000000001\t62.81250000000001\t65.8307210031348\t71.5625\t26.64576802507837\t58.7460815047022\t45.9375\n", "100.0\t63.90625\t62.20828979449669\t0.0\t0.0\texp not started\texp not started\t51.24999999999999\t40.0\t61.12852664576802\t68.65203761755487\t68.96551724137932\t72.8125\t28.589341692789972\t53.60501567398119\t47.1875\n", "65.3125\t53.98663697104678\t56.94879832810867\t39.0\t43.5\t41.5625\t55.00000000000001\t54.37499999999999\t43.5625\t53.125\t64.375\t66.77115987460816\t74.375\t26.332288401253916\t51.47335423197492\t45.9375\n", "65.3125\t61.942959001782526\t58.93416927899686\t49.6875\t57.875\t53.75\t62.18749999999999\t55.625\t50.31250000000001\t58.93416927899686\t68.75\t68.02507836990596\t74.6875\t31.598746081504704\t60.75235109717868\t57.49999999999999\n", "69.375\t62.0597414177441\t60.91954022988506\t62.0\t63.125\t61.875\t59.68750000000001\t63.74999999999999\t61.18749999999999\t59.06249999999999\t67.8125\t70.21943573667711\t75.625\t32.41379310344827\t66.52037617554859\t67.8125\n", "6.25\t5.208333333333334\t0.0\t2.75\t0.1875\t0.0\t5.0\t43.125\t7.062499999999999\t47.8125\t61.875\t54.858934169278996\t17.1875\t24.76489028213166\t13.730407523510971\t15.0\n", "model not expected for method\tmodel not expected for method\t38.62068965517241\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t24.4375\t47.8125\t49.21630094043887\t63.00940438871473\tmodel not expected for method\t15.235109717868339\t24.70219435736677\t22.1875\n", "model not expected for method\tmodel not expected for method\t42.63322884012539\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t28.749999999999996\t36.25\t38.87147335423197\t47.64890282131661\tmodel not expected for method\t11.347962382445141\t3.1418753068237604\t33.75\n", "model not expected for method\tmodel not expected for method\t22.50783699059561\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t18.9375\t39.375\t43.260188087774296\t55.79937304075236\tmodel not expected for method\t16.99059561128527\t12.100313479623823\t20.9375\n", "model not expected for method\tmodel not expected for method\t22.31974921630094\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t12.687499999999998\t33.75\t37.61755485893417\t57.68025078369906\tmodel not expected for method\t11.598746081504702\t2.633228840125392\t9.6875\n", "model not expected for method\tmodel not expected for method\t12.476489028213166\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t4.625\t10.9375\t7.836990595611286\t1.5673981191222568\tmodel not expected for method\t1.4420062695924765\t2.847324496809033\t5.625\n", "model not expected for method\tmodel not expected for method\t3.0094043887147337\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t3.9375\t4.0625\t1.2539184952978055\t0.6269592476489028\tmodel not expected for method\t1.128526645768025\t0.15368852459016394\t1.875\n", "model not expected for method\tmodel not expected for method\t2.884012539184953\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t3.875\t2.1875\t2.507836990595611\t0.6269592476489028\tmodel not expected for method\t1.0658307210031348\t0.20491803278688525\t1.875\n", "model not expected for method\tmodel not expected for method\t18.808777429467085\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t14.8125\t34.68013468013468\t39.86486486486486\t31.347962382445143\tmodel not expected for method\t12.539184952978054\t12.12733703890854\t19.375\n", "\n", "\n", "\n", "\n", "\n", "==================================================\n", "Functional Behavior Category: standard\n", "\n", "\n", "\n", "GCG\tGCG-Multi\tGCG-Transfer\tPEZ\tGBDA\tUAT\tAutoPrompt\tFewShot\tZeroShot\tPAIR\tTAP\tTAP-Transfer\tAutoDAN\tPAP-top5\tHumanJailbreaks\tDirectRequest\n", "32.075471698113205\t19.49685534591195\t15.932914046121594\t0.0\t0.0\t3.1446540880503147\t19.49685534591195\t3.1446540880503147\t0.37735849056603776\t6.918238993710692\t5.031446540880504\t3.7735849056603774\t0.0\t0.7547169811320755\t0.12578616352201258\t0.0\n", "27.67295597484277\t8.679245283018867\t13.137665967854648\t0.0\t0.37735849056603776\t0.0\t16.9811320754717\t3.7735849056603774\t0.5031446540880503\t14.465408805031446\t8.80503144654088\t3.7735849056603774\t0.0\t1.6352201257861636\t0.628930817610063\t0.628930817610063\n", "37.735849056603776\t5.660377358490567\t14.046121593291405\t0.0\t0.0\t0.0\t17.61006289308176\t2.515723270440252\t0.12578616352201258\t7.547169811320755\t8.176100628930817\t7.547169811320755\t0.628930817610063\t0.8805031446540881\t0.0\t0.0\n", "89.937106918239\t83.8993710691824\t83.08874912648497\t17.61006289308176\t16.855345911949684\t18.238993710691823\t76.10062893081762\t52.83018867924528\t26.540880503144653\t66.0377358490566\t67.9245283018868\t77.9874213836478\t89.30817610062893\t15.59748427672956\t46.666666666666664\t20.125786163522015\n", "84.90566037735849\t78.77358490566037\t71.76799440950384\t10.566037735849058\t7.672955974842767\t8.176100628930817\t50.943396226415096\t34.59119496855346\t18.49056603773585\t63.52201257861635\t69.81132075471697\t83.01886792452831\t83.01886792452831\t16.352201257861633\t47.16981132075472\t13.836477987421384\n", "81.13207547169812\t62.473794549266245\t63.52201257861635\t37.23270440251572\t32.57861635220126\t30.81761006289308\t61.63522012578616\t24.528301886792452\t26.41509433962264\t38.9937106918239\t65.40880503144653\t76.72955974842768\t72.95597484276729\t16.9811320754717\t31.69811320754717\t14.465408805031446\n", "86.16352201257862\t74.52830188679245\t57.51222921034241\t25.40880503144654\t23.52201257861635\t66.66666666666666\t76.72955974842768\t44.0251572327044\t20.88050314465409\t66.0377358490566\t70.44025157232704\t83.01886792452831\t90.56603773584906\t19.119496855345915\t36.22641509433962\t11.949685534591195\n", "79.24528301886792\t73.20754716981132\t48.2879105520615\t10.566037735849058\t9.685534591194969\t5.660377358490567\t66.66666666666666\t35.84905660377358\t9.433962264150944\t56.60377358490566\t69.81132075471697\t74.84276729559748\t61.63522012578616\t9.811320754716983\t28.427672955974842\t7.547169811320755\n", "82.38993710691824\t74.59119496855345\t45.98183088749126\t6.163522012578617\t7.29559748427673\t5.031446540880504\t55.9748427672956\t29.559748427672954\t7.79874213836478\t49.056603773584904\t56.60377358490566\t67.9245283018868\t62.264150943396224\t9.937106918238994\t32.20125786163522\t10.062893081761008\n", "exp not started\texp not started\t37.10691823899371\t6.289308176100629\t8.80503144654088\t2.515723270440252\texp not started\t30.81761006289308\t7.421383647798742\t54.088050314465406\t58.490566037735846\t66.66666666666666\t31.446540880503143\t14.59119496855346\t42.38993710691824\t7.547169811320755\n", "81.13207547169812\t78.9308176100629\t77.07896575821104\t59.874213836477985\t73.83647798742138\t71.69811320754717\t76.10062893081762\t58.490566037735846\t57.861635220125784\t64.77987421383648\t79.24528301886792\t75.47169811320755\t82.38993710691824\t18.867924528301888\t31.949685534591193\t48.42767295597484\n", "83.01886792452831\t78.74213836477988\t80.083857442348\t61.76100628930817\t71.19496855345912\t72.32704402515722\t81.13207547169812\t45.911949685534594\t46.540880503144656\t69.18238993710692\t77.35849056603774\t78.61635220125787\t86.79245283018868\t15.974842767295597\t40.125786163522015\t30.18867924528302\n", "56.60377358490566\t45.44025157232704\t80.85255066387141\t45.40880503144654\t43.144654088050316\t47.16981132075472\t41.509433962264154\t62.893081761006286\t50.56603773584906\t69.18238993710692\t74.21383647798741\t75.47169811320755\t96.85534591194968\t15.723270440251572\t50.691823899371066\t38.9937106918239\n", "56.60377358490566\t27.942497753818508\t62.3340321453529\t35.22012578616352\t33.33333333333333\t32.075471698113205\t29.559748427672954\t60.37735849056604\t48.17610062893082\t67.9245283018868\t77.35849056603774\t79.24528301886792\t93.08176100628931\t15.723270440251572\t53.333333333333336\t44.0251572327044\n", "74.84276729559748\t78.4366576819407\t74.63312368972747\t62.38993710691824\t62.38993710691824\t62.264150943396224\t71.69811320754717\t72.32704402515722\t67.0440251572327\t67.29559748427673\t81.76100628930818\t81.13207547169812\t91.82389937106919\t27.67295597484277\t74.21383647798741\t72.32704402515722\n", "85.53459119496856\t84.43396226415094\t84.0670859538784\t55.471698113207545\t60.62893081761006\t59.74842767295597\t77.9874213836478\t60.37735849056604\t45.0314465408805\t62.893081761006286\t77.9874213836478\t82.38993710691824\t92.45283018867924\t23.89937106918239\t71.0691823899371\t44.65408805031446\n", "no labels\t70.75471698113208\t78.54647099930119\tno labels\tno labels\texp not started\texp not started\t51.57232704402516\t33.58490566037736\t69.18238993710692\t83.64779874213836\t81.76100628930818\t88.67924528301887\t20.251572327044027\t61.50943396226415\t39.62264150943396\n", "84.27672955974843\t69.36208445642407\t78.12718378756115\t41.88679245283019\t51.320754716981135\t45.28301886792453\t67.29559748427673\t66.66666666666666\t46.0377358490566\t63.52201257861635\t80.50314465408806\t83.64779874213836\t97.48427672955975\t24.0251572327044\t63.64779874213836\t49.056603773584904\n", "88.0503144654088\t81.22192273135668\t74.49336128581412\t54.9685534591195\t70.81761006289308\t59.74842767295597\t79.24528301886792\t64.77987421383648\t58.867924528301884\t71.0691823899371\t86.79245283018868\t84.90566037735849\t96.22641509433963\t26.41509433962264\t76.35220125786164\t64.77987421383648\n", "88.67924528301887\t81.85085354896675\t78.54647099930119\t78.61635220125787\t80.37735849056604\t82.38993710691824\t78.61635220125787\t79.24528301886792\t81.13207547169812\t69.18238993710692\t84.27672955974843\t88.67924528301887\t96.85534591194968\t29.30817610062893\t82.8930817610063\t84.90566037735849\n", "0.0\t0.41928721174004197\t0.0\t0.12578616352201258\t0.0\t0.0\t0.0\t46.540880503144656\t0.628930817610063\t57.23270440251572\t78.61635220125787\t67.9245283018868\t8.80503144654088\t20.251572327044027\t5.283018867924529\t1.257861635220126\n", "model not expected for method\tmodel not expected for method\t44.27672955974843\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t20.251572327044027\t52.83018867924528\t54.71698113207547\t78.61635220125787\tmodel not expected for method\t10.566037735849058\t25.91194968553459\t16.352201257861633\n", "model not expected for method\tmodel not expected for method\t56.35220125786164\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t33.58490566037736\t42.138364779874216\t45.911949685534594\t60.37735849056604\tmodel not expected for method\t11.949685534591195\t3.033175355450237\t36.477987421383645\n", "model not expected for method\tmodel not expected for method\t14.59119496855346\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t11.446540880503145\t38.9937106918239\t45.28301886792453\t67.29559748427673\tmodel not expected for method\t11.949685534591195\t4.654088050314465\t10.69182389937107\n", "model not expected for method\tmodel not expected for method\t21.38364779874214\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t9.30817610062893\t41.509433962264154\t43.39622641509434\t81.76100628930818\tmodel not expected for method\t11.949685534591195\t1.3836477987421385\t6.918238993710692\n", "model not expected for method\tmodel not expected for method\t11.320754716981133\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t1.1320754716981132\t15.09433962264151\t8.176100628930817\t0.0\tmodel not expected for method\t0.628930817610063\t1.7061611374407581\t1.8867924528301887\n", "model not expected for method\tmodel not expected for method\t1.509433962264151\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t0.628930817610063\t1.8867924528301887\t1.257861635220126\t0.0\tmodel not expected for method\t0.12578616352201258\t0.0\t0.0\n", "model not expected for method\tmodel not expected for method\t1.3836477987421385\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t0.628930817610063\t2.515723270440252\t1.8867924528301887\t0.0\tmodel not expected for method\t0.12578616352201258\t0.09950248756218905\t0.0\n", "model not expected for method\tmodel not expected for method\t16.352201257861633\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t7.672955974842767\t32.88590604026846\t38.92617449664429\t30.81761006289308\tmodel not expected for method\t7.79874213836478\t11.204188481675391\t12.578616352201259\n", "\n", "\n", "\n", "\n", "\n", "==================================================\n", "Functional Behavior Category: contextual\n", "\n", "\n", "\n", "GCG\tGCG-Multi\tGCG-Transfer\tPEZ\tGBDA\tUAT\tAutoPrompt\tFewShot\tZeroShot\tPAIR\tTAP\tTAP-Transfer\tAutoDAN\tPAP-top5\tHumanJailbreaks\tDirectRequest\n", "60.49382716049383\t42.63157894736842\t43.05555555555556\t7.160493827160494\t4.938271604938271\t11.11111111111111\t24.691358024691358\t11.11111111111111\t7.654320987654321\t19.753086419753085\t24.691358024691358\t21.25\t0.0\t5.75\t2.5\t2.4691358024691357\n", "58.0246913580247\t21.323529411764707\t36.80555555555556\t6.172839506172839\t6.419753086419753\t4.938271604938271\t32.098765432098766\t13.580246913580247\t7.901234567901234\t19.753086419753085\t28.39506172839506\t15.0\t3.7037037037037033\t8.5\t4.25\t9.876543209876543\n", "67.90123456790124\t30.864197530864196\t50.0\t11.11111111111111\t8.641975308641975\t13.750000000000002\t38.2716049382716\t16.25\t11.358024691358025\t35.80246913580247\t27.160493827160494\t38.75\t6.172839506172839\t9.5\t6.75\t9.876543209876543\n", "82.71604938271605\t75.36585365853658\t75.55555555555556\t39.75308641975309\t41.48148148148148\t37.03703703703704\t72.8395061728395\t65.4320987654321\t52.8395061728395\t81.48148148148148\t70.0\t85.0\t85.18518518518519\t43.75\t61.5\t51.85185185185185\n", "87.65432098765432\t76.45259938837921\t71.38888888888889\t37.53086419753087\t35.30864197530864\t33.33333333333333\t65.4320987654321\t46.913580246913575\t44.93827160493827\t62.96296296296296\t66.25\t87.5\t88.88888888888889\t36.25\t61.0\t43.20987654320987\n", "83.9506172839506\t36.400817995910025\t56.99999999999999\t50.617283950617285\t48.39506172839506\t51.85185185185185\t64.19753086419753\t58.0246913580247\t54.567901234567906\t71.60493827160494\t72.5\t85.0\t66.66666666666666\t39.75\t46.0\t43.20987654320987\n", "72.8395061728395\t57.407407407407405\t62.916666666666664\t58.51851851851851\t55.80246913580247\t62.96296296296296\t59.25925925925925\t58.0246913580247\t53.58024691358024\t76.5432098765432\t71.25\t83.75\t55.00000000000001\t41.0\t49.5\t49.382716049382715\n", "80.55555555555556\t60.49382716049383\t53.47222222222222\t27.40740740740741\t27.160493827160494\t25.925925925925924\t61.53846153846154\t48.148148148148145\t40.24691358024691\t81.48148148148148\t70.37037037037037\t81.25\t64.19753086419753\t29.75\t39.5\t34.5679012345679\n", "84.21052631578947\t58.2716049382716\t60.55555555555555\t26.172839506172842\t26.41975308641975\t23.456790123456788\t71.21212121212122\t46.913580246913575\t37.53086419753087\t72.8395061728395\t67.90123456790124\t77.5\t72.8395061728395\t22.75\t48.0\t38.2716049382716\n", "exp not started\texp not started\t55.00000000000001\t27.160493827160494\t20.98765432098765\t24.691358024691358\texp not started\t45.67901234567901\t33.827160493827165\t58.0246913580247\t54.32098765432099\t75.0\t74.07407407407408\t31.5\t52.25\t28.39506172839506\n", "77.77777777777779\t60.550458715596335\t54.166666666666664\t47.40740740740741\t57.28395061728395\t55.55555555555556\t65.4320987654321\t50.617283950617285\t56.54320987654321\t69.1358024691358\t76.5432098765432\t78.75\t51.85185185185185\t39.25\t43.25\t55.55555555555556\n", "82.71604938271605\t70.73170731707317\t69.58333333333333\t61.23456790123457\t66.66666666666666\t66.66666666666666\t75.30864197530865\t61.72839506172839\t55.06172839506173\t70.37037037037037\t80.24691358024691\t82.5\t91.35802469135803\t33.5\t45.75\t48.148148148148145\n", "65.4320987654321\t62.69113149847095\t76.25\t58.0246913580247\t63.456790123456784\t60.49382716049383\t56.79012345679012\t59.25925925925925\t62.22222222222222\t88.88888888888889\t81.48148148148148\t91.25\t87.65432098765432\t40.5\t53.5\t72.8395061728395\n", "81.48148148148148\t62.41258741258741\t80.55555555555556\t70.37037037037037\t68.39506172839506\t72.8395061728395\t61.72839506172839\t75.30864197530865\t66.41975308641975\t80.24691358024691\t82.71604938271605\t93.75\t87.65432098765432\t44.75\t60.75000000000001\t85.18518518518519\n", "70.37037037037037\t83.04195804195804\t80.69444444444444\t82.71604938271605\t80.98765432098766\t76.5432098765432\t62.96296296296296\t70.37037037037037\t69.87654320987654\t77.77777777777779\t92.5925925925926\t93.75\t96.29629629629629\t54.75\t84.75\t82.71604938271605\n", "95.06172839506173\t85.18518518518519\t89.44444444444444\t84.93827160493828\t81.72839506172839\t83.9506172839506\t82.71604938271605\t72.8395061728395\t65.67901234567901\t82.71604938271605\t87.65432098765432\t92.5\t95.06172839506173\t52.0\t88.25\t86.41975308641975\n", "100.0\t86.41975308641975\t84.30555555555556\t0.0\t0.0\texp not started\texp not started\t76.5432098765432\t66.41975308641975\t78.75\t82.5\t93.75\t90.12345679012346\t48.75\t74.5\t81.48148148148148\n", "87.65432098765432\t71.5034965034965\t68.47222222222223\t62.71604938271606\t61.97530864197531\t67.90123456790124\t71.60493827160494\t72.8395061728395\t70.12345679012346\t77.77777777777779\t87.65432098765432\t91.25\t95.06172839506173\t47.5\t72.5\t75.30864197530865\n", "79.01234567901234\t78.28371278458845\t78.47222222222221\t77.53086419753087\t78.76543209876543\t82.71604938271605\t81.48148148148148\t82.71604938271605\t72.5925925925926\t82.5\t91.35802469135803\t91.25\t96.29629629629629\t61.25000000000001\t81.0\t87.65432098765432\n", "91.35802469135803\t77.71929824561403\t80.69444444444444\t80.49382716049382\t80.0\t75.30864197530865\t72.8395061728395\t81.48148148148148\t70.61728395061728\t86.41975308641975\t91.35802469135803\t95.0\t97.53086419753086\t59.75\t89.75\t87.65432098765432\n", "23.456790123456788\t19.34156378600823\t0.0\t10.37037037037037\t0.7407407407407408\t0.0\t19.753086419753085\t67.90123456790124\t26.666666666666668\t65.4320987654321\t76.5432098765432\t77.5\t45.67901234567901\t46.0\t35.75\t50.617283950617285\n", "model not expected for method\tmodel not expected for method\t56.75\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t43.20987654320987\t74.07407407407408\t73.75\t86.25\tmodel not expected for method\t26.25\t40.75\t45.67901234567901\n", "model not expected for method\tmodel not expected for method\t54.25\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t46.913580246913575\t60.49382716049383\t53.75\t68.75\tmodel not expected for method\t21.25\t5.326460481099656\t61.72839506172839\n", "model not expected for method\tmodel not expected for method\t47.75\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t41.23456790123457\t67.90123456790124\t71.25\t77.5\tmodel not expected for method\t29.5\t33.0\t51.85185185185185\n", "model not expected for method\tmodel not expected for method\t41.0\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t30.617283950617285\t44.44444444444444\t51.24999999999999\t61.25000000000001\tmodel not expected for method\t20.75\t7.000000000000001\t20.98765432098765\n", "model not expected for method\tmodel not expected for method\t25.75\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t16.049382716049383\t13.580246913580247\t13.750000000000002\t6.25\tmodel not expected for method\t4.5\t6.701030927835052\t17.28395061728395\n", "model not expected for method\tmodel not expected for method\t6.0\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t9.382716049382717\t6.172839506172839\t2.5\t0.0\tmodel not expected for method\t1.25\t0.0\t2.4691358024691357\n", "model not expected for method\tmodel not expected for method\t6.0\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t8.88888888888889\t2.4691358024691357\t5.0\t0.0\tmodel not expected for method\t1.0\t0.0\t2.4691358024691357\n", "model not expected for method\tmodel not expected for method\t33.0\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t21.728395061728396\t54.794520547945204\t59.72222222222222\t52.5\tmodel not expected for method\t16.5\t20.3125\t28.39506172839506\n", "\n", "\n", "\n", "\n", "\n", "==================================================\n", "Functional Behavior Category: copyright\n", "\n", "\n", "\n", "GCG\tGCG-Multi\tGCG-Transfer\tPEZ\tGBDA\tUAT\tAutoPrompt\tFewShot\tZeroShot\tPAIR\tTAP\tTAP-Transfer\tAutoDAN\tPAP-top5\tHumanJailbreaks\tDirectRequest\n", "2.5\t2.3214285714285716\t2.083333333333333\t0.0\t0.0\t0.0\t2.5\t2.5\t0.25\t3.75\t1.25\t2.5\t0.0\t3.5000000000000004\t0.0\t0.0\n", "7.5\t6.5\t3.194444444444444\t1.25\t2.25\t1.25\t5.0\t6.25\t2.5\t10.0\t10.0\t10.0\t0.0\t2.5\t1.7500000000000002\t1.25\n", "12.5\t1.25\t9.166666666666666\t1.25\t0.0\t3.75\t12.5\t10.0\t0.0\t6.25\t11.25\t8.75\t3.75\t6.0\t2.75\t2.5\n", "1.25\t0.25\t1.25\t1.0\t0.75\t0.0\t1.25\t2.5\t1.0\t1.25\t1.25\t0.0\t1.25\t1.7500000000000002\t1.0\t2.5\n", "5.0\t8.125\t5.972222222222222\t6.75\t7.249999999999999\t7.5\t7.5\t12.5\t9.75\t11.25\t8.75\t6.25\t8.75\t11.5\t6.75\t8.75\n", "2.5\t0.8333333333333334\t0.5\t2.25\t2.0\t1.25\t1.25\t1.25\t3.75\t2.5\t3.75\t0.0\t1.25\t2.75\t2.0\t1.25\n", "1.25\t3.125\t1.9444444444444444\t3.75\t3.0\t5.0\t5.0\t7.5\t7.249999999999999\t2.5\t6.25\t5.0\t5.0\t8.25\t5.0\t3.75\n", "1.25\t2.5\t1.6666666666666667\t2.5\t3.25\t2.5\t2.5\t5.0\t4.25\t3.75\t2.5\t1.25\t2.5\t2.75\t0.75\t2.5\n", "7.5\t8.5\t3.0555555555555554\t6.0\t6.75\t6.25\t3.75\t6.25\t13.25\t10.0\t11.25\t8.75\t10.0\t11.75\t5.75\t10.0\n", "exp not started\texp not started\t17.22222222222222\t15.0\t17.5\t15.0\texp not started\t21.25\t24.5\t20.0\t28.749999999999996\t17.5\t27.500000000000004\t24.75\t15.75\t25.0\n", "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n", "0.0\t0.0\t0.0\t0.0\t0.75\t0.0\t0.0\t0.0\t0.0\t1.25\t0.0\t0.0\t0.0\t0.5\t0.75\t0.0\n", "3.75\t2.5\t0.9722222222222222\t2.25\t1.25\t3.75\t2.5\t2.5\t1.0\t2.5\t1.25\t0.0\t2.5\t1.25\t1.7500000000000002\t3.75\n", "7.5\t2.3214285714285716\t2.083333333333333\t3.75\t2.75\t2.5\t8.75\t6.25\t6.25\t6.25\t6.25\t3.75\t3.75\t3.75\t2.5\t6.25\n", "6.25\t5.178571428571429\t4.722222222222222\t11.75\t10.0\t11.25\t10.0\t15.0\t16.25\t12.5\t11.25\t10.0\t7.5\t14.000000000000002\t8.75\t12.5\n", "10.0\t2.5\t1.3888888888888888\t6.75\t6.5\t8.75\t8.75\t5.0\t7.249999999999999\t5.0\t7.5\t6.25\t6.25\t6.75\t4.75\t7.5\n", "no labels\t27.500000000000004\t7.638888888888889\tno labels\tno labels\texp not started\texp not started\t25.0\t26.0\t27.500000000000004\t25.0\t18.75\t23.75\t25.0\t17.0\t27.500000000000004\n", "5.0\t5.535714285714286\t3.3333333333333335\t9.25\t9.25\t7.5\t13.750000000000002\t11.25\t11.75\t7.5\t8.75\t8.75\t7.5\t9.75\t6.25\t10.0\n", "6.25\t6.964285714285714\t8.472222222222223\t11.0\t11.0\t12.5\t8.75\t10.0\t10.75\t11.25\t10.0\t11.25\t10.0\t12.25\t9.5\t12.5\n", "8.75\t6.785714285714286\t6.111111111111111\t10.25\t11.75\t7.5\t8.75\t15.0\t12.0\t11.25\t11.25\t8.75\t11.25\t11.25\t10.75\t13.750000000000002\n", "1.25\t0.4166666666666667\t0.0\t0.25\t0.0\t0.0\t0.0\t11.25\t0.0\t11.25\t13.750000000000002\t6.25\t5.0\t12.5\t8.5\t6.25\n", "model not expected for method\tmodel not expected for method\t9.25\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t13.750000000000002\t11.25\t13.750000000000002\t8.75\tmodel not expected for method\t13.5\t6.25\t10.0\n", "model not expected for method\tmodel not expected for method\t3.75\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t0.75\t0.0\t10.0\t1.25\tmodel not expected for method\t0.25\t0.25\t0.0\n", "model not expected for method\tmodel not expected for method\t13.0\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t11.25\t11.25\t11.25\t11.25\tmodel not expected for method\t14.499999999999998\t6.0\t10.0\n", "model not expected for method\tmodel not expected for method\t5.5\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t1.25\t7.5\t12.5\t6.25\tmodel not expected for method\t1.7500000000000002\t0.75\t3.75\n", "model not expected for method\tmodel not expected for method\t1.5\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t0.0\t0.0\t1.25\t0.0\tmodel not expected for method\t0.0\t0.25\t1.25\n", "model not expected for method\tmodel not expected for method\t3.0\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t5.0\t6.25\t0.0\t2.5\tmodel not expected for method\t3.0\t0.75\t5.0\n", "model not expected for method\tmodel not expected for method\t2.75\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t5.25\t1.25\t1.25\t2.5\tmodel not expected for method\t3.0\t0.75\t5.0\n", "model not expected for method\tmodel not expected for method\t9.5\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\tmodel not expected for method\t22.0\t18.666666666666668\t22.666666666666664\t11.25\tmodel not expected for method\t18.0\t5.6640625\t23.75\n", "\n", "\n", "\n", "\n", "\n" ] } ], "source": [ "# mode = 'individual_columns'\n", "mode = 'table'\n", "\n", "for functional_behavior_category in ['all', 'standard', 'contextual', 'copyright']:\n", " print('='*50)\n", " print('Functional Behavior Category:', functional_behavior_category)\n", "\n", " table_columns = ['GCG', 'GCG-Multi', 'GCG-Transfer', 'PEZ', 'GBDA', 'UAT', 'AutoPrompt', 'FewShot', 'ZeroShot', 'PAIR', 'TAP', 'TAP-Transfer', 'AutoDAN', 'PAP-top5', 'HumanJailbreaks', 'DirectRequest']\n", " table_rows = ['llama2_7b', 'llama2_13b', 'llama2_70b', 'vicuna_7b_v1_5', 'vicuna_13b_v1_5', 'baichuan2_7b', 'baichuan2_13b', 'qwen_7b_chat', 'qwen_14b_chat', 'qwen_72b_chat', 'koala_7b', 'koala_13b', 'orca_2_7b', 'orca_2_13b', 'solar_10_7b_instruct', 'mistral_7b_v2', 'mixtral_8x7b', 'openchat_3_5_1210', 'starling_7b', 'zephyr_7b', 'zephyr_7b_robust6_data2', 'gpt-3.5-turbo-0613', 'gpt-3.5-turbo-1106', 'gpt-4-0613', 'gpt-4-1106-preview', 'claude-instant-1', 'claude-2', 'claude-2.1', 'gemini']\n", "\n", " result_list = {row: [] for row in table_rows}\n", "\n", " for proper_method_name in table_columns:\n", " expanded_pipeline_config = expanded_pipeline_configs[proper_method_name]\n", " method = expanded_pipeline_config['class_name']\n", " if mode == 'individual_columns':\n", " print()\n", " print(proper_method_name)\n", "\n", " for model in table_rows:\n", " if model not in expanded_pipeline_config['model_to_experiment']:\n", " if mode == 'individual_columns':\n", " print('model not expected for method')\n", " result_list[model].append('model not expected for method')\n", " continue\n", " \n", " experiment_name = expanded_pipeline_config['model_to_experiment'][model]\n", "\n", " if method not in all_results:\n", " if mode == 'individual_columns':\n", " print('method not started')\n", " result_list[model].append('method not started')\n", " continue\n", "\n", " experiment = all_results[method].get(experiment_name)\n", " if experiment is None:\n", " if mode == 'individual_columns':\n", " print('exp not started')\n", " result_list[model].append('exp not started')\n", " continue\n", "\n", " if experiment['status'] != 'finished':\n", " if experiment['status'] != 'missing behaviors':\n", " if mode == 'individual_columns':\n", " print(experiment['status'])\n", " result_list[model].append(experiment['status'])\n", " continue\n", "\n", " results = experiment['results'].get(model)\n", " if results is None:\n", " if mode == 'individual_columns':\n", " print('missing results')\n", " result_list[model].append('missing results')\n", " continue\n", " \n", " # compute attack success rate\n", " all_labels = []\n", " for behavior_id in results:\n", " if behavior_id not in behaviors:\n", " continue\n", " if (behaviors[behavior_id]['FunctionalCategory'] != functional_behavior_category) and (functional_behavior_category != 'all'):\n", " continue\n", " \n", " for i in range(len(results[behavior_id])):\n", " label = results[behavior_id][i]['label']\n", " all_labels.append(label == True)\n", " \n", " if len(all_labels) == 0:\n", " if mode == 'individual_columns':\n", " print('no labels')\n", " result_list[model].append('no labels')\n", " continue\n", " \n", " asr = sum(all_labels) / len(all_labels)\n", " \n", " if mode == 'individual_columns':\n", " print(f'{100 * asr}')\n", " result_list[model].append(100 * asr)\n", " # print(f'{method} {experiment_name} {model} ASR: {100 * asr:.1f}')\n", "\n", " if mode == 'table':\n", " print('\\n\\n')\n", " print('\\t'.join(table_columns))\n", " for model in table_rows:\n", " print('\\t'.join([str(x) for x in result_list[model]]))\n", " \n", " print('\\n\\n\\n\\n')" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.13" } }, "nbformat": 4, "nbformat_minor": 2 }