{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Lecture 8: Language Model Fine-tuning\n", "\n", "Lecture 8 | CMU ANLP Fall 2025 | Instructor: Sean Welleck\n", "\n", "\n", "This notebook demonstrates fine-tuning a language model for a generation task.\n", "\n", "Task: Given a name, generate its reverse (e.g., emma → amme)" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Total names: 32033\n" ] }, { "data": { "text/plain": [ "['paityn',\n", " 'evalyn',\n", " 'luz',\n", " 'nathalia',\n", " 'winnie',\n", " 'chandler',\n", " 'ciara',\n", " 'danica',\n", " 'nailah',\n", " 'rilynn']" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "data = open('names.txt').read().splitlines()\n", "print(f\"Total names: {len(data)}\")\n", "data[1000:1010]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Load the model and tokenizer" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/miniconda3/envs/anlp/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n", "The new embeddings will be initialized from a multivariate normal distribution that has old embeddings' mean and covariance. As described in this article: https://nlp.stanford.edu/~johnhew/vocab-expansion.html. To disable this, use `mean_resizing=False`\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Model parameters: 134,516,160\n" ] } ], "source": [ "import transformers\n", "import torch\n", "from transformers import AutoTokenizer, AutoModelForCausalLM\n", "\n", "model_name = \"HuggingFaceTB/SmolLM2-135M\"\n", "\n", "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", "model = AutoModelForCausalLM.from_pretrained(model_name)\n", "\n", "# Add special tokens\n", "tokenizer.add_special_tokens({\n", " \"pad_token\": \"<|pad|>\",\n", " \"bos_token\": \"<|startoftext|>\",\n", "})\n", "\n", "# Resize embeddings to account for new tokens\n", "model.resize_token_embeddings(len(tokenizer))\n", "\n", "print(f\"Model parameters: {sum(p.numel() for p in model.parameters()):,}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Test the model before fine-tuning" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Before fine-tuning:\n", "==================================================\n", "Reverse the name: emma. Answer: 0.4812529443310765.\n", "Reverse the name: noah. Answer: it is only a person of that name\n", "000B. Boole, 1\n", "Reverse the name: olivia. Answer: a. The person is named Olivia. b. The person is named Olivia. c.\n" ] } ], "source": [ "test_names = ['emma', 'noah', 'olivia']\n", "\n", "print(\"Before fine-tuning:\")\n", "print(\"=\"*50)\n", "\n", "for name in test_names:\n", " prompt = f\"<|startoftext|>Reverse the name: {name}. Answer:\"\n", " inputs = tokenizer(prompt, return_tensors=\"pt\")\n", " \n", " with torch.no_grad():\n", " outputs = model.generate(\n", " inputs[\"input_ids\"],\n", " max_new_tokens=20,\n", " temperature=1.0,\n", " do_sample=True,\n", " pad_token_id=tokenizer.pad_token_id,\n", " eos_token_id=tokenizer.eos_token_id\n", " )\n", " \n", " response = tokenizer.decode(outputs[0], skip_special_tokens=True)\n", " print(f\"{response}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Create dataset \n", "\n", "Format: `<|startoftext|>Reverse the name: emma. Answer: amme<|endoftext|>`" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from torch.utils.data import Dataset, DataLoader\n", "import random\n", "\n", "class NameReversalDataset(Dataset):\n", " def __init__(self, names, tokenizer, max_length=64):\n", " self.names = names\n", " self.tokenizer = tokenizer\n", " self.max_length = max_length\n", " \n", " def __len__(self):\n", " return len(self.names)\n", " \n", " def __getitem__(self, idx):\n", " name = self.names[idx]\n", " reversed_name = name[::-1]\n", " full_text = f\"{self.tokenizer.bos_token}Reverse the name: {name}. Answer: {reversed_name}{self.tokenizer.eos_token}\"\n", " prompt = f\"{self.tokenizer.bos_token}Reverse the name: {name}. Answer:\"\n", " \n", " return {\n", " 'full_text': full_text,\n", " 'prompt': prompt,\n", " 'name': name,\n", " 'reversed': reversed_name\n", " }\n", " \n", " def collate_fn(self, batch):\n", " full_texts = [item['full_text'] for item in batch]\n", " prompts = [item['prompt'] for item in batch]\n", " \n", " tokenized = self.tokenizer(\n", " full_texts,\n", " truncation=True,\n", " padding=True,\n", " max_length=self.max_length,\n", " return_tensors=\"pt\"\n", " )\n", " labels = tokenized['input_ids'].clone()\n", " \n", " # Mask the prompt tokens in the labels (set to -100 so they're ignored in loss)\n", " for i, prompt in enumerate(prompts):\n", " prompt_tokens = self.tokenizer(prompt, add_special_tokens=False)['input_ids']\n", " prompt_len = len(prompt_tokens)\n", " labels[i, :prompt_len] = -100 # Ignore prompt tokens in loss calculation\n", " \n", " return {\n", " 'input_ids': tokenized['input_ids'],\n", " 'attention_mask': tokenized['attention_mask'],\n", " 'labels': labels\n", " }\n", "\n", "# Split data\n", "random.seed(123)\n", "random.shuffle(data)\n", "\n", "n1 = int(0.8 * len(data))\n", "n2 = int(0.9 * len(data))\n", "\n", "train_data = data[:n1]\n", "dev_data = data[n1:n2]\n", "test_data = data[n2:]\n", "\n", "print(f\"Train: {len(train_data)}, Dev: {len(dev_data)}, Test: {len(test_data)}\")\n", "\n", "# Create datasets\n", "train_dataset = NameReversalDataset(train_data, tokenizer)\n", "dev_dataset = NameReversalDataset(dev_data, tokenizer)\n", "test_dataset = NameReversalDataset(test_data, tokenizer)\n", "\n", "# Create dataloaders\n", "batch_size = 16\n", "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=train_dataset.collate_fn)\n", "dev_loader = DataLoader(dev_dataset, batch_size=batch_size, shuffle=False, collate_fn=dev_dataset.collate_fn)\n", "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=test_dataset.collate_fn)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Examine a batch" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Sample batch shape:\n", " input_ids: torch.Size([16, 18])\n", " attention_mask: torch.Size([16, 18])\n", " labels: torch.Size([16, 18])\n", "\n", "First example:\n", "Input: Reverse the name: giavonna. Answer: annovaig\n", "Target (answer only): annovaig\n" ] } ], "source": [ "# Look at a sample batch\n", "sample_batch = next(iter(train_loader))\n", "print(\"Sample batch shape:\")\n", "print(f\" input_ids: {sample_batch['input_ids'].shape}\")\n", "print(f\" attention_mask: {sample_batch['attention_mask'].shape}\")\n", "print(f\" labels: {sample_batch['labels'].shape}\")\n", "\n", "# Decode first example\n", "print(\"\\nFirst example:\")\n", "input_text = tokenizer.decode(sample_batch['input_ids'][0], skip_special_tokens=True)\n", "print(f\"Input: {input_text}\")\n", "\n", "# Show which tokens are masked in labels\n", "label_tokens = sample_batch['labels'][0]\n", "valid_label_indices = (label_tokens != -100).nonzero(as_tuple=True)[0]\n", "if len(valid_label_indices) > 0:\n", " valid_labels = label_tokens[valid_label_indices]\n", " label_text = tokenizer.decode(valid_labels, skip_special_tokens=True)\n", " print(f\"Target (answer only): {label_text}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Training loop" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", "To disable this warning, you can either:\n", "\t- Avoid using `tokenizers` before the fork if possible\n", "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "Epoch 1/3: 100%|██████████| 1602/1602 [17:15<00:00, 1.55it/s, loss=0.8221]\n", "Validation: 100%|██████████| 201/201 [00:24<00:00, 8.14it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Epoch 1/3\n", " Average training loss: 0.8217\n", " Average validation loss: 0.3174\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Epoch 2/3: 100%|██████████| 1602/1602 [15:28<00:00, 1.72it/s, loss=0.1851]\n", "Validation: 100%|██████████| 201/201 [00:24<00:00, 8.18it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Epoch 2/3\n", " Average training loss: 0.1851\n", " Average validation loss: 0.2409\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Epoch 3/3: 100%|██████████| 1602/1602 [14:56<00:00, 1.79it/s, loss=0.0809]\n", "Validation: 100%|██████████| 201/201 [00:24<00:00, 8.33it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Epoch 3/3\n", " Average training loss: 0.0809\n", " Average validation loss: 0.2336\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "import torch.optim as optim\n", "from tqdm import tqdm\n", "\n", "learning_rate = 5e-5\n", "num_epochs = 3\n", "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", "\n", "model = model.to(device)\n", "optimizer = optim.AdamW(model.parameters(), lr=learning_rate)\n", "\n", "# Learning rate scheduler\n", "total_steps = len(train_loader) * num_epochs\n", "scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=total_steps)\n", "\n", "# Training loop\n", "for epoch in range(num_epochs):\n", " model.train()\n", " total_loss = 0\n", " progress_bar = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{num_epochs}\")\n", " \n", " for batch_idx, batch in enumerate(progress_bar):\n", " input_ids = batch['input_ids'].to(device)\n", " attention_mask = batch['attention_mask'].to(device)\n", " labels = batch['labels'].to(device)\n", " \n", " optimizer.zero_grad()\n", " \n", " outputs = model(\n", " input_ids=input_ids,\n", " attention_mask=attention_mask,\n", " labels=labels\n", " )\n", " \n", " loss = outputs.loss\n", " loss.backward()\n", " \n", " # Gradient clipping\n", " torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n", " \n", " optimizer.step()\n", " scheduler.step()\n", " \n", " total_loss += loss.item()\n", " \n", " # Update progress bar\n", " if batch_idx % 10 == 0:\n", " avg_loss = total_loss / (batch_idx + 1)\n", " progress_bar.set_postfix({'loss': f'{avg_loss:.4f}'})\n", " \n", " avg_train_loss = total_loss / len(train_loader)\n", " \n", " # Validation\n", " model.eval()\n", " total_val_loss = 0\n", " \n", " with torch.no_grad():\n", " for batch in tqdm(dev_loader, desc=\"Validation\"):\n", " input_ids = batch['input_ids'].to(device)\n", " attention_mask = batch['attention_mask'].to(device)\n", " labels = batch['labels'].to(device)\n", " \n", " outputs = model(\n", " input_ids=input_ids,\n", " attention_mask=attention_mask,\n", " labels=labels\n", " )\n", " \n", " total_val_loss += outputs.loss.item()\n", " \n", " avg_val_loss = total_val_loss / len(dev_loader)\n", " \n", " print(f\"Epoch {epoch+1}/{num_epochs}\")\n", " print(f\" Average training loss: {avg_train_loss:.4f}\")\n", " print(f\" Average validation loss: {avg_val_loss:.4f}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Generate and evaluate" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "After fine-tuning:\n", "==================================================\n", "emma → amme (expected: amme ) ✓\n", "noah → haon (expected: haon ) ✓\n", "olivia → aivilo (expected: aivilo ) ✓\n", "liam → mail (expected: mail ) ✓\n", "sophia → aihpos (expected: aihpos ) ✓\n", "mason → nosam (expected: nosam ) ✓\n", "isabella → allebasi (expected: allebasi ) ✓\n", "william → luottiw (expected: mailliw ) ✗\n", "mia → aim (expected: aim ) ✓\n", "james → semaj (expected: semaj ) ✓\n", "\n", "Accuracy: 9/10 = 90.0%\n" ] } ], "source": [ "def generate_reversal(model, tokenizer, name, device, max_new_tokens=20):\n", " prompt = f\"{tokenizer.bos_token}Reverse the name: {name}. Answer:\"\n", " \n", " inputs = tokenizer(prompt, return_tensors=\"pt\").to(device)\n", " \n", " model.eval()\n", " with torch.no_grad():\n", " outputs = model.generate(\n", " inputs[\"input_ids\"],\n", " max_new_tokens=max_new_tokens,\n", " temperature=0.1,\n", " do_sample=True,\n", " pad_token_id=tokenizer.pad_token_id,\n", " eos_token_id=tokenizer.eos_token_id\n", " )\n", " \n", " # Decode and extract the answer\n", " full_response = tokenizer.decode(outputs[0], skip_special_tokens=True)\n", " \n", " # Extract just the answer part\n", " if \"Answer:\" in full_response:\n", " answer = full_response.split(\"Answer:\")[1].strip()\n", " # Remove any extra text after the first word (the reversed name)\n", " answer = answer.split()[0] if answer else \"\"\n", " return answer\n", " return \"\"\n", "\n", "# Test on some examples\n", "print(\"After fine-tuning:\")\n", "print(\"=\"*50)\n", "test_names = ['emma', 'noah', 'olivia', 'liam', 'sophia', 'mason', 'isabella', 'william', 'mia', 'james']\n", "\n", "correct = 0\n", "for name in test_names:\n", " generated = generate_reversal(model, tokenizer, name, device)\n", " expected = name[::-1]\n", " is_correct = generated == expected\n", " correct += is_correct\n", " \n", " symbol = \"✓\" if is_correct else \"✗\"\n", " print(f\"{name:10} → {generated:10} (expected: {expected:10}) {symbol}\")\n", "\n", "print(f\"\\nAccuracy: {correct}/{len(test_names)} = {correct/len(test_names)*100:.1f}%\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Evaluate on test set" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Evaluating: 100%|██████████| 200/200 [00:13<00:00, 14.31it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Exact Match Accuracy: 81.00%\n", "\n", "Number of errors: 38/200\n", "\n", "Some examples of errors:\n", "Name\t\tExpected\tGenerated\n", "=============================================\n", "fizza \tazzif \tasirof\n", "dayra \taryad \tayrad\n", "scout \ttuocs \tkcarts\n", "meilah \thaliem \thalime\n", "ayaanreddy \tyddernaaya \tydirdaanya\n", "lejla \taljel \taljeel\n", "monserratt \tttarresnom \tttarrsecm\n", "jahnae \teanhaj \teahnaj\n", "tsion \tnoist \tnoits\n", "harmony \tynomrah \tytinohsm\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "def evaluate_accuracy(model, tokenizer, test_names, device, sample_size=None):\n", " \"\"\"Evaluate exact match accuracy on test set.\"\"\"\n", " if sample_size:\n", " test_names = random.sample(test_names, min(sample_size, len(test_names)))\n", " \n", " correct = 0\n", " errors = []\n", " \n", " for name in tqdm(test_names, desc=\"Evaluating\"):\n", " generated = generate_reversal(model, tokenizer, name, device)\n", " expected = name[::-1]\n", " \n", " if generated == expected:\n", " correct += 1\n", " else:\n", " errors.append((name, expected, generated))\n", " \n", " exact_match_acc = correct / len(test_names)\n", " \n", " return {\n", " 'exact_match_accuracy': exact_match_acc,\n", " 'errors': errors\n", " }\n", "\n", "# Evaluate on a sample of test data\n", "results = evaluate_accuracy(model, tokenizer, test_data, device, sample_size=200)\n", "\n", "print(f\"Exact Match Accuracy: {results['exact_match_accuracy']*100:.2f}%\")\n", "print(f\"\\nNumber of errors: {len(results['errors'])}/200\")\n", "\n", "# Show some errors\n", "if results['errors']:\n", " print(\"\\nSome examples of errors:\")\n", " print(\"Name\\t\\tExpected\\tGenerated\")\n", " print(\"=\"*45)\n", " for name, expected, generated in results['errors'][:10]:\n", " print(f\"{name:12}\\t{expected:12}\\t{generated}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Analysis by name length" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Evaluating: 100%|██████████| 6/6 [00:00<00:00, 16.62it/s]\n", "Evaluating: 100%|██████████| 20/20 [00:01<00:00, 16.81it/s]\n", "Evaluating: 100%|██████████| 20/20 [00:01<00:00, 14.84it/s]\n", "Evaluating: 100%|██████████| 20/20 [00:01<00:00, 14.00it/s]\n", "Evaluating: 100%|██████████| 20/20 [00:01<00:00, 11.28it/s]\n", "Evaluating: 100%|██████████| 20/20 [00:01<00:00, 12.11it/s]\n", "Evaluating: 100%|██████████| 20/20 [00:02<00:00, 9.91it/s]\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAA04AAAIjCAYAAAA0vUuxAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjYsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvq6yFwwAAAAlwSFlzAAAPYQAAD2EBqD+naQAAVNZJREFUeJzt3Qd0VFX7/v07hF6kt+RBqpRIFaQjKggC0qSDEIqAVCkWwoMgIB0FKYIgTaRLUR8UBKSIht4Fkd6rQEINbd517/9v5p00TgKTzEzy/aw1JufMmcmehueavfe9fWw2m00AAAAAANFKEv1VAAAAAABFcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIAAAAACwQnAAAAALBAcAIALzZ79mzx8fGRkydPurspCd6nn35qnuurV6+6uymIZ6+++qoULVrU3c0A4GYEJwBeGxbsl6RJk4q/v7+0bdtWzp075+7mebyPPvrIPG/NmjVzd1MQzUm6vj5169aNdJ0GZL1u7Nix4q2f2x07dognOn/+vAnHe/bscXdTAHiopO5uAAA8rSFDhkjevHnl3r17smXLFnNitnnzZjlw4ICkTJnS3c3zSDabTRYsWCB58uSRn376SW7evCnp0qVzd7MQhf/973+yc+dOKV26tLubkihocBo8eLD5bJQsWdLdzQHggehxAuC1atWqJe+88468++678s0338gHH3wgx44dkx9//FE8kQa8x48fu7UNGzZskLNnz8rMmTPl4cOHsmzZMvFUd+7ckcTq+eefl4wZM5oTeQCAZyA4AUgwqlSpYn5qeHL2999/S+PGjSVTpkymJ6pMmTLhwpUOHdIhRHPmzIl0n6tXrzbX6bf/djocsH379pI9e3ZJkSKFvPjiiyaIRAwoeruFCxfKgAEDzFDC1KlTS2hoqDx48MCcEL/wwgumPZkzZ5bKlSvLmjVrHLfft2+fGXqYL18+c0yOHDnM3/z333+f6TmaN2+eBAQEyGuvvSbVq1c321HRx9ihQwfx8/Mzj1F79rp06SL37993HHPjxg3p3bu3+YZej/nPf/4jbdq0ccwBim7+lf250Z8R55BoD8srr7xinqv+/fub63744QepU6eOoy358+eXoUOHyqNHjyK1e+vWrVK7dm0TOtKkSSPFixeXL7/80lw3a9Ys83d3794d6XbDhw8XX1/fGA311MfXtGlTee6558xr9/7775tQbFe1alUpUaJElLctVKiQ1KxZ0/JvaC+gPrfaK7hr164nHnvt2jXzpUGxYsUkbdq0pl36pcLevXujfN4XL15s3n/6ntS/o5+NkJAQCQsLk169ekm2bNnM/bRr187si+i7774zvWCpUqUyn6nmzZvLmTNnxFVi8/nSxzJs2DDz3tPPSbVq1eTo0aOR7nPy5Mnms6RtLlu2rPz+++/mPacX+/29/PLL5nd93PZhwPoednbw4EHz2dH3pz5/o0ePdtnjBuD5GKoHIMGwn6DrSbPdX3/9JZUqVTInOf369TMn03qy1aBBA1m6dKk0bNjQBCk9qdL9gYGB4e5z0aJF5v7sJ7uXLl2S8uXLm5Oq7t27S9asWeWXX34xIUNDkZ54OtMT/OTJk5sTWz0J1d91HsWIESNMT5mexOntNLzpCfIbb7xhbqch6vjx4+YkTkOTPo5p06aZnzosUf9+bOnf18fct29fs92iRQtz/xcvXjR/w3nIkrZLg1GnTp2kcOHC5mT2+++/N71A+hhu3bplguqhQ4fMSe5LL71kAoUGUu3RypIlS6zbp6FQT/j1RFx7EvXEWenJq57I9+nTx/z87bffZODAgeZ5GzNmjOP2+py99dZbkjNnThNm9DFp+zT06rYGhG7dupmwWKpUqXB/W/fpSbS+T6xoaNKwqK+hvhYTJkyQ69evy7fffmuub926tXTs2NEMGXUuKLB9+3b5559/TJCOCW3zuHHjzPvlSb2o+j5ZsWKFNGnSxARcfY9+/fXXJsDpib4GTmfabg0Q+nnQkDFx4kRJliyZJEmSxDwO/Xv2oa96f/pc22lI+eSTT8xzoO/fK1eumNtr2NVAmiFDBnkWsf18jRw50rRbP18a/jTItGrVygRouylTppj70verhlH9d0I///q51sClihQpYob+6mPV97z9S5iKFSs67kefmzfffFPefvtt8/j18/Dxxx+bwKrvWwCJgA0AvMysWbNs+s/X2rVrbVeuXLGdOXPG9v3339uyZs1qS5Eihdm2q1atmq1YsWK2e/fuOfY9fvzYVrFiRdsLL7zg2BcUFGRLliyZ7dq1a459YWFhtgwZMtjat2/v2NehQwdbzpw5bVevXg3XpubNm9vSp09vu3Pnjtlev369aWO+fPkc++xKlChhq1OnzhMfY8TbqAULFpj73LRpU6Tn4sSJE5bPmz5HeuyRI0fMdmhoqC1lypS2cePGhTuuTZs2tiRJkti2b98e6T70uVMDBw4097Vs2bJoj4mubfbnRn/aVa1a1eybOnVqjJ6Lzp0721KnTu14XR8+fGjLmzevLXfu3Lbr169H2R7VokULm5+fn+3Ro0eOfbt27TJ/W9v7JIMGDTLH1atXL9z+rl27mv179+412zdu3DDP68cffxzuuJ49e9rSpElju3Xr1hP/jj4XL774ovl98ODB5r537txptvW51O0xY8Y4jtfnwPnx2I/Tz8KQIUMiPe9Fixa13b9/P9xz4uPjY6tVq1a4+6hQoYJ5Pu1Onjxp8/X1tQ0bNizccfv377clTZo00v6I7O+HqN5XT/v5KlKkiPmc2n355Zdmv7ZJ6XWZM2e2vfzyy7YHDx44jps9e7Y5Tp9rO21XdO8D+/vz22+/dezT+86RI4etUaNGT3zcABIOhuoB8Fo61Ey/kc6VK5fpTdDeJP1m3v4tsg5h0t4J/XZYiyBoj4hetGdDe5COHDniGJqlFeZ0CJ3znJ9ff/3V9LrYq89pYQXtsdFqZ/q7/f70oven33hHHFalPVj67b4z/VZee47070fH+TY6DEz/hn4Tr6yGbkVHe1W0d61AgQJmW4dp6RA45+F6OgdLey/0MeqxEdl7uvR50OFo2mMX3TGxpcOytAfsSc+F/XXUHgHt/dJhmEp7O06cOGF6JCL2eji3R4cSao/a+vXrHfv08evfaNSoUYzaqb1Wznr06GF+/vzzz+Zn+vTppX79+qYIh75PlA4r1N5L7enQ92lMaa+T1Vwnfd6018X+d/T9rT1zOiwwqveKPgfaw2RXrlw5007tOXSm+3UIns6FU/rZ0PeHfp6c3/vas6fDTp2f06fxNJ8vfb9oD6idvadIe+GU9uTq86E9gFp90057pZx7pmNCn1PtCbXTv6s9s/a/BSDhIzgB8Fo6b0GHZ+mQGZ3XoidYehJpp8OQ9ARMhxZpwHK+DBo0yBxz+fJl81NDgA5J05NbO/1dh5y9/vrrZluHJWmQ0iFzEe/PfsJvvz87HeoUkQ4J0vspWLCgGebz4YcfmjlNzjT06UmzDlfTk3r9G/b70hPI2NK/pyf2OnxLnxf7RYcx6smlDiGzP0YdEmW1Zo3OI3P1ujY6TM75JNhOQ6YGNA0kOn9Hnwv7Caz9ubDPa7Nqkw6F1KF89rCoQUADjgadmFYX1JDgTOdcaXBxnsul4eT06dNmLo1au3atGYamw/hiQx+zhkH9QiCquVn2x6BD+rRd+v7X96w+R/qeiuq9ooUnIv4NpV9ARNyv922/Dw36+nnSvxPx/a9DIiO+92PraT5fER+LPQzpsDp16tQp89P+ZYGdhigdbhkb+oVMxC8F9O/Z/xaAhI85TgC8ln7ba+8V0W/ytcBCy5Yt5fDhw+bbYXsFO53/EN2EfOcTKu1Z0jkcGsD0JFpPVnUekP2bavv96Ul7xLlQdlqMwFnE3ial80H0RF+LHmivllYE1BPfqVOnmnkjSr/V//PPP02o0tLI9sejcyyepjLfkiVLzBynzz//3Fwi0iDh6gpu0fU8RVXUIbrnSk+kNexpYNLAqSFFiwBoz4POL4ntc6EFIPQ9Mn36dPnqq6/kjz/+MD1Qzj0Jrnic+n7T0KuFFPT11p/aM6O9pLFln+ukr8/48eOjLGyhXw5oj5HOqdOCDRrkNHBF9fzocxCV6Pbbe830vvSx6pyjqI7V9+izeJrPl1WbXSk+/xYAz0RwApAg6EmNTnrXileTJk0yE9+14IPSYUkxOWHV4KQnpzpcSE96tedFCxXY6TffGqj0xP9pToCd6cmtfouuFy20oCfXOilfg5N+g71u3TrTFueJ+U8a2mdFg5H2xth72pxpIYH58+ebv6ePUUOKFjZ4Eg0wVsfYv/3X8OPM3gsQE1rtTIda6TAxfY7sdFhexPYobZPVa6O9QRoetWKdhgB9zDGpdOf8Ojj3JGrPnZ70O/dg2AOaFlgYNWqUGf6ow8WiO/mOSa+Tvj+iChTa46rv+xkzZoTbr8/70xTpiI4+xxoS9LFrb6mrufLzZZc7d27Ha6TPkZ0OP9QeQucg9rRDTAEkHgzVA5BgaFU07YXSb+V1XpCWVdZ9GgwuXLgQ5dAgZ1pZS4fO6RA9veiQLueTdT3p1XkwGqyiCg0R7y86EUuK6zf12vNlL/1sP7mO+E12VL0NMaHzVDZt2mR6sXQuWMSLhjc9sdRKZNpTob13Gip0CF9E9jbp86DlrpcvXx7tMfYwo3/bTk+KdShWTEX1XGhJdO0tcqZV/fSEXp+jiEEt4vOoJ8t60Z4+fS01HDvPf4nJEFFnWlVORayspsPyNAR37tzZhONn6dWyz93SXreonqOIj1F7GGNSWj02tJqc/i0N2BH/nm4/a6l8V32+nGmPtJaM1x5G+1wt+xcJEYfY2eeeRXz/AIAdPU4AEhQd2qZlmfWb/vfee8+c5OoQPg1E+o2/9kLpXJPg4GBTNjviWjfa66S9PDocTEsg2yfdO5c/1knwOnFe70/XRNL5SDp0TOex6O9W9DYa6HQtHO150oCivQZaMllpj48GNi2trAUrdO6PDumL2MsSU9qbpCe29erVi/J6nR+mwUFPJvVx6dAv/Xs6RE5LM2ug1OCpJ+ObN282J/D6PGub9bnWIWL6WPSx6/BGHXKoc8Z0/R0taBEUFGSu08eq61o5n8Ba0XLQ2nOlPS09e/Y0vQJz586NdOKur5OWndbCAjq0UcOgBl8tHqFzpHQ9roi9TjqEU8U20OjroM+lDpvU95EOw9PepYhrN2nJc+3l0+dNn0MNd09Le510yF5Uwym1BLsGKn3M+nzt37/fvJb2HldX0SD82WefmdfTXtJbe4j0+dAAre8V+3P6JLom06pVqyLt18fnis+XM3v5fy3goXMV9csDbbv++6CPx7mXSbf1va3vX31cGqS0HVHNUwSQSLm7rB8AxNaTyhprWeb8+fObi5aoVseOHTMltrV0sJYc9/f3t7311lumPHdEWqpb71svmzdvjvLvX7p0ydatWzdbrly5zP3p/WrZ82nTpjmOsZdLXrJkSaTbf/bZZ7ayZcuaUuepUqWyFS5c2JRydi4RffbsWVvDhg3NMVqGuUmTJrbz58+b+9Sy2LEpR67l2J9//vknPqevvvqqLVu2bI6SzadOnTLPmb3Eu5ZV18fsXPr533//tXXv3t08n8mTJ7f95z//sQUGBoYrJa3PffXq1c19ZM+e3da/f3/bmjVroixHbi/BHdEff/xhK1++vHmutJT4Rx99ZFu9enWk+1D6mr3xxhu2dOnSmdLfxYsXt02cODHSfV64cMGU1i5YsKAtpuzlyA8ePGhr3Lix+RsZM2Y0z8Hdu3ejvM3o0aPNbYYPHx7jvxPdc6Fl1vW9EFU58r59+5oy3vocVapUyRYcHGzux7ncdnTvyeg+T/bHqyX/nS1dutRWuXJl8/zqRd+/+t44fPjwEx+X/e9Ed7EvI/Asny97ufaIJcUnTJhgSqvr+1A/e/qeKl26tO3NN98Md9wPP/xgCwgIMOXVne8nutdE3+/OJdsBJGw++h93hzcAAOKTFgDRHintXdTCCnHlyy+/dCy6GrECHNxH56TpnCodfqjD+AAgJpjjBABIdHSols63im158NjQ7yW1YIMOeSQ0uY/Od4z4HfG3335rhv3pkFkAiCnmOAEAEg1dEPngwYOm7LzO0YntWj4xcfv2bTPXS+fq6HwjLTsP99myZYvp9dP5eFooQudLaaDV+We6DwBiiqF6AIBEQ3sYdH0sXfhXizpo4Q1X02F5WlBACw107drVhDS4j74eWlhk27ZtjiIlWhBFC1Fo5U0A8IrgpCVqx4wZIzt37jQVm7Qqj34DaLWmR58+fUyVJF3lfMCAAdK2bdt4azMAAACAxMetc5x0OIOWb424JkZ0tORpnTp1zCJ2e/bsMeta6GKREcvMAgAAAECCHKqnaylY9Th9/PHHsnLlynAL4+nChbpYXVRrQgAAAABAoisOoQsNVq9ePdy+mjVrmp6n6ISFhZmLcwlSHeOsE0SdF74DAAAAkLjYbDa5efOm+Pn5RVr03quD08WLFyV79uzh9ul2aGio3L17V1KlShXpNiNGjIhypXUAAAAAUGfOnJH//Oc/kmCC09MICgoyxSTsQkJCzHoa+uQ899xzbm0bAAAAAPfRDhgtOJcuXTrLY70qOOXIkUMuXboUbp9uawCKqrdJpUiRwlwi0tsQnAAAAAD4xGAKj1ur6sVWhQoVZN26deH2rVmzxuwHAAAAgLji1uB069YtU1ZcL/Zy4/r76dOnHcPs2rRp4zj+vffek+PHj8tHH30kf//9t3z11VeyePFisyI4AAAAACTI4LRjxw4pVaqUuSidi6S/Dxw40Gzrorj2EKV0JXYtR669TLr+0+effy7ffPONqawHAAAAAAl+Haf4nACWPn16UySCOU4AAABA4hUai2zgVXOcAAAAAMAdCE4AAAAAYIHgBAAA4tTNmzelV69ekjt3brN8SMWKFWX79u2O69u2bWtKATtf3nzzTcf1YWFh0rp1azOMpmDBgrJ27dpw9z9mzBjp0aNHvD4mAImPV63jBAAAvM+7774rBw4ckLlz54qfn5989913Ur16dTl48KD4+/ubYzQozZo1y3Eb5zUYp02bJjt37pTg4GD55ZdfpGXLlmYdRw1YWpF3+vTppuAUAMQlepwAAECcuXv3rixdulRGjx4tr7zyihQoUEA+/fRT83PKlCnhgpIudG+/ZMyY0XHdoUOHpF69evLiiy9Kt27d5MqVK3L16lVzXZcuXWTUqFEUfAIQ5whOAAAgzjx8+FAePXokKVOmDLdfh+xt3rzZsb1hwwbJli2bFCpUyIShf//913GdLkGix2oIW716teTMmVOyZMki8+bNM/fbsGHDeH1MABInypEDAIA4pXOakidPLvPnz5fs2bPLggULJDAw0PQ6HT58WBYuXCipU6c26zUeO3ZM+vfvL2nTpjVD83x9feXBgwdmjtTPP/9sAtO4ceMkICBAXn75ZRO4vv76a3Mf+fPnl5kzZzqG/wGAK7MBwQkAAMQpDUPt27eXTZs2mSD00ksvmSIPOm9Jh+FFdPz4cROCtAhEtWrVorzPdu3aScmSJU3Y0qC1detWMxxQ51Lp0EAAiAnWcQIAAB5DQ9DGjRvl1q1bcubMGdm2bZvpRcqXL1+Ux+t+7Vk6evRolNevX79e/vrrL+nevbvpcapdu7akSZNGmjZtarYBIC4QnAAAQLzQcKPzk65fv27mKtWvXz/K486ePWvmOOmxEd27d88UiNDhedp7pfOnNIQp/anbABAXCE4AACBOaUhatWqVKR2+Zs0aee2116Rw4cJmuJ32Qn344YeyZcsWOXnypKxbt84EKp3/VLNmzUj3NXToUNPDVKpUKbNdqVIlWbZsmezbt08mTZpktgEgLrCOEwAAiFM6dyAoKMj0JGXKlEkaNWokw4YNk2TJkpmqexp65syZIzdu3DDrPNWoUcMEJOe1nJTOX1q8eLHs2bPHsa9x48ZmeF6VKlVMRT4tQAEAcYHiEAAAAAASpVCKQwAAAACA6xCcAAAAAMACwQkAAAAALFAcAgCARCZPv5XubkKCcXJkHXc3AUA8occJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAAAAACwQnAAAAADAAsEJAGLo5s2b0qtXL8mdO7ekSpVKKlasKNu3b3dcb7PZZODAgZIzZ05zffXq1eXIkSOO68PCwqR169by3HPPScGCBWXt2rXh7n/MmDHSo0ePeH1MAAAgZghOABBD7777rqxZs0bmzp0r+/fvlxo1aphwdO7cOXP96NGjZcKECTJ16lTZunWrpEmTRmrWrCn37t0z10+bNk127twpwcHB0qlTJ2nZsqUJW+rEiRMyffp0GTZsmFsfIwAAiBrBCQBi4O7du7J06VITjl555RUpUKCAfPrpp+bnlClTTAAaP368DBgwQOrXry/FixeXb7/9Vs6fPy8rVqww93Ho0CGpV6+evPjii9KtWze5cuWKXL161VzXpUsXGTVqlOmNAgAAnofgBAAx8PDhQ3n06JGkTJky3H4dkrd582bTY3Tx4kXTA2WXPn16KVeunOlhUiVKlDDHaghbvXq1GdKXJUsWmTdvnrnfhg0bxvvjAgAAMUNwAoAYSJcunVSoUEGGDh1qepE0RH333XcmFF24cMGEJpU9e/Zwt9Nt+3Xt27c34SkgIMAMyVu8eLFcv37dzIuaOHGi6a3SHiwd3mcf/gcAADwDwQkAYkjnNumQPH9/f0mRIoWZz9SiRQtJkiRm/5QmS5ZMJk+ebHqntKhE5cqVpW/fvtKzZ0/ZvXu3GdK3d+9eKV++vNkHAAA8B8EJAGIof/78snHjRrl165acOXNGtm3bJg8ePJB8+fJJjhw5zDGXLl0Kdxvdtl8X0fr16+Wvv/6S7t27y4YNG6R27dqmoETTpk3NNgAA8BwEJwCIJQ03Oj9Jh9npXCUtBpE3b14TkNatW+c4LjQ01FTX0yF+EWmlPS0Q8fXXX4uvr68Z+qchTOlP3QYAAJ6D4AQAMaQhadWqVWaonZYlf+2116Rw4cLSrl078fHxMWs8ffbZZ/Ljjz+acuVt2rQRPz8/adCgQaT70rlS2sNUqlQps12pUiVZtmyZ7Nu3TyZNmmS2AQCA50jq7gYAgLcICQmRoKAgOXv2rGTKlEkaNWpkijzo3CX10Ucfye3bt80aTTdu3DBzmDRoRazEd+DAAVMYYs+ePY59jRs3NsPzqlSpIoUKFZL58+fH++MDAADR87HZV19MJHTojJYI1hMg1ksBACRGefqtdHcTEoyTI+u4uwkA4ikbMFQPAAAAACwQnAAAAADAAsEJAAAAACxQHAKA12KehuswTwMAgCejxwkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwSmRefTokXzyySeSN29eSZUqleTPn1+GDh0qNpvNccynn34qhQsXljRp0kjGjBmlevXqsnXrVsf1YWFh0rp1a3nuueekYMGCsnbt2nB/Y8yYMdKjR494fVwAAABAXEoap/cOjzNq1CiZMmWKzJkzR1588UXZsWOHtGvXTtKnTy89e/Y0x2gYmjRpkuTLl0/u3r0r48aNkxo1asjRo0cla9asMm3aNNm5c6cEBwfLL7/8Ii1btpRLly6Jj4+PnDhxQqZPn27uFwAAAEgoCE6JzJ9//in169eXOnXqmO08efLIggULZNu2bY5jNAg5++KLL2TGjBmyb98+qVatmhw6dEjq1atngpeGqw8//FCuXr1qQlWXLl1MONPeKAAAACChYKheIlOxYkVZt26d/PPPP2Z77969snnzZqlVq1aUx9+/f9/0MGmPVIkSJcw+/am30d6o1atXS86cOSVLliwyb948SZkypTRs2DBeHxMAAAAQ1+hxSmT69esnoaGhZg6Tr6+vmfM0bNgwadWqVbjj/ve//0nz5s3lzp07JhitWbPGhCPVvn170/sUEBBg9i1evFiuX78uAwcOlA0bNsiAAQNk4cKFZv7UzJkzxd/f302PFgAAAHANglMioyFHe4bmz59vhtrt2bNHevXqJX5+fhIYGOg47rXXXjPX6RA8nbPUtGlTUyAiW7ZskixZMpk8eXK4+9V5UjpHavfu3bJixQrTkzV69Gizb+nSpW54pAAAAIDrMFQvkdH5SNrrpL1JxYoVM9XxevfuLSNGjAh3nFbUK1CggJQvX97Mb0qaNKn5GZX169fLX3/9Jd27dzc9TrVr1za317Cl2wAAAIC3o8cpkdGhd0mShM/LOmTv8ePHT7ydXq9lyCO6d++edOvWzfRi2Yf+2UubP3jwwGwDAAAA3o4ep0Smbt26Zk7TypUr5eTJk7J8+XJTNc9e0OH27dvSv39/2bJli5w6dcqUHdc5TefOnZMmTZpEuj9dA0p7mEqVKmW2K1WqJMuWLTNzoLSkuW4DAAAA3o4ep0Rm4sSJZgHcrl27yuXLl83cps6dO5vCDkp7jf7++2+zzpPOb8qcObO8/PLL8vvvv5s5Uc4OHDhg5kzpXCi7xo0bm+F5VapUkUKFCpm5VAAAAIC387HZx1UlElpRTktrh4SEsNYQ4OXy9Fvp7iYkGCdH/r+13ZA48NlxHT47QOLJBgzVAwAAAAALBCcAAAAAsEBwAgAAAAALFIfwAIw1dw3GmQMAACCu0OMEeAhd80orHubNm1dSpUol+fPnN+Xeneu36O9aATFnzpzmmOrVq8uRI0cc1+taW7qosU5uLFiwoKxduzbc3xgzZoz06NEjXh8XENf47AAA4gM9ToCHGDVqlEyZMsWUgtfS7zt27JB27dqZSi89e/Y0x4wePVomTJhgjtGTRD1ZrFmzphw8eFBSpkwp06ZNM2tvBQcHyy+//CItW7aUS5cuiY+Pj5w4cUKmT59u7hdISPjsAADiAz1OgIf4888/pX79+lKnTh3JkyePWROrRo0asm3bNsc35uPHj5cBAwaY44oXLy7ffvutnD9/XlasWGGOOXTokNSrV8+cPHbr1k2uXLli1uNSXbp0MSeYlOFHQsNnBwAQHwhOgIeoWLGirFu3Tv755x+zvXfvXtm8ebPUqlXLbOu33hcvXjRDjOz0G/Vy5cqZb8lViRIlzG3u3r0rq1evNsOSsmTJIvPmzTPfqjds2NBNjw6IO3x2AADxgaF6gIfo16+fWYStcOHC4uvra+ZtDBs2TFq1amWu1xM/lT179nC30237de3bt5d9+/ZJQECAOelbvHixXL9+3czt2LBhg/nGfeHChWYOyMyZM8Xf398NjxRwLT47AID4QHACPISeqOm32/PnzzfDhfbs2SO9evUSPz8/CQwMjNF9JEuWTCZPnhxun8710Hkeu3fvNsOS9Nt4ne+h+5YuXRpHjwaIP3x2AADxgaF6gIf48MMPzTfnzZs3l2LFipkKX71795YRI0aY63PkyGF+6oR1Z7ptvy6i9evXy19//SXdu3c335rXrl1b0qRJI02bNjXbQELAZwcAEB8IToCHuHPnjiRJEv4jqcOOHj9+bH7XSmB6kqdzOex0eNLWrVulQoUKke7v3r17ZpL7119/7Ri+9ODBA3Od/tRtICHgswMAiA8EJ8BD1K1b18zLWLlypZw8eVKWL18uX3zxhWNSupZF1uFHn332mfz444+yf/9+adOmjRmO1KBBg0j3p+vY6LfkpUqVMtuVKlWSZcuWmXkckyZNMttAQsBnBwAQH5jjBHiIiRMnmrVlunbtKpcvXzYndZ07dzaT0+0++ugjuX37tnTq1Elu3LghlStXllWrVpmqX84OHDhg5n3oXA87LdGsQ4yqVKkihQoVMvNBgISAzw4AID742JyXVncDnYyrK7JrZSMtB6v/Ayxbtmy0x+taHLrQ4enTp03lI/0fmo5jj/g/v+jo8AwtQxsSEuIxa3Lk6bfS3U1IEE6OrOPuJiCe8dlxHT4/iQufHdfhswN4t9hkA7cO1Vu0aJH06dNHBg0aJLt27TLBSVdy128Mo6Lf8ukEYD1eFyucMWOGuY/+/fvHe9sBAAAAJB5uDU46Br1jx46m5KuunTF16lRJnTq1WSMjutXhdWx5y5YtzerwujJ8ixYtHKvDAwAAAECCCk7379+XnTt3hlvJXasi6bZ9JfeoVofX29iD0vHjx+Xnn382k3ijExYWZrrgnC8AAAAA4BXFIa5evWpKuka1kvvff/8d5W20p0lvp5N6dWrWw4cP5b333nviUD2d/zR48GCXtx+JA/MAXId5AIkLnx3X4bMDAJ7Bq8qRa1Wj4cOHy1dffWXmRGl5WC0/q6VjoxMUFGQme9kvZ86cidc2AwAAAPB+butx0op4urBgbFZy13KzuiL8u+++a7Z1hXh7edn//ve/kRZAVClSpDAXAAAAAPC6HqfkyZNL6dKlw63krqu863ZUK7k/aXV45eaq6gAAAAASMLcugKulyAMDA6VMmTJm7SZdo0l7kLTKntKV3f39/c08Jfvq8FqJT1dzL1eunBw9etT0Qul+e4ACAAAAgAQVnJo1ayZXrlwxq7vrArglS5Y0K7nbC0boIrfOPUwDBgwQHx8f8/PcuXOSNWtWE5qGDRvmxkcBAAAAIKFza3BS3bt3N5foikE4S5o0qVn8Vi8AAAAAEF+8qqoeAAAAALgDwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAAAAALBCcAAAAAMACwQkAACCRypMnj/j4+ES6dOvWzVx/794983vmzJklbdq00qhRI7l06ZLj9teuXZO6deua60qVKiW7d+8Od/96288//zzeHxcQFwhOAAAAidT27dvlwoULjsuaNWvM/iZNmpifvXv3lp9++kmWLFkiGzdulPPnz8vbb7/tuP2wYcPk5s2bsmvXLnn11VelY8eOjuu2bNkiW7dulV69ernhkQGulzQO7hMAAABeIGvWrOG2R44cKfnz55eqVatKSEiIzJgxQ+bPny+vv/66uX7WrFlSpEgRE4rKly8vhw4dkubNm0vBggWlU6dOMm3aNHPcgwcP5L333pNvvvlGfH193fLYAFejxwkAAABy//59+e6776R9+/ZmuN7OnTtNAKpevbrjmMKFC8vzzz8vwcHBZrtEiRLy22+/ycOHD2X16tVSvHhxs3/06NGmB6pMmTJuezyAqxGcAAAAICtWrJAbN25I27ZtzfbFixclefLkkiFDhnDHZc+e3Vyn+vXrJ0mTJjW9VMuXLzc9VEeOHJE5c+bIJ598Ynqd8uXLJ02bNjU9WIA3IzgBAADAhJ5atWqJn59fjG+TPn16M5Tv1KlTZg5UQECAdO7cWcaMGSPz5s2T48ePy+HDhyV16tQyZMiQOG0/ENcITgAAAImcBp+1a9fKu+++69iXI0cOM3xPe6GcaVU9vS4qOgdKe6jq168vGzZskAYNGkiyZMlMsQndBrwZwQkAACCR08CTLVs2qVOnjmNf6dKlTehZt26dY5/2Hp0+fVoqVKgQ6T6uXLliepUmTpxoth89emTmSCn9qduAN6OqHgAAQCL2+PFjE5wCAwPNfCXnYXgdOnSQPn36SKZMmeS5556THj16mNCkFfUi0rLjffv2FX9/f7NdqVIlmTt3rtSoUcNU29NtwJsRnAAAABIxHaKnvUhaTS+icePGSZIkSczCt2FhYVKzZk356quvIh2nFfWOHj1qgpJd9+7dZceOHVKuXDkpW7asDBo0KM4fCxCXCE4AAACJmPYI2Wy2KK9LmTKlTJ482VyeRAOVXpxpQYjFixe7tK2AOzHHCQAAAAAsEJwAAAAAwALBCQAAAAAsMMcJAADAg+Tpt9LdTUgQTo78/0urA65AjxMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAuDo45cmTR4YMGSKnT5+O7U0BAAAAIHEEp169esmyZcskX7588sYbb8jChQslLCwsbloHAAAAAN4anPbs2SPbtm2TIkWKSI8ePSRnzpzSvXt32bVrV9y0EgAAAAC8cY7TSy+9JBMmTJDz58/LoEGD5JtvvpGXX35ZSpYsKTNnzhSbzebalgIAAACAmyR92hs+ePBAli9fLrNmzZI1a9ZI+fLlpUOHDnL27Fnp37+/rF27VubPn+/a1gIAAACANwQnHY6nYWnBggWSJEkSadOmjYwbN04KFy7sOKZhw4am9wkAAAAAEmVw0kCkRSGmTJkiDRo0kGTJkkU6Jm/evNK8eXNXtREAAAAAvCs4HT9+XHLnzv3EY9KkSWN6pQAAAAAgURaHuHz5smzdujXSft23Y8cOV7ULAAAAALw3OHXr1k3OnDkTaf+5c+fMdQAAAAAgiT04HTx40JQij6hUqVLmOgAAAACQxB6cUqRIIZcuXYq0/8KFC5I06VNXNwcAAACAhBOcatSoIUFBQRISEuLYd+PGDbN2k1bbAwAAAICEJtZdRGPHjpVXXnnFVNbT4Xlqz549kj17dpk7d25ctBEAAAAAvCs4+fv7y759+2TevHmyd+9eSZUqlbRr105atGgR5ZpOAAAAAODtnmpSkq7T1KlTJ9e3BgAAAAA80FNXc9AKeqdPn5b79++H21+vXj1XtAsAAAAAvDc4HT9+XBo2bCj79+8XHx8fsdlsZr/+rh49euT6VgIAAACAN1XVe//99yVv3rxy+fJlSZ06tfz111+yadMmKVOmjGzYsCFuWgkAAAAA3tTjFBwcLL/99ptkyZJFkiRJYi6VK1eWESNGSM+ePWX37t1x01IAAAAA8JYeJx2Kly5dOvO7hqfz58+b37U8+eHDh13fQgAAAADwth6nokWLmjLkOlyvXLlyMnr0aEmePLlMmzZN8uXLFzetBAAAAABvCk4DBgyQ27dvm9+HDBkib731llSpUkUyZ84sixYtios2AgAAAIB3BaeaNWs6fi9QoID8/fffcu3aNcmYMaOjsh4AAAAAJNo5Tg8ePJCkSZPKgQMHwu3PlCkToQkAAABAghWr4JQsWTJ5/vnnXbpW0+TJkyVPnjySMmVKM2dq27ZtTzz+xo0b0q1bN8mZM6ekSJFCChYsKD///LPL2gMAAAAAz1xV77///a/079/fDM97Vjonqk+fPjJo0CDZtWuXlChRwgwF1DWionL//n1544035OTJk/L999+bKn7Tp08Xf3//Z24LAAAAALhsjtOkSZPk6NGj4ufnZ0qQp0mTJtz1GoBi6osvvpCOHTtKu3btzPbUqVNl5cqVMnPmTOnXr1+k43W/BrY///zT9H4p7a0CAAAAAI8KTg0aNHDJH9beo507d0pQUJBjny6mW716dbPIblR+/PFHqVChghmq98MPP0jWrFmlZcuW8vHHH4uvr2+UtwkLCzMXu9DQUJe0HwAAAEDiEevgpMPqXOHq1atmrlT27NnD7ddtrdQXlePHj8tvv/0mrVq1MvOatOera9eupmhFdO0aMWKEDB482CVtBgAAAJA4xXqOkzs9fvxYsmXLZhbbLV26tDRr1szMudIhftHRHq2QkBDH5cyZM/HaZgAAAACJsMdJh9M9qfR4TCvuZcmSxQyvu3TpUrj9up0jR44ob6OV9HRuk/OwvCJFisjFixfN0L/kyZNHuo1W3tMLAAAAAMRbcFq+fHm4bR0mt3v3bpkzZ06shsRpyNFeo3Xr1jnmTWmPkm537949yttUqlRJ5s+fb47TAKf++ecfE6iiCk0AAAAA4JbgVL9+/Uj7GjduLC+++KIpL96hQ4cY35eWIg8MDJQyZcpI2bJlZfz48XL79m1Hlb02bdqYUuM6T0l16dLFVPV7//33pUePHnLkyBEZPny49OzZM7YPAwAAAADiLjhFp3z58tKpU6dY3UbnKF25ckUGDhxohtuVLFlSVq1a5SgYcfr0aUfPksqVK5esXr1aevfuLcWLFzehSkOUVtUDAAAAAI8OTnfv3pUJEyY81UK0OiwvuqF5GzZsiLRPy5Fv2bLlqdoJAAAAAPESnDJmzBiuOITNZpObN29K6tSp5bvvvnuqRgAAAABAggpO48aNCxecdCidLkRbrlw5E6oAAAAAQBL7Ok5t27Y1BR3sl9atW8ubb75JaAIAAABc5NNPPzWdFc6XwoULO65/9dVXI13/3nvvOa6/du2a1K1bV9KmTSulSpUyVbCddevWTT7//PN4fUyJrsdp1qxZ5gVo0qRJuP1LliyRO3fumDAFAAAA4Nlo1eq1a9c6tpMmDX/q3rFjRxkyZIhjW6fO2A0bNsxMp9m1a5dMmTLFHLtjxw5zndYL2Lp1q6lRgDjscdLS4Lp4bUTZsmUzpcEBAAAAPDsNSjly5HBcIp6Da1Byvv65555zXHfo0CFp3ry5FCxY0FS+1m37GqzaMzV16lTx9fWN98eUqIKTlgjPmzdvpP25c+c21wEAAAB4drpmqZ+fn+TLl09atWoV6Vx73rx5JkwVLVpUgoKCzOgvuxIlSshvv/0mDx8+NMv56FI+avTo0WaYn66jijgOTtqztG/fvkj79+7dK5kzZ47t3QEAAACIQAuvzZ4926xxqkPtTpw4IVWqVDHD71TLli1NRev169eb0DR37lx55513HLfv16+f6bHKnz+/LF++XGbMmGGC2Jw5c+STTz4xvU4ayJo2bSohISFufKQJeI5TixYtpGfPnpIuXTp55ZVXzL6NGzeahWi1OxAAAADAs6lVq5bjd+0t0iClI7wWL14sHTp0MMPv7IoVKyY5c+aUatWqybFjx0xYSp8+vcyfPz/cfb7++usyZswY01N1/PhxOXz4sGOeFIUi4qDHaejQoeaF0xcmVapU5lKjRg3zQjDHCQAAAHC9DBkymPlKR48ejfJ6PT9X0V2vBd70PurXry8bNmyQBg0aSLJkyUzBN91GHPQ4JU+eXBYtWiSfffaZ7NmzxwQnTbmagAEAAAC43q1bt0xvki4FFBU9L1fa8xTRlStXTK/S5s2bzfajR49MkQilP3UbcRCc7F544QVzAQAAAOBaH3zwgVmHSTsnzp8/L4MGDTJV8HTajAYoHYZXu3ZtU2NA6w/07t3bTKOxF4Fw1qtXL+nbt6/4+/ub7UqVKpk5UTpqbNq0aWYbcTBUr1GjRjJq1KhI+7VCR8S1nQAAAADE3tmzZ01IKlSokCngoAFJ11/KmjWrGQGm6ztp8NFFcTUU6Tn6Tz/9FOl+tKKeDt/r2rWrY1/37t1NYQgd3nf//n0TyhAHPU6bNm0yKxlHNYGNSWUAAADAs1u4cGG01+XKlcsUZ4uJmjVrmkvE9Z+0yATiuMdJx1dqyo1IJ5eFhobG9u4AAAAAIOEFJy0EocUhokrFAQEBrmoXAAAAAHjvUD1dMOvtt982k9K0BLlat26dLFiwQJYsWRIXbQQAAAAA7wpOWt1jxYoVZs2m77//3pQj1+odOkGtatWqcdNKAAAAwM3y9Fvp7iYkGCdH1pFEUY68Tp065hLRgQMHpGjRoq5oFwAAAAB47xyniG7evGnqv5ctW1ZKlCjhmlYBAAAAQEIITlqWvE2bNmZ14rFjx5r5TlpbHgAAAAASmlgN1bt48aLMnj1bZsyYYUqP62JcYWFhZs4TFfUAAAAASGLvcdKiELpy8b59+2T8+PFy/vx5mThxYty2DgAAAAC8qcfpl19+kZ49e0qXLl3khRdeiNtWAQAAAIA39jht3rzZFIIoXbq0lCtXTiZNmiRXr16N29YBAAAAgDcFp/Lly8v06dPlwoUL0rlzZ1m4cKH4+fnJ48ePZc2aNSZUAQAAAEBCFOuqemnSpJH27dubHqj9+/dL3759ZeTIkZItWzapV69e3LQSAAAAALx1HSctFjF69Gg5e/asLFiwwHWtAgAAAICEtACu8vX1lQYNGsiPP/7oirsDAAAAgIQXnAAAAAAgISM4AQAAAIAFghMAAAAAWCA4AQAAAICFpPIUjhw5IuvXr5fLly+bdZycDRw48GnuEgAAAAASTnDSRXC7dOkiWbJkkRw5coiPj4/jOv2d4AQAAABAEntw+uyzz2TYsGHy8ccfx02LAAAAAMDb5zhdv35dmjRpEjetAQAAAICEEJw0NP36669x0xoAAAAA8NahehMmTHD8XqBAAfnkk09ky5YtUqxYMUmWLFm4Y3v27On6VgIAAACApwencePGhdtOmzatbNy40VycaXEIghMAAACARBmcTpw4EfctAQAAAAAPxQK4AAAAAODq4NSoUSMZNWpUpP2jR4+m2h4AAACABCnWwWnTpk1Su3btSPtr1aplrgMAAAAASezB6datW5I8efJI+7W6XmhoqKvaBQAAAADeG5y0BPmiRYsi7V+4cKEEBAS4ql0AAAAA4F1V9ZzpGk5vv/22HDt2TF5//XWzb926dbJgwQJZsmRJXLQRAAAAALwrONWtW1dWrFghw4cPl++//15SpUolxYsXl7Vr10rVqlXjppUAAAAA4EaxDk6qTp065gIAAAAAiUGs5zjly5dP/v3330j7b9y4Ya4DAAAAAEnswenkyZPy6NGjSPvDwsLk3LlzrmoXAAAAAHjfUL0ff/zR8fvq1aslffr0jm0NUlogIk+ePK5vIQAAAAB4S3Bq0KCB+enj4yOBgYGR1nDS0PT555+7voUAAAAA4C3B6fHjx+Zn3rx5Zfv27ZIlS5a4bBcAAAAAeG9VvRMnTsRNSwAAAAAgIZUjv337tmzcuFFOnz4t9+/fD3ddz549XdU2AAAAAPDO4LR7926pXbu23LlzxwSoTJkyydWrVyV16tSSLVs2ghMAAACABCfW5ch79+4tdevWlevXr0uqVKlky5YtcurUKSldurSMHTs2bloJAAAAAN4UnPbs2SN9+/aVJEmSiK+vr1m/KVeuXDJ69Gjp379/3LQSAAAAALwpOGnpcQ1NSofm6Twnpes6nTlzxvUtBAAAAABvm+NUqlQpU478hRdekKpVq8rAgQPNHKe5c+dK0aJF46aVAAAAAOBNPU7Dhw+XnDlzmt+HDRsmGTNmlC5dusiVK1dk2rRpcdFGAAAAAPCuHqcyZco4fteheqtWrXJ1mwAAAADAu3ucAAAAACCxiXGP0+uvvx6j43777bdnaQ8AAAAAeG9w2rBhg+TOnVvq1KljKusBAAAAQGIR4+A0atQomTVrlixZskRatWol7du3p4oeAAAAgEQhxnOcPvzwQzl48KCsWLFCbt68KZUqVZKyZcvK1KlTJTQ0NG5bCQAAAADeVByiQoUKMn36dLlw4YJ069ZNZs6cKX5+foQnAAAAAAnWU1fV27Vrl2zcuFEOHTpkhuwx7wkAAABAQhWr4HT+/HmzAG7BggWlcePGkilTJtm6dats2bJFUqVKFXetBAAAAABvKA5Ru3ZtWb9+vdSoUUPGjBljquslTRrr9XMBAAAAwOvEOPmsWrVKcubMKadPn5bBgwebS3RD+AAAAAAgUQanQYMGxW1LAAAAAMBDEZwAAAAAIK6q6gEAAABAYkFwAgAAAAALBCcAAAAAsEBwAgAAAABXB6dvv/1WwsLCIu2/f/++uQ4AAAAAJLEHp3bt2klISEik/Tdv3jTXAQAAAIAk9uBks9nEx8cn0v6zZ89K+vTpXdUuAAAAAPC+4FSqVCl56aWXTGiqVq2a+d1+KVGihFSpUkWqV6/+VI2YPHmy5MmTR1KmTCnlypWTbdu2xeh2CxcuNO1p0KDBU/1dAAAAAHDpArj2cLJnzx6pWbOmpE2b1nFd8uTJTfBp1KiRxNaiRYukT58+MnXqVBOaxo8fb+7/8OHDki1btmhvd/LkSfnggw9MYAMAAAAAjwhOgwYNMj81IDVv3lxSpEjhkgZ88cUX0rFjR8f8KA1QK1eulJkzZ0q/fv2ivM2jR4+kVatWMnjwYPn999/lxo0bLmkLAAAAALhkjlNAQIDpdYpo69atsmPHjljdl1bi27lzZ7ghfkmSJDHbwcHB0d5uyJAhpjeqQ4cOln9DKwCGhoaGuwAAAABAnAanbt26yZkzZyLtP3funLkuNq5evWp6j7Jnzx5uv25fvHgxytts3rxZZsyYIdOnT4/R3xgxYoQpWmG/5MqVK1ZtBAAAAIBYB6eDBw+aghBRFY/Q6+KSljxv3bq1CU1ZsmSJ0W2CgoJM+XT7JarQBwAAAAAumeNkp3ObLl26JPny5Qu3/8KFC5I0aezuTsOPr6+vuT9nup0jR45Ixx87dswUhahbt65j3+PHj81P/dtaUCJ//vyR2uuq+VgAAAAAEqdY9zjVqFHD0Ytjp8UZ+vfvL2+88Uas7kur8ZUuXVrWrVsXLgjpdoUKFSIdX7hwYdm/f7+ZY2W/1KtXT1577TXzO8PwAAAAAHhEj9PYsWPllVdekdy5c5vheUpDi85Lmjt3bqwboKXIAwMDpUyZMlK2bFlTjvz27duOKntt2rQRf39/M1dJ13kqWrRouNtnyJDB/Iy4HwAAAADcFpw0xOzbt0/mzZsne/fulVSpUpmQ06JFC0mWLFmsG9CsWTO5cuWKDBw40BSEKFmypKxatcpRMOL06dOm0h4AAAAAeE1wUmnSpJFOnTq5rBHdu3c3l6hs2LDhibedPXu2y9oBAAAAAC4LTkor6GlvkK7F5EznHAEAAABAog5Ox48fl4YNG5oiDT4+PmKz2cx+/V3pukwAAAAAkJDEevLQ+++/L3nz5pXLly9L6tSp5a+//pJNmzaZ4g5Ww+oAAAAAIFH0OAUHB8tvv/1m1mDSog16qVy5sql617NnT9m9e3fctBQAAAAAvKXHSYfipUuXzvyu4en8+fPmdy1PrgvQAgAAAIAk9h4nXS9Jy5DrcL1y5crJ6NGjzUK206ZNk3z58sVNKwEAAADAm4LTgAEDzAK1asiQIfLWW29JlSpVJHPmzLJo0aK4aCMAAAAAeFdwqlmzpuP3AgUKyN9//y3Xrl2TjBkzOirrAQAAAECinuN05cqVSPsyZcpkQpOWKAcAAAAASezBqVixYrJy5cpI+8eOHStly5Z1VbsAAAAAwHuDU58+faRRo0bSpUsXuXv3rpw7d06qVatmikTMnz8/bloJAAAAAN4UnD766COzltPvv/8uxYsXN5cUKVLIvn37pGHDhnHTSgAAAADwpuBkLwqhZclPnjwpoaGh0qxZM8mRI4frWwcAAAAA3hic/vjjD9PLdOTIEdPLNGXKFOnRo4cJT9evX4+bVgIAAACANwWn119/3YSkLVu2SJEiReTdd9+V3bt3y+nTp03hCAAAAACQxL6O06+//ipVq1YNty9//vymJ2rYsGGubBsAAAAAeGePU8TQ5LijJEnkk08+cUWbAAAAAMA7g1Pt2rUlJCTEsT1y5Ei5ceOGY/vff/+VgIAA17cQAAAAALwlOK1evVrCwsIc28OHD5dr1645th8+fCiHDx92fQsBAAAAwFuCk81me+I2AAAAACRUT7WOEwAAAAAkJjEOTj4+PuYScR8AAAAAJHQxLkeuQ/Patm0rKVKkMNv37t2T9957T9KkSWO2nec/AQAAAECiDE6BgYHhtt95551Ix7Rp08Y1rQIAAAAAbwxOs2bNituWAAAAAICHojgEAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACABYITAAAAAFggOAEAAACANwSnyZMnS548eSRlypRSrlw52bZtW7THTp8+XapUqSIZM2Y0l+rVqz/xeAAAAADw+uC0aNEi6dOnjwwaNEh27dolJUqUkJo1a8rly5ejPH7Dhg3SokULWb9+vQQHB0uuXLmkRo0acu7cuXhvOwAAAIDEwe3B6YsvvpCOHTtKu3btJCAgQKZOnSqpU6eWmTNnRnn8vHnzpGvXrlKyZEkpXLiwfPPNN/L48WNZt25dvLcdAAAAQOLg1uB0//592blzpxlu52hQkiRmW3uTYuLOnTvy4MEDyZQpU5TXh4WFSWhoaLgLAAAAAHhNcLp69ao8evRIsmfPHm6/bl+8eDFG9/Hxxx+Ln59fuPDlbMSIEZI+fXrHRYf2AQAAAIBXDdV7FiNHjpSFCxfK8uXLTWGJqAQFBUlISIjjcubMmXhvJwAAAADvltSdfzxLlizi6+srly5dCrdft3PkyPHE244dO9YEp7Vr10rx4sWjPS5FihTmAgAAAABe2eOUPHlyKV26dLjCDvZCDxUqVIj2dqNHj5ahQ4fKqlWrpEyZMvHUWgAAAACJlVt7nJSWIg8MDDQBqGzZsjJ+/Hi5ffu2qbKn2rRpI/7+/maukho1apQMHDhQ5s+fb9Z+ss+FSps2rbkAAAAAQIILTs2aNZMrV66YMKQhSMuMa0+SvWDE6dOnTaU9uylTpphqfI0bNw53P7oO1Keffhrv7QcAAACQ8Lk9OKnu3bubS3QL3jo7efJkPLUKAAAAABJAVT0AAAAAiA8EJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAACwQHACAAAAAAsEJwAAAADwhuA0efJkyZMnj6RMmVLKlSsn27Zte+LxS5YskcKFC5vjixUrJj///HO8tRUAAABA4uP24LRo0SLp06ePDBo0SHbt2iUlSpSQmjVryuXLl6M8/s8//5QWLVpIhw4dZPfu3dKgQQNzOXDgQLy3HQAAAEDi4Pbg9MUXX0jHjh2lXbt2EhAQIFOnTpXUqVPLzJkzozz+yy+/lDfffFM+/PBDKVKkiAwdOlReeuklmTRpUry3HQAAAEDikNSdf/z+/fuyc+dOCQoKcuxLkiSJVK9eXYKDg6O8je7XHipn2kO1YsWKKI8PCwszF7uQkBDzMzQ0VDzF47A77m5CghAXrymvjevw+iSu14fXxnX47Hg2Xh/PxWvj2UI95Fzc3g6bzebZwenq1avy6NEjyZ49e7j9uv33339HeZuLFy9Gebzuj8qIESNk8ODBkfbnypXrmdoOz5N+vLtbgCfh9fFsvD6ei9fGs/H6eC5eG8+W3sNen5s3b0r69Ok9NzjFB+3Ncu6hevz4sVy7dk0yZ84sPj4+bm2bt9AkrkHzzJkz8txzz7m7OYiA18dz8dp4Nl4fz8Vr49l4fTwbr0/saE+ThiY/Pz/LY90anLJkySK+vr5y6dKlcPt1O0eOHFHeRvfH5vgUKVKYi7MMGTI8c9sTI/3w8QH0XLw+novXxrPx+nguXhvPxuvj2Xh9Ys6qp8kjikMkT55cSpcuLevWrQvXI6TbFSpUiPI2ut/5eLVmzZpojwcAAACAZ+X2oXo6jC4wMFDKlCkjZcuWlfHjx8vt27dNlT3Vpk0b8ff3N3OV1Pvvvy9Vq1aVzz//XOrUqSMLFy6UHTt2yLRp09z8SAAAAAAkVG4PTs2aNZMrV67IwIEDTYGHkiVLyqpVqxwFIE6fPm0q7dlVrFhR5s+fLwMGDJD+/fvLCy+8YCrqFS1a1I2PImHToY66zlbEIY/wDLw+novXxrPx+nguXhvPxuvj2Xh94o6PLSa19wAAAAAgEXP7ArgAAAAA4OkITgAAAABggeAEAAAAABYITgAAAABggeCEaE2ZMkWKFy/uWEBN18r65Zdf3N0sRGHkyJHi4+MjvXr1cndTICKffvqpeT2cL4ULF3Z3s/B/zp07J++8845kzpxZUqVKJcWKFTPLWsD98uTJE+mzo5du3bq5u2kQkUePHsknn3wiefPmNZ+d/Pnzy9ChQ4U6Y57h5s2b5jwgd+7c5vXRStTbt293d7MSFLeXI4fn+s9//mNOyLXku/6jOGfOHKlfv77s3r1bXnzxRXc3D/9H/1H8+uuvTciF59DPyNq1ax3bSZPyz60nuH79ulSqVElee+0180VQ1qxZ5ciRI5IxY0Z3Nw3/9++ZnpzbHThwQN544w1p0qSJW9uF/2fUqFHmS1U9H9B/4/QLB113M3369NKzZ093Ny/Re/fdd81nZu7cueLn5yffffedVK9eXQ4ePGjWRMWzoxw5YiVTpkwyZswY6dChg7ubAhG5deuWvPTSS/LVV1/JZ599ZtZB00Wk4f4eJ11fbs+ePe5uCiLo16+f/PHHH/L777+7uymIAf32/H//+58Jt9rzBPd66623zDqbM2bMcOxr1KiR6d3Qk3S4z927dyVdunTyww8/SJ06dRz7S5cuLbVq1TLnCHh2DNVDjOg3gAsXLpTbt2+bIXvwDDp8Rf+B1G+U4Fn0RE+/8cuXL5+0atXKLOYN9/vxxx+lTJkypgcjW7ZsUqpUKZk+fbq7m4Uo3L9/35yMt2/fntDkIXTo17p16+Sff/4x23v37pXNmzebE3O418OHD825WsqUKcPt11CrrxFcg7EjeKL9+/eboHTv3j1JmzatLF++XAICAtzdLIiYILtr1y7GL3ugcuXKyezZs6VQoUJy4cIFGTx4sFSpUsUModBvBOE+x48fN0ON+vTpI/379zefHx1ilDx5cgkMDHR38+BEe21v3Lghbdu2dXdT4NRjGxoaauZs+vr6mhP1YcOGmS+H4F76/xY9X9M5Z0WKFDE9gwsWLJDg4GApUKCAu5uXYDBUD5bf+Ok35SEhIfL999/LN998Ixs3biQ8udmZM2fMt+Zr1qxxzG169dVXGarnofTkTyfrfvHFFwxzdTMNSPrZ+fPPPx37NDhpgNITDHiOmjVrmtfrp59+cndT4PSF3YcffmiG7OscJx2OrMMp9d82vnhwv2PHjpke2k2bNplgq0P5CxYsKDt37pRDhw65u3kJAj1OeCL9n5b9mwodJ6snF19++aUpRgD30X8EL1++bP5RtNNv/vQfy0mTJklYWJj5RxOeIUOGDOZ/XkePHnV3UxK9nDlzRvriR7+dXbp0qdvahMhOnTpliqssW7bM3U2BEw1N2uvUvHlzs60VKfW1GjFiBMHJA2iVQ/1yW6dVaM+g/nvXrFkzM2QcrsEcJ8TK48ePzUk53KtatWpmGKV+22e/6LfoOlxCfyc0eV4RD/0mUP8nBvfSinqHDx8Ot0/na2iPIDzHrFmzzBw050nucL87d+5IkiThTx31/zd6bgDPkSZNGvP/G60iunr1alMRGa5BjxOiFRQUZCZ8Pv/882ZtgPnz58uGDRvMhxDuH8tctGjRSP9Q6ro0Efcj/n3wwQdSt25dczJ+/vx5GTRokDm5aNGihbubluj17t3bTHAfPny4NG3aVLZt2ybTpk0zF3gGPQnX4KQ9GJTx9yz675rOadLzAh2qp8uT6DA9HR4G99PzM52Bo/NrdYSD9hDqfDQtGQ/X4F8kREuHgrVp08ZMbtc1GnQujX4odU0NANE7e/asCUn//vuvWSeocuXKsmXLFvM73Ovll182RW70i6EhQ4aYhTx1XiCT2z2HDtHTubWcjHueiRMnmgVwu3btas4RtHJo586dZeDAge5uGkTMfHT9t03/H6TLx2ipeA26yZIlc3fTEgyKQwAAAACABeY4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQAAAIAFghMAAAAAWCA4AQDgYTZs2CA+Pj5y48YNdzcFAPB/CE4AgGfWtm1bc6I/cuTIcPtXrFhh9nsiTwknr776qvTq1cutbQAAWCM4AQBcImXKlDJq1Ci5fv26u5sCAIDLEZwAAC5RvXp1yZEjh4wYMSLaY/79919p0aKF+Pv7S+rUqaVYsWKyYMGCSD0wPXr0ML0wGTNmlOzZs8v06dPl9u3b0q5dO0mXLp0UKFBAfvnll3C3O3DggNSqVUvSpk1rbtO6dWu5evXqUz+esLAw+eCDD0xb06RJI+XKlTO9VHazZ8+WDBkyyOrVq6VIkSLm77755pty4cIFxzEPHz6Unj17muMyZ84sH3/8sQQGBkqDBg0cPXUbN26UL7/80vR+6eXkyZOO2+/cuVPKlCljnquKFSvK4cOHn/rxAACeDcEJAOASvr6+Mnz4cJk4caKcPXs2ymPu3bsnpUuXlpUrV5qg06lTJxNwtm3bFu64OXPmSJYsWcx+DVFdunSRJk2amPCwa9cuqVGjhrndnTt3zPE63O7111+XUqVKyY4dO2TVqlVy6dIladq06VM/nu7du0twcLAsXLhQ9u3bZ/6+BqMjR444jtG/P3bsWJk7d65s2rRJTp8+bcKWnfbAzZs3T2bNmiV//PGHhIaGmuGLdhqYKlSoIB07djSBSy+5cuVyXP/f//5XPv/8c/OYkiZNKu3bt3/qxwMAeEY2AACeUWBgoK1+/frm9/Lly9vat29vfl++fLnN6n81derUsfXt29exXbVqVVvlypUd2w8fPrSlSZPG1rp1a8e+CxcumPsNDg4220OHDrXVqFEj3P2eOXPGHHP48OEo/+769evN9devX4903alTp2y+vr62c+fOhdtfrVo1W1BQkPl91qxZ5vZHjx51XD958mRb9uzZHdv6+5gxY8I9lueff97xXNkf7/vvvx9l29auXevYt3LlSrPv7t27UT4eAEDcSvqswQsAAGfay6K9P849L3aPHj0yvVKLFy+Wc+fOyf37982QOB2K5qx48eLherJ0mJsO67PToXjq8uXL5ufevXtl/fr1ZrhcRMeOHZOCBQvG6jHs37/ftDXi7bSt2hY7bXf+/Pkd2zlz5nS0KSQkxPR6lS1bNtxj0R63x48fx6gdzs+D3rfS+3/++edj9XgAAM+O4AQAcKlXXnlFatasKUFBQWYOj7MxY8aY4Wnjx483QUjnDulcJg1QzpIlSxZuW+f+OO+zV+qzB5Bbt25J3bp1TWiLyB44YkPvT0OOzjHSn86cw1lU7bTZtGPINZ70mAEA8YvgBABwOS1LXrJkSSlUqFC4/TrPp379+vLOO+84QsA///wjAQEBz/T3XnrpJVm6dKnkyZPHzAV6VjpXSnuctHenSpUqT3Uf6dOnNz1j27dvN2FS6X3qHC19buySJ09u9gMAPBvFIQAALqe9Sa1atZIJEyaE2//CCy/ImjVr5M8//5RDhw5J586dzXC2Z9WtWze5du2aqdinQUWH52m1O63CZxVKdFjenj17HBcd9qdD9LT9bdq0kWXLlsmJEydMoQqtGKiFLWJKC1vobX744QdTEe/999835dqd17bSsLd161ZTTU+rANKjBACeieAEAIgTQ4YMiRQCBgwYYHqHdCiflh3X8uX20tzPws/Pz/RmaUjSinsa3HQIoJYBT5Lkyf+r094g7WGyX3QOktJKeBqc+vbta3rOtJ0aymIzv0jLj2uY0/vR6nk6zE8fu655ZadzwXQ4oPa6Zc2a1VTmAwB4Hh+tEOHuRgAAkBhokNQ1n7RM+tChQ93dHABALDDHCQCAOHLq1Cn59ddfpWrVqqYi36RJk8ywv5YtW7q7aQCAWGKoHgAAcUSHCc6ePVtefvllqVSpkplPtXbtWtPrBADwLgzVAwAAAAAL9DgBAAAAgAWCEwAAAABYIDgBAAAAgAWCEwAAAABYIDgBAAAAgAWCEwAAAABYIDgBAAAAgAWCEwAAAADIk/1/IMzcKrgW62IAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Accuracy by name length:\n", " Length 3: 83.3% (6 names in test set)\n", " Length 4: 80.0% (41 names in test set)\n", " Length 5: 90.0% (112 names in test set)\n", " Length 6: 80.0% (147 names in test set)\n", " Length 7: 95.0% (124 names in test set)\n", " Length 8: 70.0% (46 names in test set)\n", " Length 9: 55.0% (20 names in test set)\n" ] } ], "source": [ "import matplotlib.pyplot as plt\n", "from collections import defaultdict\n", "\n", "# Group test names by length\n", "names_by_length = defaultdict(list)\n", "for name in test_data[:500]: # Use first 500 for speed\n", " names_by_length[len(name)].append(name)\n", "\n", "# Evaluate accuracy by length\n", "length_accuracies = {}\n", "lengths = sorted(names_by_length.keys())\n", "\n", "for length in lengths:\n", " if len(names_by_length[length]) >= 5: # Only evaluate if we have enough samples\n", " names_subset = names_by_length[length][:20] # Limit to 20 per length for speed\n", " results = evaluate_accuracy(model, tokenizer, names_subset, device)\n", " length_accuracies[length] = results['exact_match_accuracy']\n", "\n", "# Plot\n", "if length_accuracies:\n", " plt.figure(figsize=(10, 6))\n", " lengths = list(length_accuracies.keys())\n", " accuracies = list(length_accuracies.values())\n", " \n", " plt.bar(lengths, accuracies)\n", " plt.xlabel('Name Length')\n", " plt.ylabel('Exact Match Accuracy')\n", " plt.title('Reversal Accuracy by Name Length')\n", " plt.ylim(0, 1.0)\n", " \n", " # Add percentage labels on bars\n", " for i, (l, acc) in enumerate(zip(lengths, accuracies)):\n", " plt.text(l, acc + 0.01, f'{acc*100:.0f}%', ha='center')\n", " \n", " plt.show()\n", " \n", " # Print summary\n", " print(\"Accuracy by name length:\")\n", " for length, acc in sorted(length_accuracies.items()):\n", " count = len(names_by_length[length])\n", " print(f\" Length {length}: {acc*100:.1f}% ({count} names in test set)\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Test with different prompt formats" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Testing different prompts for 'sophia' (expected: aihpos)\n", "============================================================\n", "Train :\n", " Prompt: Reverse the name: sophia. Answer:...\n", " Output: aihpos ✓\n", "Variation 1:\n", " Prompt: Reverse: sophia =>...\n", " Output: aihpos ✓\n", "Variation 2:\n", " Prompt: What is sophia backwards?...\n", " Output: ✗\n", "Variation 3:\n", " Prompt: sophia reversed is:...\n", " Output: aihsopi ✗\n" ] } ], "source": [ "# Try different prompt variations to see if the model generalizes\n", "test_prompts = [\n", " (\"Reverse the name: {name}. Answer:\", \"Train\"),\n", " (\"Reverse: {name} =>\", \"Variation 1\"),\n", " (\"What is {name} backwards?\", \"Variation 2\"),\n", " (\"{name} reversed is:\", \"Variation 3\"),\n", "]\n", "\n", "test_name = \"sophia\"\n", "expected = test_name[::-1]\n", "\n", "print(f\"Testing different prompts for '{test_name}' (expected: {expected})\")\n", "print(\"=\"*60)\n", "\n", "for prompt_template, prompt_type in test_prompts:\n", " prompt = tokenizer.bos_token + prompt_template.format(name=test_name)\n", " \n", " inputs = tokenizer(prompt, return_tensors=\"pt\").to(device)\n", " \n", " model.eval()\n", " with torch.no_grad():\n", " outputs = model.generate(\n", " inputs[\"input_ids\"],\n", " max_new_tokens=20,\n", " temperature=0.1,\n", " do_sample=True,\n", " pad_token_id=tokenizer.pad_token_id,\n", " eos_token_id=tokenizer.eos_token_id\n", " )\n", " \n", " response = tokenizer.decode(outputs[0], skip_special_tokens=True)\n", " # Extract answer (everything after the prompt)\n", " answer = response[len(prompt_template.format(name=test_name)):].strip().split()[0] if len(response) > len(prompt_template.format(name=test_name)) else \"\"\n", " \n", " is_correct = answer == expected\n", " symbol = \"✓\" if is_correct else \"✗\"\n", " \n", " print(f\"{prompt_type:10}:\")\n", " print(f\" Prompt: {prompt_template.format(name=test_name)[:40]}...\")\n", " print(f\" Output: {answer:10} {symbol}\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "anlp", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.18" } }, "nbformat": 4, "nbformat_minor": 4 }