{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lecture 4: Recurrent Neural Networks\n",
    "\n",
    "Lecture 4 | CMU ANLP Fall 2025 | Instructor: Sean Welleck\n",
    "\n",
    "### Part 2: Encoder-decoder with attention\n",
    "\n",
    "This is a notebook for [CMU CS11-711 Advanced NLP](https://cmu-l3.github.io/anlp-fall2025/) that trains an encoder-decoder model with attention."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['emma',\n",
       " 'olivia',\n",
       " 'ava',\n",
       " 'isabella',\n",
       " 'sophia',\n",
       " 'charlotte',\n",
       " 'mia',\n",
       " 'amelia',\n",
       " 'harper',\n",
       " 'evelyn']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = open('names.txt').read().splitlines()\n",
    "data[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "token_to_index = {tok: i for i, tok in enumerate('abcdefghijklmnopqrstuvwxyz')}\n",
    "# Start/stop token\n",
    "token_to_index['[S]'] = 26\n",
    "# Padding token\n",
    "token_to_index['[PAD]'] = 27\n",
    "\n",
    "index_to_token = {i: tok for tok, i in token_to_index.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task: reversing names\n",
    "\n",
    "Our task will be to reverse a name, e.g.:\n",
    "\n",
    "    sophia -> aihpos\n",
    "\n",
    "To make the model learn an additional simple pattern, we'll also add a noise character after each character of the input, e.g.:\n",
    "\n",
    "    suorpxhziwa -> aihpos\n",
    "\n",
    "As a spoiler, we'll be able to see the model attending to non-noise characters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25626, 25626, 25626)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "def build_dataset(data):\n",
    "    X, Yin, Yout = [], [], []\n",
    "    for item in data:\n",
    "        # The task is to reverse the input\n",
    "        # The input also has a 'noise' token after each token\n",
    "        # The output does not have the noise token\n",
    "        x_tokens = []\n",
    "        for token in item:\n",
    "            x_tokens.append(token)\n",
    "            # Random character\n",
    "            x_tokens.append(random.choice('abcdefghijklmnopqrstuvwxyz'))\n",
    "        y_tokens = ['[S]'] + list(reversed(item)) + ['[S]']\n",
    "\n",
    "        x_indices = [token_to_index[token] for token in x_tokens]\n",
    "        y_indices = [token_to_index[token] for token in y_tokens]\n",
    "        X.append(x_indices)\n",
    "        Yin.append(y_indices[:-1])\n",
    "        Yout.append(y_indices[1:])\n",
    "    return X, Yin, Yout\n",
    "\n",
    "# Split into train, dev, test\n",
    "import random\n",
    "random.seed(123)\n",
    "random.shuffle(data)\n",
    "\n",
    "n1 = int(0.8 * len(data))\n",
    "n2 = int(0.9 * len(data))\n",
    "\n",
    "X_train, Yin_train, Yout_train = build_dataset(data[:n1])\n",
    "X_dev, Yin_dev, Yout_dev = build_dataset(data[n1:n2])\n",
    "X_test, Yin_test, Yout_test = build_dataset(data[n2:])\n",
    "len(X_train), len(Yin_train), len(Yout_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Walking through simple attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-1.1620, -0.3774, -0.8762,  0.2412, -0.9882, -1.4356],\n",
       "        [ 1.5840, -1.8631,  0.6589, -1.2707, -0.0724,  0.6845]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "B = 2\n",
    "Tx = 3\n",
    "D = 6\n",
    "\n",
    "# This is like the encoder hidden states\n",
    "keys = torch.randn((B, Tx, D))\n",
    "values = keys\n",
    "\n",
    "# This is like the current hidden state of the decoder\n",
    "query = torch.randn((B, 1, D))\n",
    "\n",
    "# Dot product attention\n",
    "# (B, Tx, D) x (B, D, 1) -> (B, Tx, 1)\n",
    "dot = torch.bmm(keys, query.transpose(1, 2)) \n",
    "weights = torch.softmax(dot, dim=1)\n",
    "\n",
    "out = torch.sum(weights * values, dim=1) # (B, D)\n",
    "\n",
    "out\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.0000],\n",
       "        [1.0000]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights.sum(dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class DotAttention(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(DotAttention, self).__init__()\n",
    "        \n",
    "    def forward(self, query, keys, values):\n",
    "        # query: (B, Ty, D)\n",
    "        # keys: (B, Tx, D)\n",
    "        # values: (B, Tx, D)\n",
    "        dot = torch.bmm(keys, query.transpose(1, 2)) \n",
    "        weights = torch.softmax(dot, dim=1)\n",
    "        out = torch.bmm(weights.transpose(1, 2), values)\n",
    "        return out, weights\n",
    "\n",
    "\n",
    "class EncoderDecoder(nn.Module):\n",
    "    def __init__(self, vocab_size, embed_size, hidden_size):\n",
    "        super(EncoderDecoder, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.embed = nn.Embedding(vocab_size, embed_size)\n",
    "        self.encoder = nn.GRU(embed_size, hidden_size, batch_first=True)\n",
    "        self.decoder = nn.GRU(embed_size, hidden_size, batch_first=True)\n",
    "        self.attention = DotAttention()\n",
    "        self.query = nn.Linear(hidden_size, hidden_size)\n",
    "        self.out = nn.Linear(hidden_size*2, vocab_size)\n",
    "        \n",
    "    def forward(self, X, Yin):\n",
    "        # Encode\n",
    "        X_embed = self.embed(X)\n",
    "        Henc, henc_last = self.encoder(X_embed)\n",
    "        \n",
    "        # Decode\n",
    "        Yin_embed = self.embed(Yin)\n",
    "        Hdec, _ = self.decoder(Yin_embed, henc_last)\n",
    "        \n",
    "        # Attention\n",
    "        query = self.query(Hdec)\n",
    "        context, _ = self.attention(query, Henc, Henc)\n",
    "        \n",
    "        # Combine\n",
    "        out = torch.cat([Hdec, context], dim=2)\n",
    "        out = self.out(out)\n",
    "        return out\n",
    "    \n",
    "    def generate(self, X, max_len=20):\n",
    "        # Encode\n",
    "        X_embed = self.embed(X)\n",
    "        Henc, h = self.encoder(X_embed)\n",
    "        # Initialize with start token\n",
    "        token = torch.tensor([[token_to_index['[S]']]]).to(X.device)\n",
    "        result = []\n",
    "        attentions = []\n",
    "        for i in range(max_len):\n",
    "            token_embed = self.embed(token)\n",
    "            Hdec, h = self.decoder(token_embed, h)\n",
    "\n",
    "            # Attention\n",
    "            query = self.query(Hdec)\n",
    "            context, scores = self.attention(query, Henc, Henc)\n",
    "\n",
    "            # Output\n",
    "            out = torch.cat([Hdec, context], dim=2)\n",
    "            out = self.out(out)\n",
    "\n",
    "            # Select next token (greedy decoding)\n",
    "            token = torch.argmax(out, dim=2)\n",
    "            result.append(token.item())\n",
    "            attentions.append(scores)\n",
    "\n",
    "            if token[0].item() == token_to_index['[S]']:\n",
    "                break\n",
    "\n",
    "        return result, attentions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Batching and padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[18, 20,  7,  2, 14, 20, 19, 17,  0, 22, 27, 27],\n",
       "         [10, 20,  0,  2,  8, 15,  2, 23,  4, 19, 13,  4],\n",
       "         [18,  9,  0, 16, 12, 24,  8, 25, 17, 14,  0, 11]]),\n",
       " tensor([[26,  0, 19, 14,  7, 18, 27],\n",
       "         [26, 13,  4,  2,  8,  0, 10],\n",
       "         [26,  0, 17,  8, 12,  0, 18]]),\n",
       " tensor([[ 0, 19, 14,  7, 18, 26, 27],\n",
       "         [13,  4,  2,  8,  0, 10, 26],\n",
       "         [ 0, 17,  8, 12,  0, 18, 26]]))"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def pad_batch(X, Yin, Yout):\n",
    "    max_len_x = max(len(x) for x in X)\n",
    "    max_len_y = max(len(y) for y in Yin)\n",
    "    X_pad = torch.tensor([x + [token_to_index['[PAD]']] * (max_len_x - len(x)) for x in X])\n",
    "    Yin_pad = torch.tensor([y + [token_to_index['[PAD]']] * (max_len_y - len(y)) for y in Yin])\n",
    "    Yout_pad = torch.tensor([y + [token_to_index['[PAD]']] * (max_len_y - len(y)) for y in Yout])\n",
    "    return X_pad, Yin_pad, Yout_pad\n",
    "\n",
    "xp, yip, yop = pad_batch(X_train[:3], Yin_train[:3], Yout_train[:3])\n",
    "xp, yip, yop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model parameters: 16444\n",
      "Epoch [1/10], Loss: 0.8871\n",
      "Epoch [1/10], Validation Loss: 0.1182\n",
      "Epoch [2/10], Loss: 0.0664\n",
      "Epoch [2/10], Validation Loss: 0.0403\n",
      "Epoch [3/10], Loss: 0.0325\n",
      "Epoch [3/10], Validation Loss: 0.0119\n",
      "Epoch [4/10], Loss: 0.0158\n",
      "Epoch [4/10], Validation Loss: 0.0094\n",
      "Epoch [5/10], Loss: 0.0120\n",
      "Epoch [5/10], Validation Loss: 0.0069\n",
      "Epoch [6/10], Loss: 0.0073\n",
      "Epoch [6/10], Validation Loss: 0.0039\n",
      "Epoch [7/10], Loss: 0.0030\n",
      "Epoch [7/10], Validation Loss: 0.0044\n",
      "Epoch [8/10], Loss: 0.0013\n",
      "Epoch [8/10], Validation Loss: 0.0012\n",
      "Epoch [9/10], Loss: 0.0008\n",
      "Epoch [9/10], Validation Loss: 0.0009\n",
      "Epoch [10/10], Loss: 0.0005\n",
      "Epoch [10/10], Validation Loss: 0.0007\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "model = EncoderDecoder(vocab_size=len(token_to_index), embed_size=32, hidden_size=32)\n",
    "# Count model parameters\n",
    "print(f\"Model parameters: {sum(p.numel() for p in model.parameters())}\")\n",
    "\n",
    "# Hyperparameters\n",
    "learning_rate = 0.001\n",
    "num_epochs = 10\n",
    "batch_size = 8\n",
    "\n",
    "# Loss function and optimizer\n",
    "# NOTE: We ignore the loss whenever the target token is a padding token\n",
    "criterion = nn.CrossEntropyLoss(ignore_index=token_to_index['[PAD]'])\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "# Learning rate schedule (cosine)\n",
    "scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_epochs)\n",
    "\n",
    "# Training loop\n",
    "for epoch in range(num_epochs):\n",
    "    # Reshuffle the data\n",
    "    perm = torch.randperm(len(X_train))\n",
    "    X_train = [X_train[i] for i in perm]\n",
    "    Yin_train = [Yin_train[i] for i in perm]\n",
    "    Yout_train = [Yout_train[i] for i in perm]\n",
    "    \n",
    "    model.train()\n",
    "    total_loss = 0\n",
    "    for i in range(0, len(X_train), batch_size):\n",
    "        X_batch = X_train[i:i+batch_size]\n",
    "        Yin_batch = Yin_train[i:i+batch_size]\n",
    "        Yout_batch = Yout_train[i:i+batch_size]\n",
    "\n",
    "        # Pad batch\n",
    "        X_batch, Yin_batch, Yout_batch = pad_batch(X_batch, Yin_batch, Yout_batch)\n",
    "\n",
    "        # Forward\n",
    "        outputs = model(X_batch, Yin_batch)\n",
    "        outputs = outputs.view(-1, len(token_to_index)) # [batch_size * seq_len, vocab_size]\n",
    "        Y_batch = Yout_batch.view(-1)\n",
    "        loss = criterion(outputs, Y_batch)\n",
    "\n",
    "        # Backward pass and optimization\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "\n",
    "        total_loss += loss.item()\n",
    "        \n",
    "    # Scheduler step\n",
    "    scheduler.step()\n",
    "\n",
    "    avg_loss = total_loss / (len(X_train) // batch_size)\n",
    "    print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {avg_loss:.4f}')\n",
    "\n",
    "    # Evaluate validation loss\n",
    "    eval_loss = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for i in range(0, len(X_dev), batch_size):\n",
    "            X_batch = X_dev[i:i+batch_size]\n",
    "            Yin_batch = Yin_dev[i:i+batch_size]\n",
    "            Yout_batch = Yout_dev[i:i+batch_size]\n",
    "            # Pad batch \n",
    "            X_batch, Yin_batch, Yout_batch = pad_batch(X_batch, Yin_batch, Yout_batch)\n",
    "            outputs = model(X_batch, Yin_batch)\n",
    "\n",
    "            outputs = outputs.view(-1, len(token_to_index))\n",
    "            Y_batch = Yout_batch.view(-1)\n",
    "            loss = criterion(outputs, Y_batch)\n",
    "\n",
    "            eval_loss += loss.item()\n",
    "    avg_eval_loss = eval_loss / (len(X_dev) // batch_size)\n",
    "    print(f'Epoch [{epoch+1}/{num_epochs}], Validation Loss: {avg_eval_loss:.4f}')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kaidence -> ecnediak\n",
      "Correct\n",
      "\n",
      "eretria -> airtere\n",
      "Correct\n",
      "\n",
      "nadiah -> haidan\n",
      "Correct\n",
      "\n",
      "price -> ecirp\n",
      "Correct\n",
      "\n",
      "naja -> ajan\n",
      "Correct\n",
      "\n",
      "masaki -> ikasam\n",
      "Correct\n",
      "\n",
      "rook -> koor\n",
      "Correct\n",
      "\n",
      "yanitza -> aztinay\n",
      "Correct\n",
      "\n",
      "ozzie -> eizzo\n",
      "Correct\n",
      "\n",
      "lazzaro -> orazzal\n",
      "Correct\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Generate\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    for i in range(10):\n",
    "        x = torch.tensor([X_test[i]])\n",
    "        y, _ = model.generate(x)\n",
    "        y = ''.join([index_to_token[i] for i in y])\n",
    "        # Remove [S] from the end of y\n",
    "        y = y.split('[S]')[0]\n",
    "\n",
    "        # Print original and generated names\n",
    "        print(data[n2+i], end=' -> ')\n",
    "        print(y)\n",
    "        # Check if it was reversed correctly\n",
    "        correct = data[n2+i][::-1] == y\n",
    "        print('Correct' if correct else 'Incorrect')\n",
    "        print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualize attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "romulus -> sulumor\n"
     ]
    }
   ],
   "source": [
    "# Visualize attention for an example\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Random example index\n",
    "index = torch.randint(0, len(X_test), (1,)).item()\n",
    "\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    x = torch.tensor([X_test[index]])\n",
    "    y, attentions = model.generate(x)\n",
    "\n",
    "    # Print original and generated names\n",
    "    y = ''.join([index_to_token[i] for i in y])\n",
    "    # Remove [S] from the end of y\n",
    "    y = y.split('[S]')[0]\n",
    "    print(data[n2+index], end=' -> ')\n",
    "    print(y)\n",
    "\n",
    "attentions = torch.cat(attentions, dim=2).squeeze().cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAJCCAYAAAD5gwG8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABdDElEQVR4nO3deXhMZ/sH8O9km+wjJSKSaUIogthip0GRKlpFrS8RW+1bFVEVoZWi1bT2pRRtrLW9L7VFWlVaS6wRtZMisSeaMInM8/sjv0yNTFY5M2fM93Nd52rnbM89kzH33M/zzDkKIYQAERGRRKxMHQAREb3amGiIiEhSTDRERCQpJhoiIpIUEw0REUmKiYaIiCTFRENERJJioiEiIkkx0RARkaSYaEgSCoUC06ZNM3UY+Zo2bRoUCoWpwzAYh6+vL/r162f0WEzVLr3amGhkaOHChVAoFGjYsKHB7efOncO0adNw7do1g8d+//330gb4/3bu3CmrZBIQEIDXX38d+V1VqWnTpvDw8MCzZ8+MGJm8HDp0CNOmTcOjR49MHQpZCkGy06RJE+Hr6ysAiIsXL+bavnHjRgFAxMbG5tpWvXp1ERQUJH2QQojhw4eLvN5CT548EZmZmUaJI8cXX3whAIhff/3V4ParV68KhUIhRo4cKYQQIjMzUzx58sSYIRoUHh6e63V8+vSpyMjIkKS9OXPmCADi6tWrubZJ2S5ZLlY0MnP16lUcOnQIc+fOhbu7O3788UdTh1Qs9vb2sLGxMWqbvXr1gkKhQHR0tMHta9euhRACvXv3BgDY2NjA3t7emCEWmlKphK2trcW0S684U2c60jdjxgzh5uYmNBqNGDp0qKhcubLe9pUrVwoAuZbY2Fjh4+OTa/3z1c3Dhw/F6NGjhbe3t7CzsxN+fn7iiy++EFlZWbp9rl69KgCIOXPmiCVLloiKFSsKOzs7ERgYKI4cOaLbLyQkxGAcOQCI8PBwvdjj4uLE22+/LVxcXISTk5No1aqVOHz4sMHnd/DgQTF27FhRpkwZ4ejoKDp16iTu3LlT4OsXFBQkSpcubfBbeY0aNYSfn5/usaFKYs+ePaJp06ZCpVIJJycn8cYbb4iwsLBc8b1YDcTGxuaqMg8cOCC6du0q1Gq1sLOzE97e3mLMmDEiPT1d71hDcfj4+IiQkBDdY0Ovdc6SE8upU6dESEiIqFChglAqlcLDw0OEhoaKe/fu5Worr3O82K4QQly+fFl07dpVuLm5CQcHB9GwYUPxv//9z+DzX79+vfjss8+El5eXUCqVolWrVgarcrIsxv3KSQX68ccf0blzZ9jZ2aFnz55YtGgRjh49ivr16wMA3nzzTYwaNQrffvstJk+ejGrVqgEAqlWrhqioKIwcORLOzs745JNPAAAeHh4AgPT0dAQFBeHmzZv48MMP8frrr+PQoUMICwvD7du3ERUVpRdHdHQ0Hj9+jA8//BAKhQKzZ89G586dceXKFdja2uLDDz/ErVu3sHfvXqxZs6bA5xUfH4/mzZvD1dUVEyZMgK2tLZYsWYIWLVrg119/zTUeNXLkSLi5uSE8PBzXrl1DVFQURowYgfXr1+fbTu/evTF48GDs3r0bHTp00K0/c+YMzp49i6lTp+YbY4cOHRAQEIDp06dDqVTi0qVL+P333wt8foZs3LgR6enpGDp0KEqXLo0jR45g3rx5+Pvvv7Fx48YincvQazxlyhTcuXMHzs7OAIC9e/fiypUrCA0NRbly5RAfH4+lS5ciPj4ef/zxBxQKBTp37owLFy5g7dq1+Prrr1GmTBkAgLu7u8F2k5OT0aRJE6Snp2PUqFEoXbo0Vq1ahXfffRebNm3C+++/r7f/F198ASsrK4wfPx4pKSmYPXs2evfujT///LNIz5deMabOdPSvY8eOCQBi7969QgghtFqt8Pb2FqNHj9bbrzhjNDNmzBBOTk7iwoULeusnTZokrK2txY0bN4QQ/1Y0pUuXFg8ePNDtt23bNgFA/Pe//9Wty2+MBi9UNJ06dRJ2dnbi8uXLunW3bt0SLi4u4s0339Sty6kYWrduLbRarW792LFjhbW1tXj06JHB9nI8ePBAKJVK0bNnz1zPE4D466+/dOterCS+/vprAUDcvXs3z/MXpaJ5sXIRQojIyEihUCjE9evX84xDCMOVxfNmz54tAIjVq1fn297atWsFAHHgwAHduvzGaF5sd8yYMQKA+O2333TrHj9+LCpUqCB8fX111XDO869WrZrQaDS6fb/55hsBQJw5cybP50KvPo7RyMiPP/4IDw8PtGzZEkD2FOHu3btj3bp1yMrKeqlzb9y4Ec2bN4ebmxvu3bunW1q3bo2srCwcOHBAb//u3bvDzc1N97h58+YAgCtXrhS57aysLOzZswedOnVCxYoVdes9PT3Rq1cvHDx4EKmpqXrHDB48WG/Kb/PmzZGVlYXr16/n25abmxveeecdbN++HWlpaQAAIQTWrVuHwMBAvPHGG3keW6pUKQDAtm3boNVqi/o0c3FwcND9f1paGu7du4cmTZpACIETJ04U+7yxsbEICwvDyJEj0adPH4PtPX36FPfu3UOjRo0AAHFxccVqa+fOnWjQoAGaNWumW+fs7IzBgwfj2rVrOHfunN7+oaGhsLOz0z1+mfcNvTqYaGQiKysL69atQ8uWLXH16lVcunQJly5dQsOGDZGcnIyYmJiXOv/Fixexa9cuuLu76y2tW7cGANy5c0dv/9dff13vcU7SefjwYZHbvnv3LtLT01GlSpVc26pVqwatVovExMQSa793795IS0vDtm3bAGRP57127ZpuEkBeunfvjqZNm2LgwIHw8PBAjx49sGHDhmInnRs3bqBfv3547bXX4OzsDHd3dwQFBQEAUlJSinXOv//+Wxfn3Llz9bY9ePAAo0ePhoeHBxwcHODu7o4KFSq8VHvXr1/P8++Ws/15Jfm+oVcHx2hkYv/+/bh9+zbWrVuHdevW5dr+448/om3btsU+v1arRZs2bTBhwgSD21/8pm9tbW1wP2GkO3+/TPsdOnSASqVCdHQ0evXqhejoaFhbW6NHjx75Hufg4IADBw4gNjYWO3bswK5du7B+/Xq0atUKe/bsgbW1dZ4/8Hyx4szKykKbNm3w4MEDTJw4EVWrVoWTkxNu3ryJfv36FSt5ZWRkoGvXrlAqldiwYUOuWX3dunXDoUOH8PHHH6N27dpwdnaGVqvF22+/XSIVWmGY+n1D8sREIxM//vgjypYtiwULFuTatnnzZmzZsgWLFy+Gg4NDvr9mz2ubn58f/vnnH10FUxIK+6t6d3d3ODo64q+//sq17fz587CysoJarS6xuJRKJbp27YrVq1cjOTkZGzduRKtWrVCuXLkCj7WyssJbb72Ft956C3PnzsXMmTPxySefIDY2Fq1bt9Z9Q3/xx44vfrM/c+YMLly4gFWrVqFv37669Xv37i328xo1ahROnjyJAwcO6CZ55Hj48CFiYmIQERGhN+Hh4sWLuc5TlKsh+Pj45Pl3y9lOVBB2ncnAkydPsHnzZnTo0AFdu3bNtYwYMQKPHz/G9u3bAQBOTk4Acn/Y5WwztL5bt244fPgwdu/enWvbo0ePivVL+fzieJ61tTXatm2Lbdu26V3NIDk5GdHR0WjWrBlcXV2L3H5+evfujczMTHz44Ye4e/dugd1mQHbX04tq164NANBoNACyEzYAvTGtrKwsLF26VO+4nG/2z3+TF0Lgm2++KdoT+X8rV67EkiVLsGDBAjRo0CDXdkPtAcg1mxAo/N8NAN555x0cOXIEhw8f1q1LS0vD0qVL4evrC39//yI8C7JUrGhkYPv27Xj8+DHeffddg9sbNWqk+/Fm9+7dUbt2bVhbW2PWrFlISUmBUqlEq1atULZsWdSrVw+LFi3CZ599hkqVKqFs2bJo1aoVPv74Y2zfvh0dOnRAv379UK9ePaSlpeHMmTPYtGkTrl27ppvqWlj16tUDkP1NOzg4ON/uqc8++wx79+5Fs2bNMGzYMNjY2GDJkiXQaDSYPXt20V6wQggKCoK3tze2bdsGBwcHdO7cucBjpk+fjgMHDqB9+/bw8fHBnTt3sHDhQnh7e+sGw6tXr45GjRohLCwMDx48wGuvvYZ169blStRVq1aFn58fxo8fj5s3b8LV1RU//fRTscYq7t27h2HDhsHf3x9KpRI//PCD3vb3338frq6uePPNNzF79mxkZmbCy8sLe/bswdWrV3OdL+fv9sknn6BHjx6wtbVFx44ddQnoeZMmTcLatWvRrl07jBo1Cq+99hpWrVqFq1ev4qeffoKVFb+rUiGYbsIb5ejYsaOwt7cXaWlpee7Tr18/YWtrq/vx3bJly0TFihWFtbW13rTapKQk0b59e+Hi4pLrB5uPHz8WYWFholKlSsLOzk6UKVNGNGnSRHz55Ze6Hzg+/4PNF+GFKcvPnj0TI0eOFO7u7kKhUBTqB5vBwcHC2dlZODo6ipYtW4pDhw7p7ZMzffjo0aN66w1NHy7Ixx9/LACIbt26Gdz+4rTimJgY8d5774ny5csLOzs7Ub58edGzZ89cU8IvX74sWrdurftR5OTJk8XevXtzxXfu3DnRunVr4ezsLMqUKSMGDRokTp06JQCIlStX5hmHEPrTjHP+JnktOdOU//77b/H++++LUqVKCZVKJT744ANx69Ytg3+LGTNmCC8vL2FlZVXoH2yWKlVK2NvbiwYNGuT5g82NGzfqrc+J/fnnS5ZHIQRH6YiISDqse4mISFJMNEREJCkmGiIikhQTDRERSYqJhoiIJMVEQ0REkmKiATBt2rQiXZaDpNGvXz/4+voWal+tVosaNWrg888/lzSmFi1aoEWLFpK2UdK+//57KBQKHDt2zNShFEpOvM9fNUKuDMXaqFGjPK8hSNnMJtGcOXMGXbt2hY+PD+zt7eHl5YU2bdpg3rx5evvNnDkTW7duNU2QxdCvXz8oFArdolQq8cYbb2Dq1Kl4+vSpqcOTrbVr1yIxMREjRowwdSglztzew5Zu4sSJWLBgAZKSkkwdimyZRaI5dOgQAgMDcerUKQwaNAjz58/HwIEDYWVllevaUcX5RzplyhQ8efKkBCMuGqVSiTVr1mDNmjWYO3cufH19MWPGDAwYMMBkMcndnDlz0KNHD6hUKknb2bNnD/bs2SNpGy+ytETTp08fPHnyxGwv0Pnee+/B1dUVCxcuNHUosmUW1zr7/PPPoVKpcPToUd3NqXK8eB+VokhLS4OTkxNsbGxyXXLdmGxsbPCf//xH93jYsGFo0qQJ1q5di7lz5+a6Uq8c5Lx2pnDixAmcOnUKX331leRtPX8TL5KGtbV1nrcXKCnPnj2DVquV5O9pZWWlu1p4REQEu+ENMIuK5vLly6hevXquJAMAZcuW1f2/QqFAWloaVq1apeuK6tevH4B/x2HOnTuHXr16wc3NTXehRENjNAqFAiNGjMDWrVtRo0YNKJVKVK9eHbt27coVwy+//ILAwEDY29vDz88PS5YsealxH4VCgWbNmkEIkevOhD///DOaN28OJycnuLi4oH379oiPj9dt//LLL6FQKAzeiTIsLAx2dnZ6F3b8888/8fbbb0OlUsHR0RFBQUH4/fff9Y7L77VLSkpCaGgovL29oVQq4enpiffee0+vD3vbtm1o3749ypcvD6VSCT8/P8yYMaPYdw3dunUr7Ozs8OabbxqM89KlS+jXrx9KlSoFlUqF0NBQpKen6+377NkzzJgxA35+flAqlfD19cXkyZN1V2nOYWiMZt68eahevTocHR3h5uaGwMBAREdH6+1z8+ZN9O/fHx4eHrr3zooVKwp8bvm9h4HsJNuuXTu4urrC2dkZb731Fv74448Cz/vw4UM0aNAA3t7eusv+azQahIeHo1KlSlAqlVCr1ZgwYUKu16Cw/xYeP36MMWPGwNfXF0qlEmXLlkWbNm0KvLunoXEPX19fdOjQAQcPHkSDBg1gb2+PihUrYvXq1QU+12vXrkGhUODLL79EVFSU7m+cczfQ8+fPo2vXrnjttddgb2+PwMBA3ZXRnxcfH49WrVrBwcEB3t7e+Oyzz/K8r0+bNm1w/fp1nDx5ssD4LJFZVDQ+Pj44fPgwzp49ixo1auS535o1azBw4EA0aNAAgwcPBvDvZd1zfPDBB6hcuTJmzpxZ4M2YDh48iM2bN2PYsGFwcXHBt99+iy5duuDGjRsoXbo0gOx/+G+//TY8PT0RERGBrKwsTJ8+He7u7i/1nHP+0T1/O+U1a9YgJCQEwcHBmDVrFtLT07Fo0SI0a9YMJ06cgK+vL7p164YJEyZgw4YN+Pjjj/XOuWHDBrRt21Z3zv3796Ndu3aoV68ewsPDYWVlhZUrV6JVq1b47bffcl2O3tBr16VLF8THx2PkyJHw9fXFnTt3sHfvXty4cUM3sP/999/D2dkZ48aNg7OzM/bv34+pU6ciNTUVc+bMKfJrc+jQIdSoUQO2trYGt3fr1g0VKlRAZGQk4uLisHz5cpQtWxazZs3S7TNw4ECsWrUKXbt2xUcffYQ///wTkZGRSEhIwJYtW/Jse9myZRg1ahS6du2K0aNH4+nTpzh9+jT+/PNP9OrVC0D27Q8aNWqk+4B2d3fHzz//jAEDBiA1NRVjxozJ8/z5vYfj4+PRvHlzuLq6YsKECbC1tcWSJUvQokUL/Prrr2jYsKHBc967d093E7Zff/0Vfn5+0Gq1ePfdd3Hw4EEMHjwY1apVw5kzZ/D111/jwoULubruCvNvYciQIdi0aRNGjBgBf39/3L9/HwcPHkRCQgLq1q2b53POy6VLl9C1a1cMGDAAISEhWLFihe7K49WrVy/w+JUrV+Lp06cYPHgwlEolXnvtNcTHx6Np06bw8vLCpEmT4OTkhA0bNqBTp0746aef8P777wPI/gLVsmVLPHv2TLff0qVL9W6X/bycK2L//vvvqFOnTpGf6yvPpJf0LKQ9e/YIa2trYW1tLRo3biwmTJggdu/erbvi8POcnJxyXX1WiH+vkNuzZ888tz0PgLCzsxOXLl3Srcu58u68efN06zp27CgcHR3FzZs3desuXrwobGxscp3TkJCQEOHk5CTu3r0r7t69Ky5duiS+/PJLoVAoRI0aNYRWqxVCZF95uVSpUmLQoEF6xyclJQmVSqW3vnHjxqJevXp6+x05ckQAEKtXrxZCCKHVakXlypVFcHCwrg0hhEhPTxcVKlQQbdq0KfC1e/jwYZ5Xen5eenp6rnUffvihcHR0FE+fPtV7LXx8fPI9lxBCeHt7iy5duuRanxNn//799da///77onTp0rrHJ0+eFADEwIED9fYbP368ACD279+vWxcUFKR3Bez33ntPVK9ePd/4BgwYIDw9PXVX2s7Ro0cPoVKpDL4ez8vrPdypUydhZ2cnLl++rFt369Yt4eLiIt58803duuevgH379m1RvXp1UbFiRXHt2jXdPmvWrBFWVlbit99+02tj8eLFAoD4/fffdesK+29BpVKJ4cOH5/vcDMmJN+cK0kJkX0UagDhw4IBu3Z07d4RSqRQfffRRvufLuWK0q6uruHPnjt62t956S9SsWVPvfafVakWTJk1E5cqVdevGjBkjAIg///xTr32VSpUr1hx2dnZi6NChhX3aFsUsus7atGmDw4cP491338WpU6cwe/ZsBAcHw8vLy2DJm58hQ4YUet/WrVvrVUQBAQFwdXXVdWdlZWVh37596NSpE8qXL6/br1KlSmjXrl2h20lLS4O7uzvc3d1RqVIljB8/Hk2bNsW2bdt03W979+7Fo0eP0LNnT9y7d0+3WFtbo2HDhoiNjdWdr3v37jh+/DguX76sW7d+/XoolUq89957AICTJ0/i4sWL6NWrF+7fv687X1paGt566y0cOHAgVzfBi6+dg4MD7Ozs8Msvv+R7n5XnvwU+fvwY9+7dQ/PmzZGenq67U2NR3L9/X6/Se9GLcTZv3hz3799HamoqAGDnzp0AgHHjxunt99FHHwEAduzYkee5S5Uqhb///htHjx41uF0IgZ9++gkdO3aEEELvbxUcHIyUlJQCu5IMycrKwp49e9CpUydUrFhRt97T0xO9evXCwYMHdc8vx99//42goCBkZmbiwIEDeoPtGzduRLVq1VC1alW9GFu1agUAeu8noOB/CzmvzZ9//olbt24V+fkZ4u/vj+bNm+seu7u7o0qVKrm6k/PSpUsXvZ6FBw8eYP/+/ejWrZvufXjv3j3cv38fwcHBuHjxIm7evAkg+z3SqFEjvare3d093xvoubm54d69e0V9mhbBLLrOAKB+/frYvHkzMjIycOrUKWzZsgVff/01unbtipMnTxb6Tn8VKlQodJuvv/56rnVubm66D9U7d+7gyZMnqFSpUq79DK3Li729Pf773/8CyP5wmD17Nu7cuaP3AZ1zS96cD4IXPX+Hyg8++ADjxo3D+vXrMXnyZAghsHHjRl3f/vPnCwkJyTOulJQUvQ/0F187pVKJWbNm4aOPPoKHhwcaNWqEDh06oG/fvnq3TY6Pj8eUKVOwf//+XB+GKSkpeb8w+RD5dHu++HfLeQ4PHz6Eq6srrl+/Disrq1x/o3LlyqFUqVIGx7dyTJw4Efv27UODBg1QqVIltG3bFr169ULTpk0BAHfv3sWjR4+wdOnSXHfdzFGcCSx3795Feno6qlSpkmtbtWrVoNVqkZiYqNel1KdPH9jY2CAhISHXbawvXryIhISEPLt4X4yxoH8LADB79myEhIRArVajXr16eOedd9C3b1+9xFgUhWkzPy++Xy9dugQhBD799FN8+umnBo+5c+cOvLy8cP36dYNdkYZe/xxCCE4EyIPZJJocdnZ2qF+/PurXr4833ngDoaGh2LhxI8LDwwt1fF59rIbkNRMmvw+54rC2tkbr1q11j4ODg1G1alV8+OGHuootp7pYs2ZNrg8NAHqz5sqXL4/mzZtjw4YNmDx5Mv744w/cuHFDb4wi53xz5szR3a74Rc7OznqPDb12Y8aMQceOHbF161bs3r0bn376KSIjI7F//37UqVMHjx49QlBQEFxdXTF9+nT4+fnB3t4ecXFxmDhxYp6Dq/kpXbp0vh82hf27FedDoVq1avjrr7/wv//9D7t27cJPP/2EhQsXYurUqYiIiNA9n//85z95JvGAgIAit1scnTt3xurVq/HNN98gMjJSb5tWq0XNmjUxd+5cg8eq1Wq9x4V5Tbt164bmzZtjy5Yt2LNnD+bMmYNZs2Zh8+bNRarwi9Jmfl58v+b8bcaPH4/g4GCDxxTlC+KLHj16VOS71FoKs0s0zwsMDAQA3L59W7fOmN8oypYtC3t7e1y6dCnXNkPrCsvT0xNjx45FREQE/vjjDzRq1EjXbVG2bFm9pJSX7t27Y9iwYfjrr7+wfv16ODo6omPHjrrtOedzdXUt1Pny4+fnh48++ggfffQRLl68iNq1a+Orr77CDz/8gF9++QX379/H5s2b9WaJGbrFcGFVrVr1pY738fGBVqvFxYsXUa1aNd365ORkPHr0qMDfczg5OaF79+7o3r07MjIy0LlzZ3z++ecICwuDu7s7XFxckJWVVezX1dB72N3dHY6OjroZY887f/48rKysciWHkSNHolKlSpg6dSpUKhUmTZqk2+bn54dTp07hrbfeKtF/M56enhg2bBiGDRuGO3fuoG7duvj888+LlWhKWk5lZWtrW+DfxsfHR1f1P8/Q6w9kzzLMyMjQez/Rv8xijCY2Ntbgt5icvvbny1knJyc8evTIKHHlVCJbt27V65e+dOkSfv7555c698iRI+Ho6IgvvvgCQHaV4+rqipkzZyIzMzPX/nfv3tV73KVLF1hbW2Pt2rXYuHEjOnTooPe7l3r16sHPzw9ffvkl/vnnnwLPZ0h6enquqxf4+fnBxcVFN0U251vp83+/jIyMl/pxW+PGjXH27Nlc03AL65133gEAREVF6a3P+Xbfvn37PI+9f/++3mM7Ozv4+/tDCIHMzExYW1ujS5cu+Omnn3D27NlcxxfmdTX0Hra2tkbbtm2xbds2vWnAycnJiI6ORrNmzfS6T3N8+umnGD9+PMLCwrBo0SLd+m7duuHmzZtYtmxZrmOePHmCtLS0AuN8XlZWVq5u0LJly6J8+fLF/juVtLJly6JFixZYsmSJ3pfTHM//bd555x388ccfOHLkiN72H3/80eC5jx8/DgBo0qRJCUf9ajCLimbkyJFIT0/H+++/j6pVqyIjIwOHDh3C+vXr4evri9DQUN2+9erVw759+zB37lyUL18eFSpUyHPaZ0mYNm0a9uzZg6ZNm2Lo0KHIysrC/PnzUaNGjZeaU1+6dGmEhoZi4cKFSEhIQLVq1bBo0SL06dMHdevWRY8ePeDu7o4bN25gx44daNq0KebPn687vmzZsmjZsiXmzp2Lx48fo3v37nrnt7KywvLly9GuXTtUr14doaGh8PLyws2bNxEbGwtXV1fduFFeLly4gLfeegvdunWDv78/bGxssGXLFiQnJ6NHjx4Asv/hubm5ISQkBKNGjYJCocCaNWteqvvxvffew4wZM/Drr7+ibdu2RT6+Vq1aCAkJwdKlS3Vde0eOHMGqVavQqVMntGzZMs9j27Zti3LlyqFp06bw8PBAQkIC5s+fj/bt28PFxQUA8MUXXyA2NhYNGzbEoEGD4O/vjwcPHiAuLg779u3DgwcP8o0vr/fwZ599hr1796JZs2YYNmwYbGxssGTJEmg0GsyePTvP882ZMwcpKSkYPnw4XFxc8J///Ad9+vTBhg0bMGTIEMTGxqJp06bIysrC+fPnsWHDBuzevVvXY1AYjx8/hre3N7p27YpatWrB2dkZ+/btw9GjR43yw9rCWrBgAZo1a4aaNWti0KBBqFixIpKTk3H48GH8/fffOHXqFABgwoQJWLNmDd5++22MHj1aN73Zx8cHp0+fznXevXv34vXXX+fU5ryYYqpbUf3888+if//+omrVqsLZ2VnY2dmJSpUqiZEjR4rk5GS9fc+fPy/efPNN4eDgIADoponmTH29e/durvPnNb3Z0FRNHx+fXFNPY2JiRJ06dYSdnZ3w8/MTy5cvFx999JGwt7cv8LnlTG825PLly8La2lqvvdjYWBEcHCxUKpWwt7cXfn5+ol+/fuLYsWO5jl+2bJkAIFxcXMSTJ08MtnHixAnRuXNnUbp0aaFUKoWPj4/o1q2biImJ0e2T12t37949MXz4cFG1alXh5OQkVCqVaNiwodiwYYPefr///rto1KiRcHBwEOXLl9dNTwcgYmNj9V6LwkxvFkKIgIAAMWDAAL11ecVpaPpsZmamiIiIEBUqVBC2trZCrVaLsLAwvWmvQuSe3rxkyRLx5ptv6l4vPz8/8fHHH4uUlBS945KTk8Xw4cOFWq0Wtra2oly5cuKtt94SS5cuLfC55fUeFkKIuLg4ERwcLJydnYWjo6No2bKlOHTokMHne/ToUd26rKws0bNnT2FjYyO2bt0qhBAiIyNDzJo1S1SvXl0olUrh5uYm6tWrJyIiIvSeT2H+LWg0GvHxxx+LWrVqCRcXF+Hk5CRq1aolFi5cWODzzWt6c/v27XPt++Lfw5Cc6c15Tbu/fPmy6Nu3ryhXrpywtbUVXl5eokOHDmLTpk16+50+fVoEBQUJe3t74eXlJWbMmCG+++67XLFmZWUJT09PMWXKlAKfq6VSCFHCI9sEAOjUqRPi4+MN9vPSy1uzZg2GDx+OGzduGLxiBJGxbN26Fb169cLly5fh6elp6nBkySzGaOTuxQtyXrx4ETt37jS7y8ubk969e+P111/HggULTB0KWbhZs2ZhxIgRTDL5YEVTAjw9PdGvXz9UrFgR169fx6JFi6DRaHDixAlUrlzZ1OEREZmUWUwGkLu3334ba9euRVJSEpRKJRo3boyZM2cyyRARgRUNERFJjGM0REQkKSYaIiKSlMWM0Wi1Wty6dQsuLi688B0R6RFC4PHjxyhfvjysrEr++/fTp0+RkZFR4ucFsq9OYW9vL8m5S4rFJJpbt27luhYUEdHzEhMT4e3tXaLnfPr0aZEu5ltU5cqVw9WrV2WdbCwm0eRcHiQxMdHgNaGIyHKlpqZCrVbrPidKklSVTI6kpCRkZGQw0chBTneZq6srEw0RGSR1t3pJn99cJg1bTKIhIjIlhUIhSSIzh2TDWWdERCQpVjREREbAioaIiEgirGiIiIzAyspKkskAWq22RM8pBVY0REQkKVY0RERGINUYjTlgRUNERJJiRUNEZASWXNEw0RARGYElJxp2nRERkaTknWj++AOIijJ1FERELy2noinpxRzIN9E8ewYsWQJs21aswzUaDVJTU/UWIiIyPvkmGhsbYOJE4OhRYNWqIh8eGRkJlUqlW3gvGiIyJVY0cqTVAlWrAoMGAf/9L/DgQZEODwsLQ0pKim5JTEyUKFAiIsqPvGad/forkJgI9OoF5NxOtXlzYPBg4Nw5oFmz7ARUiFutKpVKKJVKiQMmIiocqS5BYw7kU9FkZABbtwJ9+wIffAB8+WX2+s6ds5eJE4F//ilUkiEiIvmQz6e2nR3w9ddAfDzg4QF89x1QrRqwciVQowZQpgxw8qSpoyQiKhZLHqORV9cZkJ1c5s7Nrl4mTQLWrctOPrduAb6+2d1nRERkNuSXaADA3j57Wb4cOH0a+O237N/T9O9v6siIiIrFnCqQkibPRAMAQgAKBRAQkL0MHAhwcJ+IzJQlJxr5jNG86MU/iJ2daeIgIqKXIt+K5kUW+k2AiF4NrGiIiIgkYj4VDRGRGWNFQ0REJBFWNERERqBQKGBVwlc20Wq1JXo+qbCiISIiSbGiISIyAinGaMxlzIeJhojICJhoLIibm5tJ/zj//POPydrOUdL9xMVhxx/gElkMi0s0RESmYMkVjem/2hIR0SuNFQ0RkRGwoiEiIpIIKxoiIiNgRUNERCQRVjREREZgZWUli58WmAITDRGREbDrjIiISCKsaIiIjIAVDRERkURY0RARGQErGiIiIonIPtGcOnUKN2/eNHUYREQvJaeiKenFHMg60WzduhXvvPMOFi1aVOTL62s0GqSmpuotRERkfLIdo9mxYwd69eqFb7/9Fu3bt4ezs3ORjo+MjERERIRE0RERFY0lj9EohBDC1EG86OnTp+jbty8qV66Mzz//HOnp6UhKSsK6devg7++Pt956Cy4uLvmeQ6PRQKPR6B6npqZCrVbDysqKNz6Twa+TeeMzkpPU1FSoVCqkpKTA1dVVknP7+PiU+L89rVaL69evSxJ3SZJlRSOEwNWrV1GuXDk8ePAA4eHhOHPmDC5fvozMzEyMGDECU6ZMyfccSqUSSqXSSBETEVFeTP/V1gAHBweMHDkSy5cvR4UKFXDz5k30798fiYmJ6NmzJ/bv3w+tVmvqMImICs2SJwPIsqIBgL59+yIwMBA3b95EmzZtdIklMzMTarUamZmZrFiIiMyAbBMNAPj7+8Pf3x8AcOHCBaxZswY//PADDh48yCRDRGbFkicDyDrR5Dh+/Di++uornDx5Er/++itq1Khh6pCIiKiQzCLR+Pv7Y+jQofD19YVarTZ1OERERcaKRuYcHBzQvHlzU4dBRETFYBaJhojoVWAuFUhJY6IhIjICKW7lLMPf2xsky9/REBHRq4MVDRGREVjyZABWNEREJClWNERERsCKhoiISCKsaIiIjMCSZ51ZXKJ5+PChrO/bQET0qrG4RENEZAqWPEbDRENEZASW3HXGyQBERCQpVjREREZgyV1nrGiIiEhSrGiIiIyAFQ0REZFEWNEQERkBZ50RERFJhBUNEZERWPIYDRMNEZERsOuMiIhIIvKvaLRaoIS/BRARGZsld53J7xM8MRHYtCn7/9etAwYNArKyTBsTEdErZMGCBfD19YW9vT0aNmyII0eO5Lt/VFQUqlSpAgcHB6jVaowdOxZPnz4tdHvyqmgyM4EJE4AbN4BDh4CoKGDZMsDausin0mg00Gg0usepqaklGCgRUdEoFIoSH6PRarVFPmb9+vUYN24cFi9ejIYNGyIqKgrBwcH466+/ULZs2Vz7R0dHY9KkSVixYgWaNGmCCxcuoF+/flAoFJg7d26h2pRXRWNrCyxalF3BREUBQ4YAAwZkbyviCxoZGQmVSqVb1Gp1ycdLRGRm5s6di0GDBiE0NBT+/v5YvHgxHB0dsWLFCoP7Hzp0CE2bNkWvXr3g6+uLtm3bomfPngVWQc+TV6IBACen7KVWLeDSJeDHH7PXW1kVKdmEhYUhJSVFtyQmJkoUMBFRwXLGaEp6AbJ7bJ5fnu/NeV5GRgaOHz+O1q1b69ZZWVmhdevWOHz4sMFjmjRpguPHj+sSy5UrV7Bz50688847hX7u8uo6A7Krmp07gYcPgYEDge++y17fu/e/kwKysgrsTlMqlVAqlRIHS0Rkei/22ISHh2PatGm59rt37x6ysrLg4eGht97DwwPnz583eO5evXrh3r17aNasGYQQePbsGYYMGYLJkycXOj75VTQAoFQC5coB334LODoCK1cCP/yQvW3KFGDoUMBM5o8TEQHSVjSJiYl6PThhYWElFvcvv/yCmTNnYuHChYiLi8PmzZuxY8cOzJgxo9DnkF9F87yKFYF584CPPgJmzcpOPH/9BezZA5jJtD4iIkCaH2zmnM/V1RWurq4F7l+mTBlYW1sjOTlZb31ycjLKlStn8JhPP/0Uffr0wcCBAwEANWvWRFpaGgYPHoxPPvmkUM9JnhXN8ypUyE42Y8cCHToAR44ADRuaOioiIrNjZ2eHevXqISYmRrdOq9UiJiYGjRs3NnhMenp6rmRi/f9DF4W9MoG8K5ocXl5A//6mjoKIqNjk8oPNcePGISQkBIGBgWjQoAGioqKQlpaG0NBQAEDfvn3h5eWFyMhIAEDHjh0xd+5c1KlTBw0bNsSlS5fw6aefomPHjrqEUxDzSDRERFQiunfvjrt372Lq1KlISkpC7dq1sWvXLt0EgRs3buhVMFOmTIFCocCUKVNw8+ZNuLu7o2PHjvj8888L3aZCmMtV2V5SamoqVCoVUlJSCtWXSUSWQ8rPh5xzN23aFDY2Jfvd/tmzZ/j9999l/7km/zEaIiIya+w6IyIyArmM0ZgCKxoiIpIUKxoiIiNgRUNERCQRVjREREYg5ZUB5I6JhojICNh1RkREJBGLq2iePXuGZ8+emaz9MmXKmKztHNevXzd1CLCzszN1CHBwcDB1CGRBLLnrzDyiJCIis2VxFQ0RkSlwjIaIiEgirGiIiIyAFQ0REZFEWNEQERmBQqEo8Vli5lLRMNEQERkBu86IiIgkwoqGiMgI+INNIiIiibCiISIyAo7REBERSYQVDRGREXCMRsbS09NNHQIREb0EWSeagwcPYvz48YiPjy/ysRqNBqmpqXoLEZGp5IzRlPRiDmSdaC5evIjt27dj6dKlSEhIKNKxkZGRUKlUukWtVksUJRFRwZhoZCo0NBQzZ87E7t27sWDBgiIlm7CwMKSkpOiWxMRECSMlIqK8yG4ywJUrV2Bvb4/y5csDAPr27QutVotZs2ZBq9Vi7NixqFy5coHnUSqVUCqVUodLRFQoljy9WVaJ5uHDh2jWrBlCQ0MxYsQIeHp6AgD69euHrKwsDB06FDY2Nhg0aBBq1qxp4miJiKgwZNV15ubmhujoaERHR2PZsmW4deuWbtuAAQMQEBCA1atX44cffkBGRoYJIyUiKhpLHqORVUUDAC1atMCaNWvQs2dPKBQKDBw4EJ6enkhPT0f9+vXRpUsX9OjRA3Z2dqYOlYiICkF2iQYAmjVrhrVr16JPnz64c+cOmjRpgvj4eBw4cACHDx+Gq6urqUMkIioSjtHIULNmzbBx40aMGzcO27Ztg4uLC9asWcMkQ0RkZmSbaAAgMDAQO3bswKNHj2Bvbw93d3dTh0REVCysaGTMxcUFLi4upg6DiOilWHKikdWsMyIievXIvqIhInoVsKIhIiKSCCsaIiIj4P1oiIiIJMKKhojICDhGQ0REJBGLq2hsbGxgY2O6p/3o0SOTtU1EpmUuFUhJs7hEQ0RkCuw6IyIikggrGiIiI2BFQ0REJBFWNERERsCKhoiISCKsaIiIjIAVDRERkURY0RARGYElVzRMNERERmDJiYZdZ0REJClWNERERsCKhoiISCKsaIiIjMCSKxp5J5rTp4Hbt4H0dKBVK0ClKvShGo0GGo1G9zg1NVWKCImIqADy7TrbtCk7uUyaBHTtCrRrl72ukCIjI6FSqXSLWq2WMFgiovzlVDQlvZgDeSaaEyeAoUOBWbOA/fuBW7eAN94A5s0Dtmwp1CnCwsKQkpKiWxITEyUOmoiIDJFnoklIAMqWBT74AChVCvDwyE46ajXw9ddARkaBp1AqlXB1ddVbiIhMhRWN3FhZARpN9tiMQgE8e5adbD77DDh4EDhwwNQREhEVCRON3NSvD/z9N7BgQfZjm/+fs6BQADVqAC4upouNiIiKRJ6zzvz8gO++A/r3z65mBgwAXF2BZcuAR4+yu9CIiMwIpzfLUa9egLU1MHgwEB0N2Ntnd6Vt2waUL2/q6IiIqJDkm2gUCqBHD6BxY+D8eSArCwgIALy9TR0ZEVGRsaKRMx+f7IWIiMyS/BMNEdErwJIrGnnOOiMiolcGKxoiIiOw5IqGiYaIyAgsOdGw64yIiCTFioaIyAhY0RAREUmEFQ0RkZGYSwVS0phojEwIYeoQULduXVOHgMOHD5s6BNjZ2Zk6BFhZsVOBXn1MNERERsAxGiIiIomwoiEiMgJLrmiYaIiIjMCSEw27zoiISFKsaIiIjIAVDRERkURY0RARGQErGiIishgLFiyAr68v7O3t0bBhQxw5ciTf/R89eoThw4fD09MTSqUSb7zxBnbu3Fno9ljREBEZgZWVVYlfCaI451u/fj3GjRuHxYsXo2HDhoiKikJwcDD++usvlC1bNtf+GRkZaNOmDcqWLYtNmzbBy8sL169fR6lSpQrdJhMNEZEFmTt3LgYNGoTQ0FAAwOLFi7Fjxw6sWLECkyZNyrX/ihUr8ODBAxw6dAi2trYAAF9f3yK1ya4zIiIjyBmjKemlKDIyMnD8+HG0bt1at87KygqtW7fO8/qD27dvR+PGjTF8+HB4eHigRo0amDlzJrKysgrdriwrmp07d8Lb2xsBAQGmDoWISPZSU1P1HiuVSiiVylz73bt3D1lZWfDw8NBb7+HhgfPnzxs895UrV7B//3707t0bO3fuxKVLlzBs2DBkZmYiPDy8UPHJrqJJTk7GiBEjEBUVhXPnzpk6HCKiEiFlRaNWq6FSqXRLZGRkicWt1WpRtmxZLF26FPXq1UP37t3xySefYPHixYU+h+wqGg8PD2zatAlDhgzB119/jTFjxqB69epFPo9Go4FGo9E9fjHjExEZk5TTmxMTE+Hq6qpbb6iaAYAyZcrA2toaycnJeuuTk5NRrlw5g8d4enrC1tYW1tbWunXVqlVDUlISMjIyCnW7DdlVNED2/VKWLFmCuLg4REVFIT4+vsjniIyM1MvwarVagkiJiEzP1dVVb8kr0djZ2aFevXqIiYnRrdNqtYiJiUHjxo0NHtO0aVNcunQJWq1Wt+7ChQvw9PQs9D2dZJloAKBOnTpYvnx5sZNNWFgYUlJSdEtiYqJEkRIRFUwOkwEAYNy4cVi2bBlWrVqFhIQEDB06FGlpabpZaH379kVYWJhu/6FDh+LBgwcYPXo0Lly4gB07dmDmzJkYPnx4oduUXdfZ83KSzcCBAxEVFVWkbrS8BsOIiCxZ9+7dcffuXUydOhVJSUmoXbs2du3apZsgcOPGDb3f56jVauzevRtjx45FQEAAvLy8MHr0aEycOLHQbSqEHO4tXIATJ05g4MCBqFu3LsaOHQt/f/8inyM1NRUqlQopKSl6fZnGJoeXm7dyzsZbOVMOKT8fcs790UcflfiXX41Gg6+++srkn2sFMYt3eU5lc/r0acyYMSPPaXhERCQ/ZpFogOxkM3/+fNy+fRsqlcrU4RARFYlcxmhMQdZjNC+qX78+du3aBXt7e1OHQkREhWRWiQYAkwwRmSVLvk2A2SUaIiJzZMmJxmzGaIiIyDyxoiEiMgJWNERERBJhRUNEZASsaIiIiCTCioaIyAgUCkWJX3KIFQ0RERFY0RARGYUlj9Ew0RiZHN4YJ06cMHUIRBbHkhMNu86IiEhSrGiIiIyAFQ0REZFEWNEQERkBKxoiIiKJsKIhIjICVjREREQSYUVDRGQEllzRMNEQERmBJScadp0REZGkWNEQERkBKxq5WbMG2LLF1FEQEVEJkF9Fk5YGrF6d/V+lEnjnHVNHRET00ljRyImTU3ai8fYG5swB/vtfU0dEREQvQV6JRgggMxPw9ASmTQMcHIDZs4Hdu4t8Ko1Gg9TUVL2FiMhUrKysJFnMgfyitLUFNmwAIiKAR4+AuDhgxAhg584inSYyMhIqlUq3qNVqaeIlIqJ8ySvRKBTAn38CoaFAcDCwciVw6lR2N9pnnwE//1zoU4WFhSElJUW3JCYmShg4EVH+csZoSnoxB8VKNNOnT0d6enqu9U+ePMH06dNfLqJTpwBfX6BnT6BKFaBSJeCHHwBra2DUKGDPnkKdRqlUwtXVVW8hIjIVJpoiioiIwD///JNrfXp6OiIiIl4uIgcHICsLyDl/Zibg5QUsXAgkJQEffwzs2vVybRARkdEUK9EIIQxm0lOnTuG11157uYgaNwauXwfmzct+bGub/d+MDKBePaB6dcDf/+XaICIyMkuuaIr0Oxo3Nzfdk3vjjTf0nmRWVhb++ecfDBky5OUiqlQJWLYM6N8/u7IZNAgoVQrYti27S+3bbwF2gxERmY0iJZqoqCgIIdC/f39ERERApVLpttnZ2cHX1xeNGzd++ah6984ekxk8GFi7FrCyAh4+BPbuZZIhIrNkyT/YLFKiCQkJAQBUqFABTZo0gW1Ot1ZJUyiyJwM0bgycPg08eQI0bJhd0RARkVkp1iVoKlSogNu3b+e5/fXXXy92QHp8fZlciOiVwIqmiHx9ffN9gllZWcUOiIiIXi3FSjQnTpzQe5yZmYkTJ05g7ty5+Pzzz0skMCKiV425VCAlrViJplatWrnWBQYGonz58pgzZw46d+780oEREb1KLLnrrEQvQVOlShUcPXq0JE9JRERmrlgVzYtXQhZC4Pbt25g2bRoqV65cIoEREb1KLLmiKVaiKVWqVK4nKISAWq3GunXrSiQwIiJ6NRQr0cTGxuo9trKygru7OypVqgQbG/ndtJOIyNRY0RRRUFBQScdBRESvqGKXH3/99RfmzZuHhIQEAEC1atUwYsQIVK1atcSCI2kIIUwdAqpUqWLqEHD69GlThwB7e3tTh0BGIsUdMV/pO2z+9NNPqFGjBo4fP45atWqhVq1aiIuLQ82aNfHTTz+VdIxERGTGilXRTJgwAWFhYbluchYeHo4JEyagS5cuJRIcEdGrwpLHaIpV0dy+fRt9+/bNtf4///lPvtdAIyKyVJZ8P5piJZoWLVrgt99+y7X+4MGDaN68+UsHRUREr45idZ29++67mDhxIo4fP45GjRoBAP744w9s3LgRERER2L59u96+RESWzpK7zoqVaIYNGwYAWLhwIRYuXGhwG5D9IvBKzkRElq1YiUar1ZZ0HERErzRObyYiIpJIsX+wGRMTg5iYGNy5cydXhbNixYqXDoyI6FXCMZoiioiIwPTp0xEYGAhPT0+zebJERGR8xUo0ixcvxvfff48+ffqUdDxERK8kVjRFlJGRgSZNmpR0LEREryxLTjTFmgwwcOBAREdHl3QsRET0CipWRfP06VMsXboU+/btQ0BAAGxtbfW2z507t0SCA4D09HQ4OjqW2PmIiEzBkiuaYiWa06dPo3bt2gCAs2fPlmQ8eg4ePIjo6GgMHz4c1atXL9KxGo0GGo1G9/jF208TEZFxlMgdNqVy8eJFbN++Hba2thgyZAiqVatW6GMjIyMREREhYXRERIVnyT/YLFKi6dy5c4H7KBSKErsnTWhoKKytrTFz5kxkZWVh+PDhhU42YWFhGDdunO5xamoq1Gp1icRFRESFV6REo1KppIpD58qVK7C3t0f58uUBAH379oVWq8WsWbOg1WoxduxYVK5cucDzKJVKKJVKqcMlIioUjtEU0sqVK6WKAwDw8OFDNGvWDKGhoRgxYgQ8PT0BAP369UNWVhaGDh0KGxsbDBo0CDVr1pQ0FiIiKhmy6uBzc3NDdHQ0oqOjsWzZMty6dUu3bcCAAQgICMDq1avxww8/ICMjw4SREhEVnSXe9Ax4iWudSaVFixZYs2YNevbsCYVCgYEDB8LT0xPp6emoX78+unTpgh49esDOzs7UoRIRUSHILtEAQLNmzbB27Vr06dMHd+7cQZMmTRAfH48DBw7g8OHDcHV1NXWIRERFwjEaGWrWrBk2btyIcePGYdu2bXBxccGaNWuYZIjILHF6s0wFBgZix44dePToEezt7eHu7m7qkIiIqIhknWgAwMXFBS4uLqYOg4jopVhy15l51F1ERGS2ZF/REBG9CljREBERSYQVDRGREbCiISIikggrGiIiI7DkioaJhojICPiDTbIocvgWdOHCBVOHQERGwkRDRGQEltx1Zh51FxERmS1WNERERsCKhoiISCJMNERERlDSd9d8mQppwYIF8PX1hb29PRo2bIgjR44U6rh169ZBoVCgU6dORWqPiYaIyIKsX78e48aNQ3h4OOLi4lCrVi0EBwfjzp07+R537do1jB8/Hs2bNy9ym0w0RERGkPM7mpJeimru3LkYNGgQQkND4e/vj8WLF8PR0RErVqzI85isrCz07t0bERERqFixYtGfe5GPICKiIpOy6yw1NVVv0Wg0BmPIyMjA8ePH0bp1a906KysrtG7dGocPH84z9unTp6Ns2bIYMGBAsZ47Ew0RkZlTq9VQqVS6JTIy0uB+9+7dQ1ZWFjw8PPTWe3h4ICkpyeAxBw8exHfffYdly5YVOz5ObyYiMgIppzcnJibC1dVVt16pVJbI+R8/fow+ffpg2bJlKFOmTLHPw0RDRGTmXF1d9RJNXsqUKQNra2skJyfrrU9OTka5cuVy7X/58mVcu3YNHTt21K3TarUAABsbG/z111/w8/MrsF12nRERGYEcpjfb2dmhXr16iImJ0a3TarWIiYlB48aNc+1ftWpVnDlzBidPntQt7777Llq2bImTJ09CrVYXql1WNEREFmTcuHEICQlBYGAgGjRogKioKKSlpSE0NBQA0LdvX3h5eSEyMhL29vaoUaOG3vGlSpUCgFzr8yP/RJOeDjg6mjoKIqKXIpdL0HTv3h13797F1KlTkZSUhNq1a2PXrl26CQI3btwo8dsPKIQQokTPWJIOHgSio4Hhw4Hq1Yt0qEaj0Zvil5qaCrVajZSUlEL1ZRKR5UhNTYVKpZLk8yHn3Js2bYJjCX9pTk9PR9euXWX/uSbvMZqLF4Ht24GlS4GEhCIdGhkZqTfdr7B9iUREUlAoFCX+Y01eVLMkhIYCM2cCu3cDCxYUKdmEhYUhJSVFtyQmJkoYKBFR/uQwGcBU5DdGc+UKYG8PlC+f/bhvX0CrBWbNyv7v2LFA5coFnkapVJbYXHIiIio+eSWahw+BZs2yK5kRIwBPz+z1/foBWVnA0KGAjQ0waBBQs6ZJQyUiKgq5TAYwBXl1nbm5ZQ/+R0cDy5YBt279u23AACAgAFi9GvjhByAjw3RxEhFRocmrogGAFi2ANWuAnj0BhQIYODC7sklPB+rXB7p0AXr0AOzsTB0pEVGhWXJFI79EA2R3n61dC/TpA9y5AzRpAsTHAwcOAIcPAzKexkdERPrkmWiA7GSzcSMwbhywbRvg4pJd6TDJEJEZKu79Ywo6pzmQb6IBgMBAYMcO4NGj7Jlo7u6mjoiIiIpI3okGyK5kXFxMHQUR0Uux5DEa86i7iIjIbDHREBGRpOTfdUZE9Apg1xkREZFEWNEQERkBKxoiIiKJsKIhIjICS65omGjIJORwY9ei3PNcKmfPnjV1CGbzYUXmi4mGiMgIWNEQEZGkLDnRcDIAERFJihUNEZERsKIhIiKSCCsaIiIjYEVDREQkESYaIiKSFBMNERFJimM0RERGYMljNEw0RERGYMmJhl1nREQkKVY0RERGwIrGDGi1WlOHQERExSDriiYhIQH379+HWq2Gj49PkY7VaDTQaDS6x6mpqSUdHhFRkZhLBVLSZFvRbN26FQ0aNEBoaCiqVKmCpUuX4vHjx4U+PjIyEiqVSreo1WoJoyUiorzILtEIIfDgwQN8+eWX+Oqrr/Dzzz8jPDwcQ4YMwfz585GSklKo84SFhSElJUW3JCYmShw5EVHecsZoSnoxB7LrOtNoNHBwcEBQUBA++OADuLm5ISwsDE5OThgzZgwAYPjw4XB1dc33PEqlEkql0ggRExFRfmSVaLZt24ZFixYhMTERWq0W3bt3h5ubGwBg1KhRAIDx48fjyZMnGD9+fIHJhohILjjrTAaOHTuGvn37okKFCmjQoAEuX76MFStW4Pr167p9Ro0ahRkzZmD+/PnIzMw0YbREREXDrjMTu3z5Mv773/8iLCwMkyZNAgA0aNAAM2fOhIODA4YMGaKbdTZx4kQMHjxYV+kQEZG8mTzRpKamokePHrh27RoGDx6sWz906FBotVpERkbC2toaAwYMQIUKFQAApUqVMlG0RERUVCbvOnN1dcXSpUvh5uaGX3/9FWfPntVtGz58OKZMmYKvvvoKa9aswbNnzwCYT78kERHJINEAQJ06dbBp0yakpaVh3rx5iI+P123Lmdbcs2dP2NiYvAAjIioWSx6jkUWiAYCAgACsWLECx44dQ1RUFM6dO6fbNmDAAFSuXNmE0RERUXHJJtEA2ZXN8uXLcfr0acyYMQPnz583dUhERCWCFY2M1KlTB/Pnz8ft27ehUqlMHQ4REb0kWQ561K9fH7t27YK9vb2pQyEiKhH8waYMMckQEb0aZFnREBG9aiy5omGiISIyAktONLLtOiMiolcDKxoyCTl8E3v+h8FkWqa+SK4x2mdFQ0REJBFWNERERsCKhoiISCKsaIiIjIAVDRERkUSYaIiISFLsOiMiMgJ2nREREUmEFQ0RkRGwoiEiIpIIKxoiIiNgRUNERCQRVjREREbAisYcaLWmjoCIiIpB3hVNQgJw/z6gVgM+PkU6VKPRQKPR6B6npqaWdHRERIXGikaOtm4FGjQAQkOBKlWApUuBx48LfXhkZCRUKpVuUavV0sVKRFSAnERT0os5kF+iEQJ48AD48kvgq6+An38GwsOBIUOA+fOBlJRCnSYsLAwpKSm6JTExUeLAiYjIEPl1nWk0gIMDEBQEfPAB4OYGhIUBTk7AmDHZ+wwfDri65nsapVIJpVIpfbxERJQveSWabduARYuAxMTswf/u3bMTDQCMGpX93/HjgSdPsv9bQLIhIiLTk0/X2bFjQN++QIUK2WMzly8DK1YA16//u8+oUcCMGdldaCa+xzgRUVFZ4vgMIJeK5vJl4L//ze4imzQpe12DBsDMmdndaEOG/DvrbOJEYPDgfysdIiKSNdMnmtRUoEcP4Nq17ASSY+jQ7O6zyEjA2hoYMCC72gGAUqVMESkRUbFxerMpubpmT112cwN+/RU4e/bfbcOHA1OmZM8+W7MGePYse72ZvLhERCSHRAMAdeoAmzYBaWnAvHlAfPy/23KmNffsCdiYvgAjIqKikUeiAYCAgOzB/2PHgKgo4Ny5f7cNGABUrmyy0IiIXhZ/sCkXdeoAy5cDp09nzy47f97UERER0UuSV6IBspPN/PnA7duASmXqaIiISoQlVzTyHPSoXx/YtQuwtzd1JERE9JLkmWgAJhkieqVwejMREZFEmGiIiEhSTDRERCQp+Y7REEksPT3d1CHAycnJ1CFAK4PbpD98+NCk7T8uwk0Vi0tOYzQLFizAnDlzkJSUhFq1amHevHlo0KCBwX2XLVuG1atX4+z/X7WlXr16mDlzZp77G8KKhojICOQyvXn9+vUYN24cwsPDERcXh1q1aiE4OBh37twxuP8vv/yCnj17IjY2FocPH4ZarUbbtm1x8+bNQrfJRENEZEHmzp2LQYMGITQ0FP7+/li8eDEcHR2xYsUKg/v/+OOPGDZsGGrXro2qVati+fLl0Gq1iImJKXSbTDRERGYuNTVVb9FoNAb3y8jIwPHjx9G6dWvdOisrK7Ru3RqHDx8uVFvp6enIzMzEa6+9Vuj4mGiIiMycWq2GSqXSLZGRkQb3u3fvHrKysuDh4aG33sPDA0lJSYVqa+LEiShfvrxesioIJwMQERmBlJMBEhMT4frcre2VSmWJtpPjiy++wLp16/DLL7/Avgg/qmeiISIyc66urnqJJi9lypSBtbU1kpOT9dYnJyejXLly+R775Zdf4osvvsC+ffsQEBBQpPjYdUZEZARymHVmZ2eHevXq6Q3k5wzsN27cOM/jZs+ejRkzZmDXrl0IDAws8nNnRUNEZEHGjRuHkJAQBAYGokGDBoiKikJaWhpCQ0MBAH379oWXl5dunGfWrFmYOnUqoqOj4evrqxvLcXZ2hrOzc6HaZKIhIjICufxgs3v37rh79y6mTp2KpKQk1K5dG7t27dJNELhx4wasrP7t7Fq0aBEyMjLQtWtXvfOEh4dj2rRphWqTiYaIyMKMGDECI0aMMLjtl19+0Xt87dq1l26PYzRERCQpVjREREYgl64zU2BFQ0REkmJFQ0RkBJZc0cg60Zw+fRq3b99Geno6WrVqBZVKVehjNRqN3vV+UlNTpQiRiIgKINuus02bNqFVq1aYNGkSunbtinbt2mHTpk2FPj4yMlLv2j9qtVrCaImI8ieHH2yaiiwTzYkTJzB06FDMmjUL+/fvx61bt/DGG29g3rx52LJlS6HOERYWhpSUFN2SmJgocdRERGSILBNNQkICypYtiw8++AClSpWCh4cHZs2aBbVaja+//hoZGRkFnkOpVOqu/1PY6wAREVHJk2WisbKygkajQXp6OhQKBZ49ewYPDw989tlnOHjwIA4cOGDqEImIioRdZzJTv359/P3331iwYAEAwMYme86CQqFAjRo14OLiYsrwiIioCGQ568zPzw/fffcd+vfvj2fPnmHAgAFwdXXFsmXL8OjRIw7sE5HZ4fRmGerVqxesra0xePBgREdHw97eHunp6di2bRvKly9v6vCIiKiQZJtoFAoFevTogcaNG+P8+fPIyspCQEAAvL29TR0aEREVgWwTTQ4fHx/4+PiYOgwiIiom2ScaIqJXgSWP0chy1hkREb06WNEQERmBJVc0TDREREZgyYmGXWdERCQpJhoiIpIUEw0REUmKYzREREZiLmMqJY2JhiyWo6OjqUOAEMLUISAzM9PUIeD8+fMmbT8tLc2k7b/qmGiIiIyAs86IiIgkwkRDRESSYtcZEZERsOuMiIhIIqxoiIiMgBUNERGRRJhoiIhIUkw0REQkKY7REBEZAcdoiIiIJMJEQ0REkmKiISIiSZnHGI0QgJn0RRIRGWLJYzTyTjQPHgD29kAxLueu0Wig0Wh0j1NTU0syMiIiKiT5dp1t3Qq8+y5Qpw4QEQGcOFGkwyMjI6FSqXSLWq2WJk4iIsqXPBNNXBzQrx8QHAy0awds3w58/jlw8GChTxEWFoaUlBTdkpiYKF28REQFyOk6K+nFHMiv6+zyZWDnTuDjj4FPPslet2MH8NVX2QsANGtW4GmUSiWUSqWEgRIRUWHIK9GkpgI9egA3bgD9+/+7vn377AkBX30FfPMNkJkJtGxpujiJiKjQ5NV15uoKLF0KlCoF/PYbEB//77YOHbKrnMuXge++A548MVmYRERUePKqaIDswf+NG4GQEODbb4FRo4Dq1bO3vfMOYGMDVKkCODiYNk4ioiKw5OnN8qpocgQEACtWAMeOAVFRwLlz/25r2xbw8TFZaEREVDTyTDRAdmWzfDlw+jQwYwZw/rypIyIiomKQb6IBspPN/PnA7duASmXqaIiIio3Tm+Wsfn1g167sKwQQEZHZkXdFk4NJhojIbJlHoiEiIrMl/64zIqJXAKc3ExERSYSJhoiIJMVEQ0REkuIYDRGREVjyGA0TDZEJCSFMHQKcnZ1NHQKWLFli0vaf8CK9kmLXGRERSYqJhoiIJMVEQ0REkuIYDRGREVjyZABWNEREJCkmGiIikhQTDRERSYpjNERERsAxGiIiIokw0RARkaTYdUZEZATsOiMiIpIIEw0REUlK9olm+/btWLFihanDICKiYpJ1ojl+/DhCQ0MBAFqttkjHajQapKam6i1ERKaSM0ZT0os5kG2iuXTpErZv344PP/wQ/fv3L/ILGhkZCZVKpVvUarVEkRIRUX5kmWhSU1PRs2dPLFq0CE+fPgWQ/W2gKDeJCgsLQ0pKim5JTEyUKlwiIsqHLBONq6srli5dilKlSiE2NhanT58GULSpfEqlEq6urnoLEZGpsOtMhurUqYNNmzZBq9Vi3rx5iI+PN3VIRERUDLJNNAAQEBCA77//HnFxcYiKisK5c+dMHRIRERWRrBMNkF3ZLF++HKdPn8aMGTNw/vx5U4dERERFIPtEA2Qnm/nz5+P27dtQqVSmDoeIqMgseYzGbK51Vr9+fezatQv29vamDoWIiIrALCqaHEwyRETmx6wSDRERmR+z6TojIjJnvE0AERGRRJhoiIhIUkw0RERGIKfpzQsWLICvry/s7e3RsGFDHDlyJN/9N27ciKpVq8Le3h41a9bEzp07i9QeEw0RkQVZv349xo0bh/DwcMTFxaFWrVoIDg7GnTt3DO5/6NAh9OzZEwMGDMCJEyfQqVMndOrUCWfPni10m0w0REQWZO7cuRg0aBBCQ0Ph7++PxYsXw9HRMc8bTH7zzTd4++238fHHH6NatWqYMWMG6tati/nz5xe6TYuZdZZziwHeAI3kpCi3vniVY3jy5Iks2pfytZDisyfnnC+eW6lUQqlU5to/IyMDx48fR1hYmG6dlZUVWrdujcOHDxts4/Dhwxg3bpzeuuDgYGzdurXQcVpMonn8+DEA8AZoRDI0bNgwU4cAIPtzoqQvc2VnZ4dy5cpJ9tnj7Oyc69zh4eGYNm1arn3v3buHrKwseHh46K338PDI8zqSSUlJBvdPSkoqdIwWk2jKly+PxMREuLi4FGsALTU1FWq1GomJiSa7tw1jYAyMQZoYhBB4/PgxypcvX8LRZV/R5OrVq8jIyCjxcwPZsb/4mWaomjEli0k0VlZW8Pb2funzyOEmaoyBMTCGko9Bygv22tvby+ISWmXKlIG1tTWSk5P11icnJ6NcuXIGjylXrlyR9jeEkwGIiCyEnZ0d6tWrh5iYGN06rVaLmJgYNG7c2OAxjRs31tsfAPbu3Zvn/oZYTEVDRETAuHHjEBISgsDAQDRo0ABRUVFIS0tDaGgoAKBv377w8vJCZGQkAGD06NEICgrCV199hfbt22PdunU4duwYli5dWug2mWgKSalUIjw83KR9n4yBMTAGecZgTrp37467d+9i6tSpSEpKQu3atbFr1y7dgP+NGzdgZfVvZ1eTJk0QHR2NKVOmYPLkyahcuTK2bt2KGjVqFLpNhZDD3EYiInplcYyGiIgkxURDRESSYqIhIiJJMdEQEZGkmGgoXzt37sTp06dNHQYRmTEmGspTcnIyRowYgaioKJw7d87U4RDl6ffff8/1o0KSDyaaQjh16hRu3rxp6jCQnp5u1PY8PDywadMmnD17Fl9//TXi4+ON2n5ejP06FIZWqzV1CEa3ffv2PC8tb0wLFixA8+bNMWfOHMTGxpo6HDKAiaYAW7duxTvvvINFixbhn3/+MVkcBw8exPjx443+YV+3bl0sWbIEcXFxiIqKMnmyMdXrYEhCQgIOHjyI69ev6/3AzRhOnz6N3bt3Y8uWLUhJSTFq2wBw/Phx3S/JTZlkr1y5gpUrVyIsLAyZmZn49ttvsX//fpPFQ3kQlKf//e9/wsHBQSxbtkzcunXLpLGsWLFCeHl5iVGjRolz584Zvf24uDhRt25dMXDgQHH27Fmjt5/D1K9Dji1btghnZ2dRqVIloVQqxZIlS0RqaqpR2t64caMoXbq0qF27trCyshKNGzcWGzduNErbQghx8eJFMXXqVBEWFiaEEEKr1Rqt7RelpaWJv//+WwghRGxsrGjatKno1KmTiImJMVlMlBsTTR6ePHkiPvjgAzF58mQhRPYb+vLly+Lzzz8XW7ZsMdqHyvNWrVolqlSpIoYPH27RycaUr4NWqxX3798XTZs2FUuWLBEXL14UM2fOFAqFQsycOVM8evRI0vbj4uJEmTJlxPLly8WDBw9EUlKSCAkJEW+++abYvHmzpG0LIURKSooIDAwU7u7uYuzYsbr1xkw2WVlZem0+e/ZM9/+//PILk40MMdHkIT09XQQGBoqRI0eK+/fvixEjRoigoCDh7e0tPDw8xIwZMySP4fLly+LmzZt661auXCmqVq0qhg4dKi5cuCB5DC96PtnEx8cbpU05vQ5PnjwR6enpYvLkyeLBgwe69d98840u2aSkpEjW/o8//ij8/f1FSkqK7sM1KSlJ9O7dWzRv3lxoNBrJ2s4RFxcnKleuLGrXri1OnToleXvPe/bsmRAi+9/nsmXLxO3bt4UQ+omOyUZ+mGjysWrVKuHg4CBcXV3F+++/L1atWiWEEGLMmDGiZcuWum9WUnjw4IHw9PQUkydPztVtt3z5cmFraytGjhwpTp8+LVkMeYmLixMNGjQQPXr0EAkJCZK2JafXYevWrSI4OFj4+/uLqlWr5vqQ/eabb4Stra349NNPJUs2a9euFX5+froP2MzMTCGEEFevXhUKhULs3btXknZfdOrUKREQEGDU6jbn31tqaqpo1qyZaNGihZgzZ47BfZls5IWJpgDx8fFiz549Qoh/3+jDhw8Xffv2FU+fPpW07djYWOHr6ysiIiJyfaOvV6+eUKlUYsKECUb5FvuiI0eOiKCgIKOMXcnhdTh69KhwdXUVQ4YMEf369RO2trZi9OjR4tq1a3r7ffHFF8LNzU3cu3dPkjguXboklEqlmDJlit76a9euiZo1a4o//vhDknYNMWZ1m1OxPH78WFStWlV07dpVXL16NdffPKfiEYLJRk6YaIogISFBTJ48WahUKnHmzBmjtPnbb78Jb29vMX36dN2HelpamhgyZIiYOXOmuHLlilHiMOTJkydGa8uUr8OlS5fE1KlTRWRkpG7dwoULhbe3t5g0aVKuZPN8l5oUfvjhB2FnZycmTZokLl68KJKTk8Unn3wi1Gp1rkQsNWNWt1lZWWLw4MHi3Xff1Uswz549E8+ePRP379/Pdcz+/ftFkyZNRNeuXcXu3bsljY/yxkRTSMeOHRM9e/YU1apVEydPnjRq27/99pvw9fUVI0aMENHR0eKTTz7R9dNbElO8DjmD32XKlNFNDMkxf/584eXlJT755BO9RCf1wLhWqxVr164VLi4u4vXXXxdvvPGG8Pb2FsePH5e03bxIXd0+fPhQCCHE06dPRatWrcTXX3+t23bq1Ckxffp0UblyZVGmTBkxZswYcenSJSHEv3+HCxcuCA8PD9GjRw+DyYikx0RTSOnp6eLAgQPixo0bJmn/6NGjonnz5kKtVgt/f3+TfaiYmileh5zB76ZNm+aqZBctWiTs7e1FRESEbrzEWK5duyZ27dolduzYIRITE43a9oukqm6//fZb0aJFC/H48WORkZEh2rdvL7p06SIePXokNm/eLAICAkS7du3ExIkTxdy5c4WTk5OIiooSQvzb1b1nzx6hUCjE//73P0lipIIx0ZiR1NRUcePGDXHnzh1Th2JSpngdTp06JWrXri0GDx6ca/B7+fLlJpkB+KrbuXOnUCgU4osvvtCtW7ZsmQgICBAeHh7CxsZGTJ48WRw7dky3vXPnziIoKEiX9DMzM8WWLVvEvn37jB4//Yt32CQqpBMnTmDgwIGoW7cuxo4dC39/f1OH9Eo7deoUPvjgA7Rr1w4DBgxAQEAAtFotTp48iVu3bsHb2xu1a9fW7Z+ZmYkBAwbAw8MDc+bM0a3XarVGv3ID6WOiISqCEydOYMiQIahYsSLCw8NRtWpVU4f0Sjt58iQGDBiAunXrYuTIkQgICMhz39WrV2PChAlYt24dWrRoYbwgqUBM80RFUKdOHcyfPx+3b9+GSqUydTivvNq1a2P58uWIi4vDvHnzDF7j7rfffsOMGTMwYsQIfPvtt0wyMsSKhqgYnj59Cnt7e1OHYTGe77YcM2YMqlevDiD7IqtLlixBcnIyxo4di3bt2kEIAYVCYeKI6XlMNERkFgyNkWVlZSEhIQGlSpWCt7c3cj7OmGjkhYmGiMzG88lm+PDhepMBSL44RkNEZqNOnTq6MZsFCxYgJSUFe/fuxYULF0wdGuWDFQ0RmZ0TJ05g6NChyMzMxIkTJxAbG4ugoCBTh0V5YEVDRGanTp06WLBgAQBg27ZtTDIyx4qGiMzW48eP4eLiYuowqABMNEREJCl2nRERkaSYaIiISFJMNEREJCkmGiIikhQTDRERSYqJhmQvMTER/fv3R/ny5WFnZwcfHx+MHj0a9+/fL/Q5rl27BoVCgZMnT0oSo0KhwNatWyU5N5G5Y6IhWbty5QoCAwNx8eJFrF27FpcuXcLixYsRExODxo0b48GDB6YOkYgKwERDsjZ8+HDY2dlhz549CAoKwuuvv4527dph3759uHnzJj755BMAhiuKUqVK4fvvvwcAVKhQAUD2L8oVCoXuniX9+vVDp06dEBERAXd3d7i6umLIkCHIyMjQncfX1xdRUVF6565duzamTZum2w4A77//PhQKhe4xEWVjoiHZevDgAXbv3o1hw4bBwcFBb1u5cuXQu3dvrF+/HoX5zfGRI0cAAPv27cPt27exefNm3baYmBgkJCTgl19+wdq1a7F582ZEREQUOs6jR48CAFauXInbt2/rHhNRNiYakq2LFy9CCIFq1aoZ3F6tWjU8fPgQd+/eLfBc7u7uAIDSpUujXLlyeO2113Tb7OzssGLFClSvXh3t27fH9OnT8e2330Kr1RYqzpxzlypVCuXKldM9JqJsTDQke1JfJalWrVpwdHTUPW7cuDH++ecfJCYmStoukaVgoiHZqlSpEhQKBRISEgxuT0hIgJubG9zd3aFQKHIlpMzMzBKJw8rKSrJzE1kCJhqSrdKlS6NNmzZYuHAhnjx5orctKSkJP/74I7p37w6FQgF3d3fcvn1bt/3ixYtIT0/XPbazswMAZGVl5Wrn1KlTeuf/448/4OzsDLVaDQC5zp2amoqrV6/qncPW1tbguYmIiYZkbv78+dBoNAgODsaBAweQmJiIXbt2oU2bNvDy8sLnn38OAGjVqhXmz5+PEydO4NixYxgyZAhsbW115ylbtiwcHBywa9cuJCcnIyUlRbctIyMDAwYMwLlz57Bz506Eh4djxIgRsLKy0p17zZo1+O2333DmzBmEhITA2tpaL05fX1/ExMQgKSkJDx8+NMIrQ2RGBJHMXbt2TYSEhAgPDw9ha2sr1Gq1GDlypLh3755un5s3b4q2bdsKJycnUblyZbFz506hUqnEypUrdfssW7ZMqNVqYWVlJYKCgoQQQoSEhIj33ntPTJ06VZQuXVo4OzuLQYMGiadPn+qOS0lJEd27dxeurq5CrVaL77//XtSqVUuEh4fr9tm+fbuoVKmSsLGxET4+PhK/IkTmhfejIYvWr18/PHr0iL/qJ5IQu86IiEhSTDRERCQpdp0REZGkWNEQEZGkmGiIiEhSTDRERCQpJhoiIpIUEw0REUmKiYaIiCTFRENERJJioiEiIkkx0RARkaT+D1yYGtqRgneWAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 600x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "xtext = X_test[index]\n",
    "xtext = [index_to_token[i] for i in xtext]\n",
    "ytext = list(y) + ['[S]']\n",
    "\n",
    "# Recall that each column is the attention weights for a token in the output\n",
    "plt.figure(figsize=(6, 6))\n",
    "\n",
    "plt.imshow(attentions, cmap='Greys')\n",
    "# Matplot lib reverses the y-axis, so we need to reverse the labels\n",
    "plt.gca().invert_yaxis()\n",
    "plt.xticks(range(len(ytext)), ytext, rotation=45)\n",
    "\n",
    "# Make the non-noise tokens blue in the yticks, and the noise tokens red\n",
    "plt.yticks(range(len(xtext)), xtext, rotation=45)\n",
    "for i, label in enumerate(plt.gca().get_yticklabels()):\n",
    "    if i % 2 == 0:\n",
    "        label.set_color('black')\n",
    "    else:\n",
    "        label.set_color('red')\n",
    "\n",
    "plt.ylabel('Input')\n",
    "plt.xlabel('Output')\n",
    "\n",
    "plt.title('Attention Visualization\\nString Reversal (noise tokens in red)')\n",
    "\n",
    "plt.colorbar()\n",
    "plt.savefig('attention.png', dpi=300)\n",
    "# plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "prototype",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}