Skip to content

Instantly share code, notes, and snippets.

@pmineiro

pmineiro/mnistigl.ipynb

Last active March 5, 2022 21:55
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save pmineiro/902b40b3054a77a1e85af6d5ffd469fe to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save pmineiro/902b40b3054a77a1e85af6d5ffd469fe to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
mnistigl.ipynb
{
"cells": [
{
"cell_type": "markdown",
"id": "f90e9f40",
"metadata": {},
"source": [
"# Util"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "393278c2",
"metadata": {
"code_folding": [
2,
18,
26,
47,
48,
54,
69,
82,
91,
100,
104,
117,
156,
157,
163,
174,
187,
195,
196,
208,
228,
261,
267,
268,
274,
285,
298,
306,
307,
319,
340,
373,
379,
395,
401,
411,
412,
444,
447,
465,
468,
475,
485,
512,
515,
518,
539,
549,
559,
591,
594,
602,
603,
609,
627,
630,
631,
638,
662,
665
]
},
"outputs": [],
"source": [
"from abc import ABC, abstractmethod\n",
"\n",
"class Batch(object):\n",
" def __init__(self):\n",
" super().__init__()\n",
"\n",
" @abstractmethod\n",
" def getContext(self):\n",
" pass\n",
"\n",
" @abstractmethod\n",
" def getFeedback(self, action):\n",
" pass\n",
" \n",
" @abstractmethod\n",
" def getReward(self, action):\n",
" pass\n",
"\n",
"class Simulator(object):\n",
" def __init__(self):\n",
" super().__init__()\n",
" \n",
" @abstractmethod\n",
" def trainIterator(self):\n",
" pass\n",
" \n",
"class MnistSimulator(Simulator):\n",
" def __init__(self, batch_size):\n",
" import torchvision\n",
"\n",
" super().__init__()\n",
" self.batch_size = batch_size\n",
" \n",
" transform = torchvision.transforms.Compose([\n",
" torchvision.transforms.ToTensor(),\n",
" torchvision.transforms.Normalize((0.1307,), (0.3081,))\n",
" ])\n",
" self.mnist_train = torchvision.datasets.MNIST('/tmp/mnist', train=True, download=True, transform=transform)\n",
" \n",
" @abstractmethod\n",
" def computeHilo(self):\n",
" pass\n",
" \n",
" @abstractmethod\n",
" def trainIterator(self):\n",
" pass\n",
" \n",
"class MnistFullCI(MnistSimulator):\n",
" def __init__(self, *, batch_size, decodability):\n",
" super().__init__(batch_size)\n",
" self.decodability = decodability\n",
" \n",
" self._makeFeedbacks()\n",
" \n",
" def _makeFeedbacks(self):\n",
" import torch\n",
" zero_one_loader = torch.utils.data.DataLoader(self.mnist_train, batch_size=1, shuffle=True)\n",
" zeros, ones = [], []\n",
" for bno, (images, labels) in enumerate(zero_one_loader):\n",
" flat = images.reshape(images.shape[0], -1)\n",
" if labels[0] == 0:\n",
" zeros.append(flat)\n",
" elif labels[0] == 1:\n",
" ones.append(flat)\n",
"\n",
" if len(zeros) > 100 and len(ones) > 100:\n",
" break \n",
" self.zeros, self.ones = torch.cat(zeros[:100], dim=0), torch.cat(ones[:100], dim=0)\n",
"\n",
" def computeHilo(self):\n",
" import numpy\n",
" import torch\n",
" \n",
" with torch.no_grad():\n",
" quantile_loader = torch.utils.data.DataLoader(self.mnist_train, batch_size=10000, shuffle=True)\n",
" for bno, (images, labels) in enumerate(quantile_loader):\n",
" flat = images.view(images.shape[0], -1)\n",
" hilo = numpy.quantile(flat.numpy(), [ 0.01, 0.99 ], axis=0)\n",
" break\n",
" \n",
" return hilo, hilo\n",
" \n",
" def trainIterator(self):\n",
" import torch\n",
" \n",
" train_loader = torch.utils.data.DataLoader(self.mnist_train, batch_size=self.batch_size, shuffle=True)\n",
"\n",
" for images, labels in train_loader:\n",
" yield self.MyBatch(images, labels, self.zeros, self.ones, self.decodability)\n",
" \n",
" class MyBatch(Batch):\n",
" def __init__(self, images, labels, zeros, ones, decodability):\n",
" super().__init__()\n",
"\n",
" self.images = images\n",
" self.labels = labels\n",
" self.zeros = zeros\n",
" self.ones = ones\n",
" self.decodability = decodability\n",
" \n",
" def getContext(self):\n",
" return self.images.view(self.images.shape[0], -1)\n",
" \n",
" # feedback is an image of 1 if correct else 0\n",
" def getFeedback(self, action):\n",
" import torch\n",
" with torch.no_grad():\n",
" reward = (action == self.labels.unsqueeze(1)).float()\n",
" zerossample = torch.randint(low=0, high=self.zeros.shape[0], size=(action.shape[0], 1))\n",
" goodfeedbacks = torch.gather(input=self.zeros, index=zerossample.expand(-1, self.zeros.shape[1]), dim=0)\n",
" onessample = torch.randint(low=0, high=self.ones.shape[0], size=(action.shape[0], 1))\n",
" badfeedbacks = torch.gather(input=self.ones, index=onessample.expand(-1, self.ones.shape[1]), dim=0)\n",
" noise = torch.rand(size=(action.shape[0], 1), device=action.device)\n",
" shouldflip = (noise <= ((1 + self.decodability)/2)).long()\n",
" noisyreward = reward + shouldflip * (1 - 2 * reward)\n",
" feedback = badfeedbacks + noisyreward * (goodfeedbacks - badfeedbacks)\n",
" \n",
" if False:\n",
" import matplotlib.pyplot as plt\n",
"\n",
" fig, axs = plt.subplots(1, 10)\n",
" for n, (s, f) in enumerate(zip(self.labels, goodfeedbacks)):\n",
" if n > 9:\n",
" break\n",
" axs[n].imshow(f.reshape(28, 28))\n",
" axs[n].set_title(f'{s.item()}')\n",
"\n",
" plt.show()\n",
"\n",
" fig, axs = plt.subplots(1, 10)\n",
" for n, (s, f) in enumerate(zip(self.labels, badfeedbacks)):\n",
" if n > 9:\n",
" break\n",
" axs[n].imshow(f.reshape(28, 28))\n",
" axs[n].set_title(f'{s.item()}')\n",
"\n",
" plt.show()\n",
" \n",
" fig, axs = plt.subplots(1, 10)\n",
" for n, (a, r, nr, f) in enumerate(zip(action, reward, noisyreward, feedback)):\n",
" if n > 9:\n",
" break\n",
" axs[n].imshow(f.reshape(28, 28))\n",
" axs[n].set_title(f'{a.item()} {r.long().item()} {nr.long().item()}')\n",
"\n",
" plt.show()\n",
" assert False\n",
" \n",
" return feedback\n",
" \n",
" def getReward(self, action):\n",
" import torch\n",
" with torch.no_grad():\n",
" reward = (action == self.labels.unsqueeze(1)).float()\n",
" return torch.mean(reward)\n",
"\n",
"class MnistActionCI(MnistSimulator):\n",
" def __init__(self, *, batch_size, decodability):\n",
" super().__init__(batch_size)\n",
" self.decodability = decodability\n",
" \n",
" self._makeFeedbacks()\n",
" \n",
" def _makeFeedbacks(self):\n",
" import torch\n",
" feedback_loader = torch.utils.data.DataLoader(self.mnist_train, batch_size=1, shuffle=True)\n",
" feedbacks = [ [] for _ in range(10) ]\n",
" for bno, (images, labels) in enumerate(feedback_loader):\n",
" flat = images.reshape(images.shape[0], -1)\n",
" feedbacks[labels[0]].append(flat)\n",
" if all(len(x) > 100 for x in feedbacks):\n",
" break \n",
" self.feedbacks = torch.cat([ torch.cat(x[:100], dim=0).unsqueeze(0) for x in feedbacks ], dim=0)\n",
"\n",
" def computeHilo(self):\n",
" import numpy\n",
" import torch\n",
" \n",
" with torch.no_grad():\n",
" quantile_loader = torch.utils.data.DataLoader(self.mnist_train, batch_size=10000, shuffle=True)\n",
" for bno, (images, labels) in enumerate(quantile_loader):\n",
" flat = images.view(images.shape[0], -1)\n",
" hilo = numpy.quantile(flat.numpy(), [ 0.01, 0.99 ], axis=0)\n",
" break\n",
" \n",
" return hilo, hilo\n",
" \n",
" def trainIterator(self):\n",
" import torch\n",
" \n",
" train_loader = torch.utils.data.DataLoader(self.mnist_train, batch_size=self.batch_size, shuffle=True)\n",
"\n",
" for images, labels in train_loader:\n",
" yield self.MyBatch(images, labels, self.feedbacks, self.decodability)\n",
" \n",
" class MyBatch(Batch):\n",
" def __init__(self, images, labels, feedbacks, decodability):\n",
" super().__init__()\n",
"\n",
" self.images = images\n",
" self.labels = labels\n",
" self.feedbacks = feedbacks\n",
" self.decodability = decodability\n",
" \n",
" def getContext(self):\n",
" return self.images.view(self.images.shape[0], -1)\n",
" \n",
" # feedback is an image of (x + 1) % 10 if correct else (x - 1) % 10\n",
" def getFeedback(self, action):\n",
" import torch\n",
" with torch.no_grad():\n",
" reward = (action == self.labels.unsqueeze(1)).float()\n",
" pixels = self.getContext().shape[1]\n",
" \n",
" # this assumes a particular majorization (Torch tensors are row-major)\n",
" bigfeedbacks = self.feedbacks.unsqueeze(0).expand(action.shape[0], -1, -1, -1).reshape(action.shape[0], -1, pixels) # Batch x (A x Rep) x Pixels\n",
" goodwhich = self.feedbacks.shape[1] * torch.remainder(self.labels + 1, 10) + torch.randint(low=0, high=self.feedbacks.shape[1], size=(action.shape[0],))\n",
" goodwhich = goodwhich.unsqueeze(1).unsqueeze(2).expand(-1, -1, pixels)\n",
" goodfeedbacks = torch.gather(input=bigfeedbacks, index=goodwhich, dim=1).squeeze(1)\n",
" badwhich = self.feedbacks.shape[1] * torch.remainder(self.labels - 1, 10) + torch.randint(low=0, high=self.feedbacks.shape[1], size=(action.shape[0],))\n",
" badwhich = badwhich.unsqueeze(1).unsqueeze(2).expand(-1, -1, pixels)\n",
" badfeedbacks = torch.gather(input=bigfeedbacks, index=badwhich, dim=1).squeeze(1)\n",
" \n",
" noise = torch.rand(size=(action.shape[0], 1), device=action.device)\n",
" shouldflip = (noise <= ((1 + self.decodability)/2)).long()\n",
" noisyreward = reward + shouldflip * (1 - 2 * reward)\n",
" feedback = badfeedbacks + noisyreward * (goodfeedbacks - badfeedbacks)\n",
" \n",
" if False:\n",
" import matplotlib.pyplot as plt\n",
" \n",
" fig, axs = plt.subplots(1, 10)\n",
" for n, (s, f) in enumerate(zip(self.labels, goodfeedbacks)):\n",
" if n > 9:\n",
" break\n",
" axs[n].imshow(f.reshape(28, 28))\n",
" axs[n].set_title(f'{s.item()}')\n",
"\n",
" plt.show()\n",
"\n",
" fig, axs = plt.subplots(1, 10)\n",
" for n, (s, f) in enumerate(zip(self.labels, badfeedbacks)):\n",
" if n > 9:\n",
" break\n",
" axs[n].imshow(f.reshape(28, 28))\n",
" axs[n].set_title(f'{s.item()}')\n",
"\n",
" plt.show()\n",
" \n",
" fig, axs = plt.subplots(1, 10)\n",
" for n, (a, r, nr, f) in enumerate(zip(action, reward, noisyreward, feedback)):\n",
" if n > 9:\n",
" break\n",
" axs[n].imshow(f.reshape(28, 28))\n",
" axs[n].set_title(f'{a.item()} {r.long().item()} {nr.long().item()}')\n",
"\n",
" plt.show()\n",
" assert False\n",
" \n",
" return feedback\n",
" \n",
" def getReward(self, action):\n",
" import torch\n",
" with torch.no_grad():\n",
" reward = (action == self.labels.unsqueeze(1)).float()\n",
" return torch.mean(reward)\n",
"\n",
"class MnistContextCI(MnistSimulator):\n",
" def __init__(self, *, batch_size, decodability):\n",
" super().__init__(batch_size)\n",
" self.decodability = decodability\n",
" \n",
" self._makeFeedbacks()\n",
" \n",
" def _makeFeedbacks(self):\n",
" import torch\n",
" feedback_loader = torch.utils.data.DataLoader(self.mnist_train, batch_size=1, shuffle=True)\n",
" feedbacks = [ [] for _ in range(10) ]\n",
" for bno, (images, labels) in enumerate(feedback_loader):\n",
" flat = images.reshape(images.shape[0], -1)\n",
" feedbacks[labels[0]].append(flat)\n",
" if all(len(x) > 100 for x in feedbacks):\n",
" break \n",
" self.feedbacks = torch.cat([ torch.cat(x[:100], dim=0).unsqueeze(0) for x in feedbacks ], dim=0)\n",
"\n",
" def computeHilo(self):\n",
" import numpy\n",
" import torch\n",
" \n",
" with torch.no_grad():\n",
" quantile_loader = torch.utils.data.DataLoader(self.mnist_train, batch_size=10000, shuffle=True)\n",
" for bno, (images, labels) in enumerate(quantile_loader):\n",
" flat = images.view(images.shape[0], -1)\n",
" hilo = numpy.quantile(flat.numpy(), [ 0.01, 0.99 ], axis=0)\n",
" break\n",
" \n",
" return hilo, hilo\n",
" \n",
" def trainIterator(self):\n",
" import torch\n",
" \n",
" train_loader = torch.utils.data.DataLoader(self.mnist_train, batch_size=self.batch_size, shuffle=True)\n",
"\n",
" for images, labels in train_loader:\n",
" yield self.MyBatch(images, labels, self.feedbacks, self.decodability)\n",
" \n",
" class MyBatch(Batch):\n",
" def __init__(self, images, labels, feedbacks, decodability):\n",
" super().__init__()\n",
"\n",
" self.images = images\n",
" self.labels = labels\n",
" self.feedbacks = feedbacks\n",
" self.decodability = decodability\n",
" \n",
" def getContext(self):\n",
" return self.images.view(self.images.shape[0], -1)\n",
" \n",
" # feedback is an image of (a + 1) % 10 if correct else (a - 1) % 10\n",
" def getFeedback(self, action):\n",
" import torch\n",
" with torch.no_grad():\n",
" reward = (action == self.labels.unsqueeze(1)).float()\n",
" pixels = self.getContext().shape[1]\n",
" \n",
" # this assumes a particular majorization (Torch tensors are row-major)\n",
" shortaction = action.squeeze(1)\n",
" bigfeedbacks = self.feedbacks.unsqueeze(0).expand(action.shape[0], -1, -1, -1).reshape(action.shape[0], -1, pixels) # Batch x (A x Rep) x Pixels\n",
" goodwhich = self.feedbacks.shape[1] * torch.remainder(shortaction + 1, 10) + torch.randint(low=0, high=self.feedbacks.shape[1], size=(action.shape[0],))\n",
" goodwhich = goodwhich.unsqueeze(1).unsqueeze(2).expand(-1, -1, pixels)\n",
" goodfeedbacks = torch.gather(input=bigfeedbacks, index=goodwhich, dim=1).squeeze(1)\n",
" badwhich = self.feedbacks.shape[1] * torch.remainder(shortaction - 1, 10) + torch.randint(low=0, high=self.feedbacks.shape[1], size=(action.shape[0],))\n",
" badwhich = badwhich.unsqueeze(1).unsqueeze(2).expand(-1, -1, pixels)\n",
" badfeedbacks = torch.gather(input=bigfeedbacks, index=badwhich, dim=1).squeeze(1)\n",
" \n",
" noise = torch.rand(size=(action.shape[0], 1), device=action.device)\n",
" shouldflip = (noise <= ((1 + self.decodability)/2)).long()\n",
" noisyreward = reward + shouldflip * (1 - 2 * reward)\n",
" feedback = badfeedbacks + noisyreward * (goodfeedbacks - badfeedbacks)\n",
" \n",
" if False:\n",
" import matplotlib.pyplot as plt\n",
" \n",
" fig, axs = plt.subplots(1, 10)\n",
" for n, (s, f) in enumerate(zip(shortaction, goodfeedbacks)):\n",
" if n > 9:\n",
" break\n",
" axs[n].imshow(f.reshape(28, 28))\n",
" axs[n].set_title(f'{s.item()}')\n",
"\n",
" plt.show()\n",
"\n",
" fig, axs = plt.subplots(1, 10)\n",
" for n, (s, f) in enumerate(zip(shortaction, badfeedbacks)):\n",
" if n > 9:\n",
" break\n",
" axs[n].imshow(f.reshape(28, 28))\n",
" axs[n].set_title(f'{s.item()}')\n",
"\n",
" plt.show()\n",
" \n",
" fig, axs = plt.subplots(1, 10)\n",
" for n, (a, r, nr, f) in enumerate(zip(action, reward, noisyreward, feedback)):\n",
" if n > 9:\n",
" break\n",
" axs[n].imshow(f.reshape(28, 28))\n",
" axs[n].set_title(f'{a.item()} {r.long().item()} {nr.long().item()}')\n",
"\n",
" plt.show()\n",
" assert False\n",
" \n",
" return feedback\n",
" \n",
" def getReward(self, action):\n",
" import torch\n",
" with torch.no_grad():\n",
" reward = (action == self.labels.unsqueeze(1)).float()\n",
" return torch.mean(reward)\n",
"\n",
"class Algorithm(object):\n",
" def __init__(self):\n",
" super().__init__()\n",
" \n",
" @abstractmethod\n",
" def sample(self, x):\n",
" pass\n",
" \n",
" @abstractmethod\n",
" def greedy(self, x):\n",
" pass\n",
" \n",
" @abstractmethod\n",
" def update(self, sample, feedback):\n",
" pass\n",
"\n",
"class Util(object):\n",
" import torch\n",
" \n",
" def __init__(self):\n",
" super().__init__()\n",
" \n",
" class Sample(object):\n",
" def __init__(self, x, action, probs):\n",
" super().__init__()\n",
" self.x = x\n",
" self.action = action\n",
" self.probs = probs\n",
"\n",
" def getAction(self):\n",
" return self.action\n",
" \n",
" class RFFSoftmax(torch.nn.Module):\n",
" def __init__(self, hilo, naction, numrff, sigma):\n",
" from math import pi\n",
" import numpy as np\n",
" import torch\n",
"\n",
" super().__init__()\n",
"\n",
" nobs = hilo.shape[1]\n",
" high = hilo[1, :]\n",
" low = hilo[0, :]\n",
"\n",
" self.rff = torch.nn.Linear(nobs, numrff)\n",
" self.rff.weight.data = torch.matmul(torch.empty(numrff, nobs).cauchy_(sigma = sigma), \n",
" torch.diag(torch.tensor([ 1.0/v if v > 1e-6 else 0. for v in high - low ])).float())\n",
" self.rff.weight.requires_grad = False\n",
" self.rff.bias.data = 2 * pi * torch.rand(numrff)\n",
" self.rff.bias.requires_grad = False\n",
" self.sqrtrff = np.sqrt(numrff)\n",
" self.final = torch.nn.Linear(numrff, naction)\n",
" self.final.weight.data *= 0\n",
" self.final.bias.data *= 0\n",
" self.sigmoid = torch.nn.Sigmoid()\n",
"\n",
" def forward(self, x):\n",
" import torch\n",
" with torch.no_grad():\n",
" rff = self.rff(x).cos() / self.sqrtrff\n",
" return self.final(rff)\n",
" \n",
" def predict(self, logits):\n",
" return self.sigmoid(logits)\n",
"\n",
"class IKAlgorithm(Algorithm):\n",
" import torch\n",
" \n",
" def __init__(self, *, hilo, sampler, lr):\n",
" from math import log\n",
" import itertools\n",
" import numpy\n",
" import torch\n",
" \n",
" super().__init__()\n",
" self.sampler = sampler\n",
" \n",
" util = Util()\n",
" \n",
" self.pi = util.RFFSoftmax(hilo[0], 10, 2000, 0.01)\n",
" doublehilo = numpy.concatenate((hilo[0], hilo[1]), axis=1)\n",
" self.decoder = util.RFFSoftmax(doublehilo, 10, 2000, 0.01)\n",
" self.alpha = 1/3\n",
" self.decoder.final.bias.data.fill_(log(self.alpha / (1 - self.alpha)))\n",
" self.opt = torch.optim.Adam(( p for p in itertools.chain(self.pi.parameters(), \n",
" self.decoder.parameters()) \n",
" if p.requires_grad ), \n",
" lr=lr)\n",
" \n",
" def sample(self, x):\n",
" import torch\n",
" \n",
" with torch.no_grad():\n",
" fhatlogit = self.pi(x)\n",
" fhat = self.pi.predict(fhatlogit)\n",
" sample, probs = self.sampler.sample(fhat, keepdim=True)\n",
" \n",
" return Util().Sample(x, sample, probs)\n",
"\n",
" def greedy(self, x):\n",
" import torch\n",
" \n",
" with torch.no_grad():\n",
" fhatlogit = self.pi(x)\n",
" _, pred = torch.max(fhatlogit, dim=1, keepdim=True)\n",
" _, anti = torch.min(fhatlogit, dim=1, keepdim=True)\n",
" \n",
" return pred, anti\n",
" \n",
" def update(self, sample, feedback):\n",
" import torch\n",
" \n",
" self.opt.zero_grad()\n",
" \n",
" fhatlogit = self.pi(sample.x)\n",
" \n",
" with torch.no_grad():\n",
" probs = sample.probs\n",
" d = torch.nn.functional.one_hot(sample.action.squeeze(1), num_classes=fhatlogit.shape[1]).float()\n",
" dprobs = self.alpha * d + ((1 - self.alpha) / (fhatlogit.shape[1] - 1)) * (1 - d)\n",
" dweights = (dprobs / probs) / torch.mean(dprobs / probs) \n",
" \n",
" dhatlogit = self.decoder(torch.cat((sample.x, feedback), dim=1))\n",
" dhat_log_loss = torch.nn.BCEWithLogitsLoss(weight=dweights)\n",
" dhat_loss = dhat_log_loss(dhatlogit, d)\n",
" \n",
" with torch.no_grad():\n",
" fakereward = self.decoder.sigmoid(dhatlogit)\n",
" loggedfakereward = torch.gather(input=fakereward, index=sample.action, dim=1)\n",
" \n",
" fhat_log_loss = torch.nn.BCEWithLogitsLoss()\n",
" loggedfhatlogit = torch.gather(input=fhatlogit, index=sample.action, dim=1)\n",
" fhat_loss = fhat_log_loss(loggedfhatlogit, loggedfakereward)\n",
"\n",
" loss = dhat_loss + fhat_loss\n",
" loss.backward()\n",
" \n",
" self.opt.step()\n",
" \n",
" return loss.item(), torch.mean(loggedfakereward)\n",
"\n",
" def __str__(self):\n",
" return f'IKAlgorithm(lr={self.opt.defaults[\"lr\"]} sampler={self.sampler})'\n",
"\n",
"class Sampler(object):\n",
" def __init__(self):\n",
" super().__init__()\n",
" \n",
" @abstractmethod\n",
" def sample(fhat, *, keepdim):\n",
" pass\n",
"\n",
"class EpsilonGreedy(Sampler):\n",
" def __init__(self, *, t0):\n",
" super().__init__()\n",
"\n",
" self.t0 = t0\n",
" self.t = 0\n",
"\n",
" def sample(self, fhat, *, keepdim=False):\n",
" import torch\n",
" N, K = fhat.shape\n",
" epsilon = (self.t0 / (self.t0 + self.t))**(1/3)\n",
" self.t += 1\n",
" fhatstar, ahatstar = torch.max(fhat, dim=1, keepdim=True)\n",
" rando = torch.randint(high=K, size=(N, 1), device=fhat.device)\n",
" unif = torch.rand(size=(N, 1), device=fhat.device)\n",
" shouldexplore = (unif <= epsilon).long()\n",
" actions = ahatstar + shouldexplore * (rando - ahatstar)\n",
" phatstar = (1 - epsilon) + epsilon / K\n",
" prando = epsilon / K\n",
" pactions = phatstar + shouldexplore * (prando - phatstar)\n",
" if not keepdim:\n",
" actions = actions.squeeze(1)\n",
" pactions = pactions.squeeze(1)\n",
" return actions, pactions\n",
" \n",
" def __str__(self):\n",
" return f'EpsilonGreedy(t0={self.t0})'\n",
"\n",
"class SquareCB(Sampler):\n",
" def __init__(self, *, t0, gamma0):\n",
" super().__init__()\n",
"\n",
" self.t0 = t0\n",
" self.gamma0 = gamma0\n",
" self.t = 0\n",
"\n",
" def sample(self, fhat, *, keepdim=False):\n",
" import torch\n",
" \n",
" self.t += 1\n",
" gamma = self.gamma0 * ((self.t0 + self.t) / self.t0)**(1/2)\n",
" \n",
" N, K = fhat.shape\n",
" fhatstar, ahatstar = torch.max(fhat, dim=1, keepdim=True)\n",
" probs = 1 / (K + gamma * (fhatstar - fhat))\n",
" #probs = (1 - fhat) / (K * (1 - fhat) + gamma * (fhatstar - fhat))\n",
" psum = torch.sum(probs, dim=1, keepdim=True)\n",
" phatstar = (1 - psum) + torch.gather(input=probs, dim=1, index=ahatstar)\n",
"\n",
" rando = torch.randint(high=K, size=(N, 1), device=fhat.device)\n",
" prando = torch.gather(input=probs, dim=1, index=rando)\n",
" unif = torch.rand(size=(N, 1), device=fhat.device)\n",
" shouldexplore = (unif <= K * prando).long()\n",
" actions = ahatstar + shouldexplore * (rando - ahatstar)\n",
" pactions = phatstar + shouldexplore * (prando - phatstar)\n",
" if not keepdim:\n",
" actions = actions.squeeze(1)\n",
" pactions = pactions.squeeze(1)\n",
" return actions, pactions\n",
" \n",
" def __str__(self):\n",
" return f'SquareCB(gamma0={self.gamma0} t0={self.t0})'\n",
"\n",
"def run_sim_helper(*, passes, simulator, algorithm):\n",
" import itertools\n",
" \n",
" class EasyAcc:\n",
" def __init__(self):\n",
" self.n = 0\n",
" self.sum = 0\n",
" self.sumsq = 0\n",
"\n",
" def __iadd__(self, other):\n",
" self.n += 1\n",
" self.sum += other\n",
" self.sumsq += other*other\n",
" return self\n",
"\n",
" def __isub__(self, other):\n",
" self.n += 1\n",
" self.sum -= other\n",
" self.sumsq += other*other\n",
" return self\n",
"\n",
" def mean(self):\n",
" return self.sum / max(self.n, 1)\n",
"\n",
" def var(self):\n",
" from math import sqrt\n",
" return sqrt(self.sumsq / max(self.n, 1) - self.mean()**2)\n",
"\n",
" def semean(self):\n",
" from math import sqrt\n",
" return self.var() / sqrt(max(self.n, 1))\n",
"\n",
" print('{:<5s}\\t{:<8s} {:<8s}\\t{:<8s} {:<8s}\\t{:<8s} {:<8s}\\t{:<8s} {:<8s}\\t{:<8s} {:<8s}'.format(\n",
" 'bno', \n",
" 'loss', 'since', \n",
" 'pred', 'since',\n",
" 'anti', 'since',\n",
" 'reward', 'since',\n",
" 'fake', 'since',\n",
" ),\n",
" flush=True)\n",
" \n",
" avloss, avreward, avfakereward, avpred, avanti = [ EasyAcc() for _ in range(5) ]\n",
" avlosssl, avrewardsl, avfakerewardsl, avpredsl, avantisl = [ EasyAcc() for _ in range(5) ]\n",
" \n",
" for bno, batch in enumerate(itertools.chain(*[ simulator.trainIterator() for _ in range(passes) ])):\n",
" x = batch.getContext()\n",
" sample = algorithm.sample(x)\n",
" feedback = batch.getFeedback(sample.getAction())\n",
" loss, fakereward = algorithm.update(sample, feedback)\n",
" \n",
" avloss += loss\n",
" avlosssl += loss\n",
"\n",
" avfakereward += fakereward\n",
" avfakerewardsl += fakereward\n",
" \n",
" reward = batch.getReward(sample.getAction())\n",
" pred, anti = algorithm.greedy(x)\n",
" predreward = batch.getReward(pred)\n",
" antireward = batch.getReward(anti)\n",
" \n",
" avreward += reward\n",
" avrewardsl += reward\n",
" avpred += predreward\n",
" avpredsl += predreward \n",
" avanti += antireward\n",
" avantisl += antireward\n",
" \n",
" if (bno & (bno - 1) == 0):\n",
" print('{:<5d}\\t{:<8.5f} {:<8.5f}\\t{:<8.5f} {:<8.5f}\\t{:<8.5f} {:<8.5f}\\t{:<8.5f} {:<8.5f}\\t{:<8.5f} {:<8.5f}'.format(\n",
" bno, \n",
" avloss.mean(), avlosssl.mean(),\n",
" avpred.mean(), avpredsl.mean(),\n",
" avanti.mean(), avantisl.mean(),\n",
" avreward.mean(), avrewardsl.mean(),\n",
" avfakereward.mean(), avfakerewardsl.mean(),\n",
" ),\n",
" flush=True)\n",
" avlosssl, avrewardsl, avfakerewardsl, avpredsl, avantisl = [ EasyAcc() for _ in range(5) ]\n",
"\n",
" print('{:<5d}\\t{:<8.5f} {:<8.5f}\\t{:<8.5f} {:<8.5f}\\t{:<8.5f} {:<8.5f}\\t{:<8.5f} {:<8.5f}\\t{:<8.5f} {:<8.5f}'.format(\n",
" bno, \n",
" avloss.mean(), avlosssl.mean(),\n",
" avpred.mean(), avpredsl.mean(),\n",
" avanti.mean(), avantisl.mean(),\n",
" avreward.mean(), avrewardsl.mean(),\n",
" avfakereward.mean(), avfakerewardsl.mean(),\n",
" ),\n",
" flush=True)"
]
},
{
"cell_type": "markdown",
"id": "d1a9fb44",
"metadata": {},
"source": [
"# Mnist Action CI $(y_a \\perp a | r_a, x)$\n",
"Feedback is $(x + 1) \\mod 10$ if correct else $(x - 1) \\mod 10$."
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "fd88f658",
"metadata": {
"code_folding": [
0
]
},
"outputs": [],
"source": [
"def run_mnist_action_ci(*, seed, alglambda):\n",
" import torch\n",
" print(f'***** seed = {seed} *****')\n",
" torch.manual_seed(seed)\n",
" hilo = MnistActionCI(batch_size=64, decodability=1).computeHilo()\n",
" \n",
" for decodability in (1, -1, 0.5, -0.5,):\n",
" import copy\n",
" torch.manual_seed(seed)\n",
" sim = MnistActionCI(batch_size=64, decodability=decodability)\n",
" print(f'***** decodability = {decodability} *****')\n",
" with torch.random.fork_rng():\n",
" alg = alglambda(hilo)\n",
" print(f'***** alg = {alg} *****')\n",
" with torch.random.fork_rng():\n",
" run_sim_helper(passes=1, simulator=copy.deepcopy(sim), algorithm=alg)"
]
},
{
"cell_type": "code",
"execution_count": 35,
"id": "5e596247",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"***** seed = 13 *****\n",
"***** decodability = 1 *****\n",
"***** alg = IKAlgorithm(lr=0.05 sampler=EpsilonGreedy(t0=1)) *****\n",
"bno \tloss since \tpred since \tanti since \treward since \tfake since \n",
"0 \t1.32966 1.32966 \t0.10938 0.10938 \t0.09375 0.09375 \t0.10938 0.10938 \t0.33333 0.33333 \n",
"1 \t1.30444 1.27921 \t0.08594 0.06250 \t0.10938 0.12500 \t0.11719 0.12500 \t0.33463 0.33592 \n",
"2 \t1.30393 1.30293 \t0.06771 0.03125 \t0.11979 0.14062 \t0.08854 0.03125 \t0.33477 0.33506 \n",
"4 \t1.30399 1.30408 \t0.06562 0.06250 \t0.11250 0.10156 \t0.09687 0.10938 \t0.33652 0.33914 \n",
"8 \t1.29672 1.28764 \t0.08160 0.10156 \t0.09549 0.07422 \t0.10590 0.11719 \t0.34312 0.35137 \n",
"16 \t1.29588 1.29494 \t0.11121 0.14453 \t0.09099 0.08594 \t0.11029 0.11523 \t0.35378 0.36579 \n",
"32 \t1.29838 1.30103 \t0.15436 0.20020 \t0.08523 0.07910 \t0.13968 0.17090 \t0.38594 0.42010 \n",
"64 \t1.30807 1.31806 \t0.17596 0.19824 \t0.07067 0.05566 \t0.16226 0.18555 \t0.41429 0.44352 \n",
"128 \t1.30980 1.31156 \t0.20591 0.23633 \t0.06202 0.05322 \t0.18302 0.20410 \t0.43660 0.45927 \n",
"256 \t1.31787 1.32600 \t0.26204 0.31860 \t0.05052 0.03894 \t0.22781 0.27295 \t0.45920 0.48197 \n",
"512 \t1.32409 1.33034 \t0.30431 0.34674 \t0.04173 0.03290 \t0.26687 0.30609 \t0.49435 0.52965 \n",
"937 \t1.33276 1.34322 \t0.39582 0.50629 \t0.03077 0.01754 \t0.35419 0.45960 \t0.52439 0.56064 \n",
"***** decodability = -1 *****\n",
"***** alg = IKAlgorithm(lr=0.05 sampler=EpsilonGreedy(t0=1)) *****\n",
"bno \tloss since \tpred since \tanti since \treward since \tfake since \n",
"0 \t1.32966 1.32966 \t0.10938 0.10938 \t0.09375 0.09375 \t0.10938 0.10938 \t0.33333 0.33333 \n",
"1 \t1.30487 1.28007 \t0.09375 0.07812 \t0.11719 0.14062 \t0.11719 0.12500 \t0.33465 0.33597 \n",
"2 \t1.30430 1.30317 \t0.07812 0.04688 \t0.11979 0.12500 \t0.08854 0.03125 \t0.33357 0.33142 \n",
"4 \t1.30306 1.30120 \t0.11250 0.16406 \t0.11250 0.10156 \t0.10000 0.11719 \t0.33489 0.33687 \n",
"8 \t1.29671 1.28877 \t0.10243 0.08984 \t0.09375 0.07031 \t0.11458 0.13281 \t0.34906 0.36677 \n",
"16 \t1.29965 1.30295 \t0.11581 0.13086 \t0.09007 0.08594 \t0.11857 0.12305 \t0.38112 0.41718 \n",
"32 \t1.30370 1.30800 \t0.14205 0.16992 \t0.08002 0.06934 \t0.13258 0.14746 \t0.39104 0.40158 \n",
"64 \t1.30846 1.31338 \t0.15986 0.17822 \t0.06731 0.05420 \t0.14543 0.15869 \t0.40492 0.41923 \n",
"128 \t1.31436 1.32034 \t0.19077 0.22217 \t0.05790 0.04834 \t0.16982 0.19458 \t0.43110 0.45769 \n",
"256 \t1.32377 1.33325 \t0.21948 0.24841 \t0.05235 0.04675 \t0.19303 0.21643 \t0.45743 0.48396 \n",
"512 \t1.33082 1.33791 \t0.29898 0.37878 \t0.04075 0.02911 \t0.26495 0.33716 \t0.48782 0.51834 \n",
"937 \t1.33090 1.33100 \t0.39592 0.51294 \t0.03305 0.02375 \t0.35553 0.46485 \t0.52269 0.56478 \n",
"***** decodability = 0.5 *****\n",
"***** alg = IKAlgorithm(lr=0.05 sampler=EpsilonGreedy(t0=1)) *****\n",
"bno \tloss since \tpred since \tanti since \treward since \tfake since \n",
"0 \t1.32966 1.32966 \t0.10938 0.10938 \t0.09375 0.09375 \t0.10938 0.10938 \t0.33333 0.33333 \n",
"1 \t1.30485 1.28004 \t0.09375 0.07812 \t0.11719 0.14062 \t0.11719 0.12500 \t0.33435 0.33537 \n",
"2 \t1.30426 1.30308 \t0.07812 0.04688 \t0.13542 0.17188 \t0.08854 0.03125 \t0.33383 0.33279 \n",
"4 \t1.30349 1.30235 \t0.08438 0.09375 \t0.12188 0.10156 \t0.09687 0.10938 \t0.33555 0.33813 \n",
"8 \t1.29700 1.28888 \t0.09201 0.10156 \t0.09896 0.07031 \t0.10590 0.11719 \t0.34245 0.35107 \n",
"16 \t1.29599 1.29486 \t0.10662 0.12305 \t0.10386 0.10938 \t0.10938 0.11328 \t0.35196 0.36266 \n",
"32 \t1.30241 1.30924 \t0.13068 0.15625 \t0.09991 0.09570 \t0.12926 0.15039 \t0.37391 0.39723 \n",
"64 \t1.31644 1.33092 \t0.11827 0.10547 \t0.08702 0.07373 \t0.12043 0.11133 \t0.39610 0.41900 \n",
"128 \t1.32278 1.32921 \t0.12718 0.13623 \t0.08006 0.07300 \t0.12476 0.12915 \t0.41618 0.43656 \n",
"256 \t1.33804 1.35343 \t0.12239 0.11755 \t0.07551 0.07092 \t0.11977 0.11475 \t0.43168 0.44731 \n",
"512 \t1.35660 1.37523 \t0.12601 0.12964 \t0.07657 0.07764 \t0.12031 0.12085 \t0.45100 0.47039 \n",
"937 \t1.37982 1.40785 \t0.14880 0.17632 \t0.06935 0.06063 \t0.14169 0.16750 \t0.45475 0.45927 \n",
"***** decodability = -0.5 *****\n",
"***** alg = IKAlgorithm(lr=0.05 sampler=EpsilonGreedy(t0=1)) *****\n",
"bno \tloss since \tpred since \tanti since \treward since \tfake since \n",
"0 \t1.32966 1.32966 \t0.10938 0.10938 \t0.09375 0.09375 \t0.10938 0.10938 \t0.33333 0.33333 \n",
"1 \t1.30532 1.28098 \t0.10156 0.09375 \t0.11719 0.14062 \t0.11719 0.12500 \t0.33389 0.33444 \n",
"2 \t1.30427 1.30217 \t0.08333 0.04688 \t0.11458 0.10938 \t0.08854 0.03125 \t0.33239 0.32940 \n",
"4 \t1.30410 1.30385 \t0.08438 0.08594 \t0.10938 0.10156 \t0.09375 0.10156 \t0.33464 0.33801 \n",
"8 \t1.29698 1.28807 \t0.09201 0.10156 \t0.09201 0.07031 \t0.10243 0.11328 \t0.34187 0.35091 \n",
"16 \t1.30001 1.30342 \t0.10662 0.12305 \t0.08272 0.07227 \t0.10754 0.11328 \t0.36231 0.38530 \n",
"32 \t1.30547 1.31126 \t0.13210 0.15918 \t0.07812 0.07324 \t0.12311 0.13965 \t0.37633 0.39124 \n",
"64 \t1.31463 1.32407 \t0.12476 0.11719 \t0.07548 0.07275 \t0.12308 0.12305 \t0.39423 0.41269 \n",
"128 \t1.32324 1.33199 \t0.12270 0.12061 \t0.08031 0.08521 \t0.11664 0.11011 \t0.41592 0.43795 \n",
"256 \t1.33952 1.35593 \t0.12044 0.11816 \t0.08208 0.08386 \t0.11278 0.10889 \t0.42968 0.44354 \n",
"512 \t1.35598 1.37249 \t0.11882 0.11719 \t0.07782 0.07355 \t0.11355 0.11432 \t0.44759 0.46556 \n",
"937 \t1.37896 1.40669 \t0.13551 0.15566 \t0.07348 0.06824 \t0.12883 0.14728 \t0.45464 0.46314 \n"
]
}
],
"source": [
"run_mnist_action_ci(seed=13, alglambda = lambda hilo: IKAlgorithm(hilo=hilo, sampler=EpsilonGreedy(t0=1), lr=5e-2))"
]
},
{
"cell_type": "markdown",
"id": "25aaf5cf",
"metadata": {},
"source": [
"# Mnist Full CI ($y_a \\perp x, a | r_a$)\n",
"Feedback is a \"1\" image if correct or a \"0\" image if incorrect."
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "be02f211",
"metadata": {
"code_folding": [
0
]
},
"outputs": [],
"source": [
"def run_mnist_full_ci(*, seed, alglambda):\n",
" import torch\n",
" print(f'***** seed = {seed} *****')\n",
" torch.manual_seed(seed)\n",
" hilo = MnistFullCI(batch_size=64, decodability=1).computeHilo()\n",
" \n",
" for decodability in (1, -1, 0.5, -0.5,):\n",
" import copy\n",
" torch.manual_seed(seed)\n",
" sim = MnistFullCI(batch_size=64, decodability=decodability)\n",
" print(f'***** decodability = {decodability} *****')\n",
" with torch.random.fork_rng():\n",
" alg = alglambda(hilo)\n",
" print(f'***** alg = {alg} *****')\n",
" with torch.random.fork_rng():\n",
" run_sim_helper(passes=1, simulator=copy.deepcopy(sim), algorithm=alg)"
]
},
{
"cell_type": "markdown",
"id": "ce18fc54",
"metadata": {
"heading_collapsed": true
},
"source": [
"## IK, Epsilon-Greedy"
]
},
{
"cell_type": "code",
"execution_count": 120,
"id": "12c6fc3c",
"metadata": {
"hidden": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"***** seed = 13 *****\n",
"***** decodability = 1 *****\n",
"***** alg = IKAlgorithm(lr=0.05 sampler=EpsilonGreedy(t0=1)) *****\n",
"bno \tloss since \tpred since \tanti since \treward since \tfake since \n",
"0 \t1.32966 1.32966 \t0.10938 0.10938 \t0.09375 0.09375 \t0.10938 0.10938 \t0.33333 0.33333 \n",
"1 \t1.30507 1.28049 \t0.09375 0.07812 \t0.12500 0.15625 \t0.11719 0.12500 \t0.33461 0.33589 \n",
"2 \t1.30434 1.30288 \t0.08333 0.06250 \t0.13021 0.14062 \t0.09375 0.04688 \t0.33435 0.33383 \n",
"4 \t1.30183 1.29805 \t0.08750 0.09375 \t0.11875 0.10156 \t0.10000 0.10938 \t0.33558 0.33743 \n",
"8 \t1.29784 1.29286 \t0.08854 0.08984 \t0.09722 0.07031 \t0.10590 0.11328 \t0.34345 0.35328 \n",
"16 \t1.30046 1.30342 \t0.13695 0.19141 \t0.09743 0.09766 \t0.12040 0.13672 \t0.35886 0.37619 \n",
"32 \t1.30231 1.30427 \t0.16998 0.20508 \t0.09470 0.09180 \t0.14867 0.17871 \t0.37876 0.39991 \n",
"64 \t1.31192 1.32184 \t0.21274 0.25684 \t0.07933 0.06348 \t0.18293 0.21826 \t0.40531 0.43269 \n",
"128 \t1.31666 1.32146 \t0.25787 0.30371 \t0.06831 0.05713 \t0.22081 0.25928 \t0.43104 0.45718 \n",
"256 \t1.32381 1.33101 \t0.29754 0.33752 \t0.05952 0.05066 \t0.25517 0.28979 \t0.45657 0.48229 \n",
"512 \t1.32461 1.32541 \t0.36528 0.43329 \t0.05172 0.04388 \t0.32002 0.38513 \t0.49113 0.52582 \n",
"937 \t1.32471 1.32484 \t0.45266 0.55813 \t0.03711 0.01949 \t0.40302 0.50320 \t0.52373 0.56309 \n",
"***** decodability = -1 *****\n",
"***** alg = IKAlgorithm(lr=0.05 sampler=EpsilonGreedy(t0=1)) *****\n",
"bno \tloss since \tpred since \tanti since \treward since \tfake since \n",
"0 \t1.32966 1.32966 \t0.10938 0.10938 \t0.09375 0.09375 \t0.10938 0.10938 \t0.33333 0.33333 \n",
"1 \t1.30546 1.28126 \t0.07812 0.04688 \t0.11719 0.14062 \t0.11719 0.12500 \t0.33346 0.33360 \n",
"2 \t1.30742 1.31135 \t0.08854 0.10938 \t0.11979 0.12500 \t0.09375 0.04688 \t0.33497 0.33798 \n",
"4 \t1.30894 1.31121 \t0.07500 0.05469 \t0.10938 0.09375 \t0.09687 0.10156 \t0.34675 0.36442 \n",
"8 \t1.30966 1.31057 \t0.09549 0.12109 \t0.09375 0.07422 \t0.10417 0.11328 \t0.35984 0.37621 \n",
"16 \t1.30897 1.30820 \t0.10570 0.11719 \t0.09835 0.10352 \t0.11121 0.11914 \t0.35881 0.35765 \n",
"32 \t1.31093 1.31301 \t0.09991 0.09375 \t0.10701 0.11621 \t0.10511 0.09863 \t0.39271 0.42872 \n",
"64 \t1.32061 1.33060 \t0.10577 0.11182 \t0.12812 0.14990 \t0.11322 0.12158 \t0.41979 0.44773 \n",
"128 \t1.32243 1.32428 \t0.09460 0.08325 \t0.13227 0.13647 \t0.09629 0.07910 \t0.44220 0.46496 \n",
"256 \t1.33451 1.34668 \t0.10372 0.11292 \t0.12099 0.10962 \t0.10044 0.10461 \t0.46115 0.48024 \n",
"512 \t1.34569 1.35691 \t0.12850 0.15338 \t0.10304 0.08502 \t0.12141 0.14246 \t0.48409 0.50712 \n",
"937 \t1.36260 1.38302 \t0.18943 0.26298 \t0.09245 0.07967 \t0.17769 0.24563 \t0.50181 0.52320 \n",
"***** decodability = 0.5 *****\n",
"***** alg = IKAlgorithm(lr=0.05 sampler=EpsilonGreedy(t0=1)) *****\n",
"bno \tloss since \tpred since \tanti since \treward since \tfake since \n",
"0 \t1.32966 1.32966 \t0.10938 0.10938 \t0.09375 0.09375 \t0.10938 0.10938 \t0.33333 0.33333 \n",
"1 \t1.30474 1.27983 \t0.09375 0.07812 \t0.12500 0.15625 \t0.11719 0.12500 \t0.33459 0.33585 \n",
"2 \t1.30399 1.30249 \t0.07292 0.03125 \t0.13542 0.15625 \t0.08854 0.03125 \t0.33390 0.33250 \n",
"4 \t1.30216 1.29940 \t0.08750 0.10938 \t0.12188 0.10156 \t0.10000 0.11719 \t0.33365 0.33329 \n",
"8 \t1.29573 1.28770 \t0.09722 0.10938 \t0.10069 0.07422 \t0.10243 0.10547 \t0.34649 0.36253 \n",
"16 \t1.30262 1.31038 \t0.12776 0.16211 \t0.09743 0.09375 \t0.12040 0.14062 \t0.36855 0.39337 \n",
"32 \t1.30842 1.31457 \t0.14015 0.15332 \t0.10275 0.10840 \t0.12879 0.13770 \t0.37469 0.38122 \n",
"64 \t1.31988 1.33170 \t0.13534 0.13037 \t0.08702 0.07080 \t0.13197 0.13525 \t0.39178 0.40940 \n",
"128 \t1.32899 1.33825 \t0.15540 0.17578 \t0.07982 0.07251 \t0.14462 0.15747 \t0.40806 0.42460 \n",
"256 \t1.34702 1.36519 \t0.14105 0.12659 \t0.08554 0.09131 \t0.13132 0.11792 \t0.42628 0.44465 \n",
"512 \t1.36833 1.38972 \t0.15278 0.16455 \t0.07392 0.06226 \t0.14215 0.15302 \t0.44862 0.47105 \n",
"937 \t1.39125 1.41892 \t0.17161 0.19434 \t0.06875 0.06250 \t0.16081 0.18335 \t0.45148 0.45493 \n",
"***** decodability = -0.5 *****\n",
"***** alg = IKAlgorithm(lr=0.05 sampler=EpsilonGreedy(t0=1)) *****\n",
"bno \tloss since \tpred since \tanti since \treward since \tfake since \n",
"0 \t1.32966 1.32966 \t0.10938 0.10938 \t0.09375 0.09375 \t0.10938 0.10938 \t0.33333 0.33333 \n",
"1 \t1.30487 1.28009 \t0.09375 0.07812 \t0.10156 0.10938 \t0.11719 0.12500 \t0.33412 0.33491 \n",
"2 \t1.30595 1.30810 \t0.07812 0.04688 \t0.11458 0.14062 \t0.08854 0.03125 \t0.33473 0.33595 \n",
"4 \t1.30764 1.31018 \t0.06250 0.03906 \t0.10938 0.10156 \t0.09375 0.10156 \t0.33889 0.34512 \n",
"8 \t1.30273 1.29660 \t0.08160 0.10547 \t0.09375 0.07422 \t0.10243 0.11328 \t0.34147 0.34469 \n",
"16 \t1.30562 1.30887 \t0.08548 0.08984 \t0.08915 0.08398 \t0.10386 0.10547 \t0.34891 0.35729 \n",
"32 \t1.31029 1.31525 \t0.07955 0.07324 \t0.10417 0.12012 \t0.09659 0.08887 \t0.36639 0.38495 \n",
"64 \t1.32097 1.33199 \t0.07187 0.06396 \t0.11010 0.11621 \t0.08918 0.08154 \t0.40254 0.43981 \n",
"128 \t1.32663 1.33237 \t0.07001 0.06812 \t0.11810 0.12622 \t0.08043 0.07153 \t0.42110 0.43996 \n",
"256 \t1.34333 1.36015 \t0.06980 0.06958 \t0.11570 0.11328 \t0.07496 0.06946 \t0.44035 0.45975 \n",
"512 \t1.36080 1.37834 \t0.08114 0.09253 \t0.11507 0.11444 \t0.08373 0.09253 \t0.45726 0.47424 \n",
"937 \t1.38823 1.42134 \t0.09522 0.11221 \t0.10439 0.09151 \t0.09637 0.11162 \t0.46219 0.46815 \n"
]
}
],
"source": [
"run_mnist_full_ci(seed=13, alglambda = lambda hilo: IKAlgorithm(hilo=hilo, sampler=EpsilonGreedy(t0=1), lr=5e-2))"
]
},
{
"cell_type": "markdown",
"id": "5ffdb723",
"metadata": {
"heading_collapsed": true
},
"source": [
"## IK, SquareCB"
]
},
{
"cell_type": "code",
"execution_count": 101,
"id": "11219b00",
"metadata": {
"hidden": true,
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"***** seed = 13 *****\n",
"***** decodability = 1 *****\n",
"***** alg = IKAlgorithm(lr=0.05 sampler=SquareCB(gamma0=10 t0=10)) *****\n",
"bno \tloss since \tpred since \tanti since \treward since \tfake since \n",
"0 \t1.32966 1.32966 \t0.10938 0.10938 \t0.09375 0.09375 \t0.10938 0.10938 \t0.33333 0.33333 \n",
"1 \t1.30494 1.28021 \t0.09375 0.07812 \t0.13281 0.17188 \t0.10938 0.10938 \t0.33776 0.34219 \n",
"2 \t1.30257 1.29784 \t0.08854 0.07812 \t0.14062 0.15625 \t0.08333 0.03125 \t0.33513 0.32985 \n",
"4 \t1.30280 1.30315 \t0.10312 0.12500 \t0.12500 0.10156 \t0.09687 0.11719 \t0.33583 0.33688 \n",
"8 \t1.29567 1.28675 \t0.12153 0.14453 \t0.10069 0.07031 \t0.11458 0.13672 \t0.33917 0.34336 \n",
"16 \t1.29371 1.29151 \t0.16085 0.20508 \t0.10938 0.11914 \t0.12592 0.13867 \t0.34092 0.34287 \n",
"32 \t1.29202 1.29022 \t0.21070 0.26367 \t0.10890 0.10840 \t0.15152 0.17871 \t0.35087 0.36145 \n",
"64 \t1.29390 1.29584 \t0.25697 0.30469 \t0.09207 0.07471 \t0.16779 0.18457 \t0.36396 0.37746 \n",
"128 \t1.29290 1.29189 \t0.34969 0.44385 \t0.06480 0.03711 \t0.21427 0.26147 \t0.38296 0.40226 \n",
"256 \t1.28826 1.28358 \t0.43020 0.51135 \t0.04596 0.02698 \t0.27243 0.33105 \t0.41878 0.45488 \n",
"512 \t1.28455 1.28083 \t0.50433 0.57874 \t0.03326 0.02051 \t0.34957 0.42700 \t0.46282 0.50702 \n",
"937 \t1.27793 1.26993 \t0.57862 0.66831 \t0.02434 0.01357 \t0.43420 0.53636 \t0.50870 0.56408 \n",
"***** decodability = -1 *****\n",
"***** alg = IKAlgorithm(lr=0.05 sampler=SquareCB(gamma0=10 t0=10)) *****\n",
"bno \tloss since \tpred since \tanti since \treward since \tfake since \n",
"0 \t1.32966 1.32966 \t0.10938 0.10938 \t0.09375 0.09375 \t0.10938 0.10938 \t0.33333 0.33333 \n",
"1 \t1.30513 1.28060 \t0.08594 0.06250 \t0.10938 0.12500 \t0.10938 0.10938 \t0.33781 0.34229 \n",
"2 \t1.30370 1.30084 \t0.09375 0.10938 \t0.11458 0.12500 \t0.08333 0.03125 \t0.33464 0.32829 \n",
"4 \t1.30378 1.30391 \t0.08125 0.06250 \t0.10938 0.10156 \t0.09687 0.11719 \t0.33765 0.34217 \n",
"8 \t1.29949 1.29413 \t0.09028 0.10156 \t0.09028 0.06641 \t0.10590 0.11719 \t0.34235 0.34822 \n",
"16 \t1.29919 1.29884 \t0.10202 0.11523 \t0.10294 0.11719 \t0.11857 0.13281 \t0.34206 0.34174 \n",
"32 \t1.29828 1.29731 \t0.11932 0.13770 \t0.12074 0.13965 \t0.12358 0.12891 \t0.34872 0.35580 \n",
"64 \t1.29811 1.29794 \t0.12212 0.12500 \t0.13197 0.14355 \t0.11731 0.11084 \t0.36252 0.37675 \n",
"128 \t1.30163 1.30521 \t0.12875 0.13550 \t0.10938 0.08643 \t0.11810 0.11890 \t0.37397 0.38560 \n",
"256 \t1.30894 1.31631 \t0.16786 0.20728 \t0.08810 0.06665 \t0.14014 0.16235 \t0.39465 0.41550 \n",
"512 \t1.31513 1.32135 \t0.24050 0.31342 \t0.06768 0.04718 \t0.19219 0.24445 \t0.42662 0.45871 \n",
"937 \t1.31834 1.32222 \t0.31413 0.40301 \t0.05549 0.04077 \t0.25568 0.33232 \t0.46456 0.51035 \n",
"***** decodability = 0.5 *****\n",
"***** alg = IKAlgorithm(lr=0.05 sampler=SquareCB(gamma0=10 t0=10)) *****\n",
"bno \tloss since \tpred since \tanti since \treward since \tfake since \n",
"0 \t1.32966 1.32966 \t0.10938 0.10938 \t0.09375 0.09375 \t0.10938 0.10938 \t0.33333 0.33333 \n",
"1 \t1.30494 1.28022 \t0.09375 0.07812 \t0.14062 0.18750 \t0.10938 0.10938 \t0.33699 0.34065 \n",
"2 \t1.30267 1.29813 \t0.09375 0.09375 \t0.14062 0.14062 \t0.08333 0.03125 \t0.33467 0.33003 \n",
"4 \t1.30326 1.30416 \t0.11250 0.14062 \t0.12500 0.10156 \t0.09687 0.11719 \t0.33453 0.33433 \n",
"8 \t1.29485 1.28433 \t0.12500 0.14062 \t0.10243 0.07422 \t0.11458 0.13672 \t0.34006 0.34697 \n",
"16 \t1.29546 1.29614 \t0.14522 0.16797 \t0.11213 0.12305 \t0.12132 0.12891 \t0.33862 0.33701 \n",
"32 \t1.29664 1.29790 \t0.16146 0.17871 \t0.09517 0.07715 \t0.13494 0.14941 \t0.34379 0.34927 \n",
"64 \t1.29866 1.30075 \t0.15986 0.15820 \t0.08053 0.06543 \t0.12933 0.12354 \t0.34898 0.35433 \n",
"128 \t1.30344 1.30830 \t0.19634 0.23340 \t0.06734 0.05396 \t0.13929 0.14941 \t0.35562 0.36236 \n",
"256 \t1.31659 1.32984 \t0.19163 0.18689 \t0.06463 0.06189 \t0.13831 0.13733 \t0.36600 0.37646 \n",
"512 \t1.33155 1.34657 \t0.21040 0.22925 \t0.05781 0.05096 \t0.15850 0.17877 \t0.37977 0.39359 \n",
"937 \t1.34935 1.37084 \t0.21973 0.23099 \t0.05574 0.05324 \t0.17377 0.19221 \t0.39498 0.41335 \n",
"***** decodability = -0.5 *****\n",
"***** alg = IKAlgorithm(lr=0.05 sampler=SquareCB(gamma0=10 t0=10)) *****\n",
"bno \tloss since \tpred since \tanti since \treward since \tfake since \n",
"0 \t1.32966 1.32966 \t0.10938 0.10938 \t0.09375 0.09375 \t0.10938 0.10938 \t0.33333 0.33333 \n",
"1 \t1.30492 1.28018 \t0.09375 0.07812 \t0.08594 0.07812 \t0.10938 0.10938 \t0.33714 0.34094 \n",
"2 \t1.30358 1.30090 \t0.08333 0.06250 \t0.09896 0.12500 \t0.08333 0.03125 \t0.33515 0.33118 \n",
"4 \t1.30400 1.30463 \t0.08438 0.08594 \t0.10000 0.10156 \t0.10000 0.12500 \t0.33447 0.33346 \n",
"8 \t1.29777 1.28998 \t0.09722 0.11328 \t0.08507 0.06641 \t0.10764 0.11719 \t0.33946 0.34568 \n",
"16 \t1.29687 1.29587 \t0.09743 0.09766 \t0.08548 0.08594 \t0.11489 0.12305 \t0.33790 0.33615 \n",
"32 \t1.29696 1.29706 \t0.11222 0.12793 \t0.09612 0.10742 \t0.12311 0.13184 \t0.34567 0.35394 \n",
"64 \t1.30000 1.30314 \t0.10481 0.09717 \t0.09832 0.10059 \t0.11034 0.09717 \t0.35347 0.36151 \n",
"128 \t1.30487 1.30981 \t0.09448 0.08398 \t0.09278 0.08716 \t0.10514 0.09985 \t0.35846 0.36353 \n",
"256 \t1.31660 1.32842 \t0.10171 0.10901 \t0.09168 0.09058 \t0.10445 0.10376 \t0.37122 0.38409 \n",
"512 \t1.33261 1.34868 \t0.12189 0.14215 \t0.08784 0.08398 \t0.11364 0.12286 \t0.38679 0.40243 \n",
"937 \t1.34934 1.36953 \t0.13471 0.15018 \t0.08136 0.07353 \t0.12328 0.13493 \t0.39942 0.41465 \n"
]
}
],
"source": [
"run_mnist_full_ci(seed=13, alglambda = lambda hilo: IKAlgorithm(hilo=hilo, sampler=SquareCB(gamma0=10, t0=10), lr=5e-2))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8c587b93",
"metadata": {
"hidden": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.7"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment