viewof game = {
const {q, rewards} = machines;
// N only indexes the reward tables below; the warm-up never shows it.
const state = {N: [0, 0], history: []};
const progress = html`<span></span>`;
const total = html`<span></span>`;
const newGame = html`<button type="button" class="mab-new">New game</button>`;
newGame.onclick = () => {
const input = viewof gameNo;
input.value = randomGameNo(gameNo);
input.dispatchEvent(new Event("input", {bubbles: true}));
};
const cards = machineNames.map((name, a) => {
const reward = html`<span class="mab-reward">–</span>`;
const button = html`<button type="button" class="mab-machine"
style="--machine: ${machineColors[a]}" aria-label="Pull machine ${name}">
<span class="mab-name">Machine ${name}</span>${reward}<span class="mab-cta">Pull</span>
</button>`;
button.onclick = () => pull(a);
const paid = html`<div class="mab-paid" aria-label="Rewards paid by machine ${name}"></div>`;
const reveal = html`<div class="mab-reveal"></div>`;
return {name, reward, button, paid, reveal};
});
const message = html`<div class="mab-message" aria-live="polite"></div>`;
const root = html`<div class="mab-game">
<div class="mab-status">${progress}${total}<span class="mab-spacer"></span>${newGame}</div>
<div class="mab-machines">${cards.map(c => html`<div>${c.button}${c.paid}${c.reveal}</div>`)}</div>
${message}
</div>`;
// Quarto's OJS runtime gives every button in a cell output Bootstrap's .btn classes,
// whose hover, focus and disabled states would repaint these cards; take them off again.
const unstyle = new MutationObserver(() => {
for (const b of root.querySelectorAll("button.btn")) b.classList.remove("btn", "btn-quarto");
});
unstyle.observe(root, {subtree: true, attributeFilter: ["class"]});
invalidation.then(() => unstyle.disconnect());
// One pull: the first, second, ... pull of machine a pays rewards[a][0], rewards[a][1], ...
function pull(a) {
if (state.history.length >= T) return;
const reward = rewards[a][state.N[a]];
state.N[a] += 1;
state.history.push({t: state.history.length + 1, a, machine: machineNames[a], reward});
render();
root.value = snapshot();
root.dispatchEvent(new Event("input", {bubbles: true}));
}
function snapshot() {
return {gameNo, T, q, history: state.history.slice(), done: state.history.length === T};
}
function render() {
const t = state.history.length;
const done = t === T;
const last = state.history[t - 1];
const better = q[0] >= q[1] ? 0 : 1;
progress.replaceChildren(html`Pulls: <strong>${t}</strong> / ${T}`);
total.replaceChildren(html`Total reward: <strong>${fmt(state.history.reduce((s, h) => s + h.reward, 0))}</strong>`);
// Separate nodes: html`` wraps several top-level elements in one span, which would
// make each row a single flex item.
cards.forEach((c, a) => {
const own = state.history.filter(h => h.a === a);
c.reward.textContent = own.length ? signed(own[own.length - 1].reward) : "–";
c.button.classList.toggle("is-latest", last?.a === a);
c.button.disabled = done;
c.paid.replaceChildren(...(own.length
? [html`<div class="mab-muted">Paid so far:</div>`,
html`<div class="mab-paid-values">${own.flatMap(h => [html`<span>${fmt(h.reward)}</span>`, " "])}</div>`]
: [html`<div class="mab-muted">Not pulled yet</div>`]));
c.reveal.replaceChildren(...(done
? [html`<span>Average payout:</span>`, " ", html`<span><strong>${fmt(q[a])}</strong></span>`,
...(a === better ? [" ", html`<span class="mab-badge">better</span>`] : [])]
: []));
});
message.replaceChildren(
last
? html`<div><strong>Pull ${last.t}:</strong> machine ${last.machine} paid <strong>${fmt(last.reward)}</strong>.</div>`
: html`<span class="mab-muted">Click a machine to make your first pull.</span>`,
...(done ? [html`<div><strong>That was your last pull.</strong> See how you did below.</div>`] : []));
}
render();
root.value = snapshot();
return root;
}6 A Two-armed Bandit Warm-up
Play this game before reading Multi-armed Bandits. In front of you are two slot machines, A and B. Each pull of a machine pays a random reward: sometimes more, sometimes less, sometimes negative. One machine pays more on average than the other, but you do not know which. You have a fixed number of pulls, and your goal is to collect as much reward as possible.
6.1 How to play
Click a machine to pull it. Under each machine, you see every reward it has paid you so far; use them however you like to decide which machine to pull next. When your pulls run out, the average payout of each machine is revealed.
6.2 The game
6.3 Your results
6.4 Things to think about
- How did you decide which machine to pull next? Try to write your rule in one sentence.
- Which numbers did you use to compare the machines: the last reward, the largest one, the average, or something else?
- When did you stop trying the other machine? Could you have been wrong about which machine was better?
- Would you play differently with 10 pulls than with 50? Why?
The chapter Multi-armed Bandits turns these questions into an algorithm. After reading it, play A Two-armed Bandit Game, which shows the numbers that the algorithm keeps and compares your choices with it.