Panel not getting focus

The Panel class was designed as container, it avoids taking the focus so a child control will always get it. You'll need some surgery to fix that. I threw in the code to get cursor key strokes in the KeyDown event as well:

using System;
using System.Drawing;
using System.Windows.Forms;

class SelectablePanel : Panel {
    public SelectablePanel() {
        this.SetStyle(ControlStyles.Selectable, true);
        this.TabStop = true;
    }
    protected override void OnMouseDown(MouseEventArgs e) {
        this.Focus();
        base.OnMouseDown(e);
    }
    protected override bool IsInputKey(Keys keyData) {
        if (keyData == Keys.Up || keyData == Keys.Down) return true;
        if (keyData == Keys.Left || keyData == Keys.Right) return true;
        return base.IsInputKey(keyData);
    }
    protected override void OnEnter(EventArgs e) {
        this.Invalidate();
        base.OnEnter(e);
    }
    protected override void OnLeave(EventArgs e) {
        this.Invalidate();
        base.OnLeave(e);
    }
    protected override void OnPaint(PaintEventArgs pe) {
        base.OnPaint(pe);
        if (this.Focused) {
            var rc = this.ClientRectangle;
            rc.Inflate(-2, -2);
            ControlPaint.DrawFocusRectangle(pe.Graphics, rc);
        }
    }
}

The code from Hans Passant translated to VB.NET

Imports System
Imports System.Drawing
Imports System.Windows.Forms

Public Class SelectablePanel
    Inherits Panel

    Public Sub New()
        Me.SetStyle(ControlStyles.Selectable, True)
        Me.TabStop = True
    End Sub
    
    Protected Overrides Sub OnMouseDown(ByVal e As MouseEventArgs)
        Me.Focus()
        MyBase.OnMouseDown(e)
    End Sub

    Protected Overrides Function IsInputKey(ByVal keydata As Keys) As Boolean
        If (keydata = Keys.Up OrElse keydata = Keys.Down) Then Return True
        If (keydata = Keys.Left OrElse keydata = Keys.Right) Then Return True
        Return MyBase.IsInputKey(keydata)
    End Function

    Protected Overrides Sub OnEnter(ByVal e As EventArgs)
        Me.Invalidate()
        MyBase.OnEnter(e)
    End Sub

    Protected Overrides Sub OnLeave(ByVal e As EventArgs)
        Me.Invalidate()
        MyBase.OnLeave(e)
    End Sub

    Protected Overrides Sub OnPaint(ByVal pe As PaintEventArgs)
        MyBase.OnPaint(pe)
        If (Me.Focused) Then
            Dim rc As Rectangle = Me.ClientRectangle
            rc.Inflate(-2, -2)
            ControlPaint.DrawFocusRectangle(pe.Graphics, rc)
        End If
    End Sub

End Class