Specify absolute colour for 3D points in MayaVi

After struggling with this for most of today, I found a relatively simple way to do exactly what the question asks -- specify an RGB tuple for each point. The trick is just to define a color map with exactly the same number of entries as there are points to plot, and then set the argument to be a list of indices:

# Imports
import numpy as np
from mayavi.mlab import quiver3d, draw

# Primitives
N = 200 # Number of points
ones = np.ones(N)
scalars = np.arange(N) # Key point: set an integer for each point

# Define color table (including alpha), which must be uint8 and [0,255]
colors = (np.random.random((N, 4))*255).astype(np.uint8)
colors[:,-1] = 255 # No transparency

# Define coordinates and points
x, y, z = colors[:,0], colors[:,1], colors[:,2] # Assign x, y, z values to match color
pts = quiver3d(x, y, z, ones, ones, ones, scalars=scalars, mode='sphere') # Create points
pts.glyph.color_mode = 'color_by_scalar' # Color by scalar

# Set look-up table and redraw
pts.module_manager.scalar_lut_manager.lut.table = colors
draw()

I've found a better way to set the colors directly.

You can create your own direct LUT pretty easily. Let's say we want 256**3 granularity:

#create direct grid as 256**3 x 4 array 
def create_8bit_rgb_lut():
    xl = numpy.mgrid[0:256, 0:256, 0:256]
    lut = numpy.vstack((xl[0].reshape(1, 256**3),
                        xl[1].reshape(1, 256**3),
                        xl[2].reshape(1, 256**3),
                        255 * numpy.ones((1, 256**3)))).T
    return lut.astype('int32')

# indexing function to above grid
def rgb_2_scalar_idx(r, g, b):
    return 256**2 *r + 256 * g + b

#N x 3 colors. <This is where you are storing your custom colors in RGB>
colors = numpy.array([_.color for _ in points])

#N scalars
scalars = numpy.zeros((colors.shape[0],))

for (kp_idx, kp_c) in enumerate(colors):
    scalars[kp_idx] = rgb_2_scalar_idx(kp_c[0], kp_c[1], kp_c[2])

rgb_lut = create_8bit_rgb_lut()

points_mlab = mayavi.mlab.points3d(x, y, z, scalars, mode='point')

#magic to modify lookup table 
points_mlab.module_manager.scalar_lut_manager.lut._vtk_obj.SetTableRange(0, rgb_lut.shape[0])
points_mlab.module_manager.scalar_lut_manager.lut.number_of_colors = rgb_lut.shape[0]
points_mlab.module_manager.scalar_lut_manager.lut.table = rgb_lut