How do I set up a nix-shell script for lua?

You have to set LUA_PATH and LUA_CPATH

This shell.nix should work,

with import <nixpkgs> {};
with luaPackages;

let
  libs = [lua cjson luasocket luasec];
in
stdenv.mkDerivation rec {
  name = "lua-env";
  buildInputs = libs;

  shellHook = ''
    export LUA_CPATH="${lib.concatStringsSep ";" (map getLuaCPath libs)}"
    export LUA_PATH="${lib.concatStringsSep ";" (map getLuaPath libs)}"
  '';
}

To check that those paths are exported,

$ nix-shell --run 'echo $LUA_CPATH; echo $LUA_PATH'
/nix/store/lp0ns0hjwx1klk6amnyic3f62bw7h8j7-lua-5.2.3/lib/lua/5.2/?.so;/nix/store/lp0ns0hjwx1klk6amnyic3f62bw7h8j7-lua-5.2.3/share/lua/5.2/?.so;/nix/store/ppdspfcm5nnfz0fk9zarjmpyv5lcmn18-lua5.2-cjson-2.1.0/lib/lua/5.2/?.so;/nix/store/ppdspfcm5nnfz0fk9zarjmpyv5lcmn18-lua5.2-cjson-2.1.0/share/lua/5.2/?.so;/nix/store/b71xyq7gkc8iccj6wr84zq31gqc3m9ix-lua5.2-socket-3.0-rc1/lib/lua/5.2/?.so;/nix/store/b71xyq7gkc8iccj6wr84zq31gqc3m9ix-lua5.2-socket-3.0-rc1/share/lua/5.2/?.so;/nix/store/6q8xn2bcfskmrfzql47jld5d2irvn5jr-lua5.2-sec-0.6/lib/lua/5.2/?.so;/nix/store/6q8xn2bcfskmrfzql47jld5d2irvn5jr-lua5.2-sec-0.6/share/lua/5.2/?.so
/nix/store/lp0ns0hjwx1klk6amnyic3f62bw7h8j7-lua-5.2.3/lib/lua/5.2/?.lua;/nix/store/lp0ns0hjwx1klk6amnyic3f62bw7h8j7-lua-5.2.3/share/lua/5.2/?.lua;/nix/store/ppdspfcm5nnfz0fk9zarjmpyv5lcmn18-lua5.2-cjson-2.1.0/lib/lua/5.2/?.lua;/nix/store/ppdspfcm5nnfz0fk9zarjmpyv5lcmn18-lua5.2-cjson-2.1.0/share/lua/5.2/?.lua;/nix/store/b71xyq7gkc8iccj6wr84zq31gqc3m9ix-lua5.2-socket-3.0-rc1/lib/lua/5.2/?.lua;/nix/store/b71xyq7gkc8iccj6wr84zq31gqc3m9ix-lua5.2-socket-3.0-rc1/share/lua/5.2/?.lua;/nix/store/6q8xn2bcfskmrfzql47jld5d2irvn5jr-lua5.2-sec-0.6/lib/lua/5.2/?.lua;/nix/store/6q8xn2bcfskmrfzql47jld5d2irvn5jr-lua5.2-sec-0.6/share/lua/5.2/?.lua