what's the correct way to send a file from REST web service to client?

I don't recommend encoding binary data in base64 and wrapping it in JSON. It will just needlessly increase the size of the response and slow things down.

Simply serve your file data using GET and application/octect-streamusing one of the factory methods of javax.ws.rs.core.Response (part of the JAX-RS API, so you're not locked into Jersey):

@GET
@Produces(MediaType.APPLICATION_OCTET_STREAM)
public Response getFile() {
  File file = ... // Initialize this to the File path you want to serve.
  return Response.ok(file, MediaType.APPLICATION_OCTET_STREAM)
      .header("Content-Disposition", "attachment; filename=\"" + file.getName() + "\"" ) //optional
      .build();
}

If you don't have an actual File object, but an InputStream, Response.ok(entity, mediaType) should be able to handle that as well.

If you want to return a File to be downloaded, specially if you want to integrate with some javascript libs of file upload/download, then the code bellow should do the job:

@GET
@Path("/{key}")
public Response download(@PathParam("key") String key,
                         @Context HttpServletResponse response) throws IOException {
    try {
        //Get your File or Object from wherever you want...
            //you can use the key parameter to indentify your file
            //otherwise it can be removed
        //let's say your file is called "object"
        response.setContentLength((int) object.getContentLength());
        response.setHeader("Content-Disposition", "attachment; filename="
                + object.getName());
        ServletOutputStream outStream = response.getOutputStream();
        byte[] bbuf = new byte[(int) object.getContentLength() + 1024];
        DataInputStream in = new DataInputStream(
                object.getDataInputStream());
        int length = 0;
        while ((in != null) && ((length = in.read(bbuf)) != -1)) {
            outStream.write(bbuf, 0, length);
        }
        in.close();
        outStream.flush();
    } catch (S3ServiceException e) {
        e.printStackTrace();
    } catch (ServiceException e) {
        e.printStackTrace();
    }
    return Response.ok().build();
}

Change the machine address from localhost to IP address you want your client to connect with to call below mentioned service.

Client to call REST webservice:

package in.india.client.downloadfiledemo;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;

import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response.Status;

import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.multipart.BodyPart;
import com.sun.jersey.multipart.MultiPart;

public class DownloadFileClient {

    private static final String BASE_URI = "http://localhost:8080/DownloadFileDemo/services/downloadfile";

    public DownloadFileClient() {

        try {
            Client client = Client.create();
            WebResource objWebResource = client.resource(BASE_URI);
            ClientResponse response = objWebResource.path("/")
                    .type(MediaType.TEXT_HTML).get(ClientResponse.class);

            System.out.println("response : " + response);
            if (response.getStatus() == Status.OK.getStatusCode()
                    && response.hasEntity()) {
                MultiPart objMultiPart = response.getEntity(MultiPart.class);
                java.util.List<BodyPart> listBodyPart = objMultiPart
                        .getBodyParts();
                BodyPart filenameBodyPart = listBodyPart.get(0);
                BodyPart fileLengthBodyPart = listBodyPart.get(1);
                BodyPart fileBodyPart = listBodyPart.get(2);

                String filename = filenameBodyPart.getEntityAs(String.class);
                String fileLength = fileLengthBodyPart
                        .getEntityAs(String.class);
                File streamedFile = fileBodyPart.getEntityAs(File.class);

                BufferedInputStream objBufferedInputStream = new BufferedInputStream(
                        new FileInputStream(streamedFile));

                byte[] bytes = new byte[objBufferedInputStream.available()];

                objBufferedInputStream.read(bytes);

                String outFileName = "D:/"
                        + filename;
                System.out.println("File name is : " + filename
                        + " and length is : " + fileLength);
                FileOutputStream objFileOutputStream = new FileOutputStream(
                        outFileName);
                objFileOutputStream.write(bytes);
                objFileOutputStream.close();
                objBufferedInputStream.close();
                File receivedFile = new File(outFileName);
                System.out.print("Is the file size is same? :\t");
                System.out.println(Long.parseLong(fileLength) == receivedFile
                        .length());
            }
        } catch (UniformInterfaceException e) {
            e.printStackTrace();
        } catch (ClientHandlerException e) {
            e.printStackTrace();
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }

    }

    public static void main(String... args) {
        new DownloadFileClient();
    }
}

Service to response client:

package in.india.service.downloadfiledemo;

import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;

import com.sun.jersey.multipart.MultiPart;

@Path("downloadfile")
@Produces("multipart/mixed")
public class DownloadFileResource {

    @GET
    public Response getFile() {

        java.io.File objFile = new java.io.File(
                "D:/DanGilbert_2004-480p-en.mp4");
        MultiPart objMultiPart = new MultiPart();
        objMultiPart.type(new MediaType("multipart", "mixed"));
        objMultiPart
                .bodyPart(objFile.getName(), new MediaType("text", "plain"));
        objMultiPart.bodyPart("" + objFile.length(), new MediaType("text",
                "plain"));
        objMultiPart.bodyPart(objFile, new MediaType("multipart", "mixed"));

        return Response.ok(objMultiPart).build();

    }
}

JAR needed:

jersey-bundle-1.14.jar
jersey-multipart-1.14.jar
mimepull.jar

WEB.XML:

<?xml version="1.0" encoding="UTF-8"?>
<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xmlns="http://java.sun.com/xml/ns/javaee" xmlns:web="http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"
    xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"
    id="WebApp_ID" version="2.5">
    <display-name>DownloadFileDemo</display-name>
    <servlet>
        <display-name>JAX-RS REST Servlet</display-name>
        <servlet-name>JAX-RS REST Servlet</servlet-name>
        <servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
        <init-param>
             <param-name>com.sun.jersey.config.property.packages</param-name> 
             <param-value>in.india.service.downloadfiledemo</param-value>
        </init-param>
        <load-on-startup>1</load-on-startup>
    </servlet>
    <servlet-mapping>
        <servlet-name>JAX-RS REST Servlet</servlet-name>
        <url-pattern>/services/*</url-pattern>
    </servlet-mapping>
    <welcome-file-list>
        <welcome-file>index.jsp</welcome-file>
    </welcome-file-list>
</web-app>