Elasticsearch show all results using scroll in node js
Thanks @Ceilingfish. Here's a modified ES6 version of the above using await
let allRecords = [];
// first we do a search, and specify a scroll timeout
var { _scroll_id, hits } = await esclient.search({
index: 'test',
type: 'records',
scroll: '10s',
body: {
query: {
"match_all": {}
},
_source: false
}
})
while(hits && hits.hits.length) {
// Append all new hits
allRecords.push(...hits.hits)
console.log(`${allRecords.length} of ${hits.total}`)
var { _scroll_id, hits } = await esclient.scroll({
scrollId: _scroll_id,
scroll: '10s'
})
}
console.log(`Complete: ${allRecords.length} records retrieved`)
You need to repeatedly call client.scroll
until no more records are returned. There's a good example in the elasticsearch documentation. I've reproduced their example code below, slightly modified to match your question
var allRecords = [];
// first we do a search, and specify a scroll timeout
client.search({
index: 'test',
type: 'records',
scroll: '10s',
body: {
query: {
"match_all": {}
}
}
}, function getMoreUntilDone(error, response) {
// collect all the records
response.body.hits.hits.forEach(function (hit) {
allRecords.push(hit);
});
if (response.body.hits.total.value !== allRecords.length) {
// now we can call scroll over and over
client.scroll({
scroll_id: response.body._scroll_id,
scroll: '10s'
}, getMoreUntilDone);
} else {
console.log('all done', allRecords);
}
});