I am trying to fetch all data from elastis search using Scroll API. But I am always getting same records back from each call.
let
todate = Int16.From(Date.FromText(Date.ToText(DateTime.Date(DateTime.LocalNow()),"yyyy-MM-dd")) - #date(1970, 1, 1))*86400,
fromdate = Int16.From(Date.FromText(Date.ToText(Date.AddDays(DateTime.Date(DateTime.LocalNow()),-30),"yyyy-MM-dd")) - #date(1970, 1, 1))*86400,
querySize = 10,
GetPage1 = () =>
let url = "http://elasticsearch:9200/cases/_search",
url1= "?scroll=1m&size=10&pretty=true&q=creation:[",
path = Text.Combine({url1,Number.ToText(fromdate),"+TO+",Number.ToText(todate),"]"}),
Source = Json.Document(Web.Contents(url,[RelativePath=path])),
scroll = Source[_scroll_id],
hits = Source[hits],
hits1 = hits[hits]
in
Source,
GetScroll = (Scroll_ID) =>
let Scroll_Url = "http://elasticsearch:9200/_search/scroll",
Scroll_Path= "?scroll=1m&scroll_id=" & Scroll_ID,
Scroll_Source = Json.Document(Web.Contents(Scroll_Url,[RelativePath=Scroll_Path]))
in
Scroll_Source,
Raw_Data = GetPage1(),
scroll = Raw_Data[_scroll_id],
data = Raw_Data[hits],
Mod_Res = Number.Mod(data[total],querySize),
Hits= data[hits],
num = if (Mod_Res>5) then
0
else
1,
loop = Int16.From(data[total]/querySize)+num,
scrollList = List.Generate(()=> [i=0] ,each [i] < loop, each [i =[i]+1], each GetScroll(scroll)),
ScrollData = GetScroll(scroll),
scroll_hits = ScrollData[hits],
hit_scroll = scroll_hits[hits],
FinalList = List.Combine({Hits,hit_scroll})
in
scrollList
IF THERE IS SOME OTHER METHOD TO FETCH ALL DATA FROM ELASTIC SEARCH. Please let me know.
