Forum Discussion
Paginated api call issue - Function do not call the last page
- 1 year ago
freidav1984
Hi, The issue with yourList.Generate()function stopping before retrieving the last page is due to the ExitCondition being triggered too early. Specifically, when thenextfield isnull(or empty),List.Generate()immediately stops before processing the last page's data.
use this codelet
fn_getExpData = (initialUrl as text) =>
let
BaseUrl = "https://xxx.xxxxx.xxxxx.co.hu/2/expenses",AllData = List.Generate(
() =>
[
CurrentUrl = initialUrl,
Data = {},
Continue = true
],
each [Continue], // Keep iterating while Continue = true
each
let
CurrentUrl = [CurrentUrl],// Fetch API response
Response = try Json.Document(Web.Contents(BaseUrl, [Query=if CurrentUrl <> null then [next=CurrentUrl] else [], Headers=[Authorization="Bearer " & Token]])) otherwise null,// Extract Data
DataChunk = try Response[results][expenses] otherwise {},// Add metadata
Timestamp = DateTime.LocalNow(),
DataWithMetadata = List.Transform(DataChunk, each
Record.AddField(
Record.AddField(_, "API_URL", CurrentUrl),
"Response_Timestamp", Timestamp
)
),// Get Next Page URL
NextUrl = try Response[next] otherwise null,// Continue if there's data OR if there's another page
NewContinue = NextUrl <> null
in
[
CurrentUrl = NextUrl,
Data = DataWithMetadata,
Continue = NewContinue
],
each [Data] // Extract only the Data part
),FlattenedData = List.Combine(AllData) // Flatten list
in
FlattenedData
in
fn_getExpDataDid I answer your question? Mark my post as a solution! Appreciate your Kudos !!
freidav1984
Hi, The issue with your List.Generate() function stopping before retrieving the last page is due to the ExitCondition being triggered too early. Specifically, when the next field is null (or empty), List.Generate() immediately stops before processing the last page's data.
use this code
let
fn_getExpData = (initialUrl as text) =>
let
BaseUrl = "https://xxx.xxxxx.xxxxx.co.hu/2/expenses",
AllData = List.Generate(
() =>
[
CurrentUrl = initialUrl,
Data = {},
Continue = true
],
each [Continue], // Keep iterating while Continue = true
each
let
CurrentUrl = [CurrentUrl],
// Fetch API response
Response = try Json.Document(Web.Contents(BaseUrl, [Query=if CurrentUrl <> null then [next=CurrentUrl] else [], Headers=[Authorization="Bearer " & Token]])) otherwise null,
// Extract Data
DataChunk = try Response[results][expenses] otherwise {},
// Add metadata
Timestamp = DateTime.LocalNow(),
DataWithMetadata = List.Transform(DataChunk, each
Record.AddField(
Record.AddField(_, "API_URL", CurrentUrl),
"Response_Timestamp", Timestamp
)
),
// Get Next Page URL
NextUrl = try Response[next] otherwise null,
// Continue if there's data OR if there's another page
NewContinue = NextUrl <> null
in
[
CurrentUrl = NextUrl,
Data = DataWithMetadata,
Continue = NewContinue
],
each [Data] // Extract only the Data part
),
FlattenedData = List.Combine(AllData) // Flatten list
in
FlattenedData
in
fn_getExpData
Did I answer your question? Mark my post as a solution! Appreciate your Kudos !!