Forum Discussion
Will multiple queries help with 502 Bad Gateway errors?
- Anonymous2 years ago
getItems()
1) start with the project list.
2) List.Generate() to paginate every 1000 records.
3) getProjectURIpaginated([projectNo])
4) append all the queries
let Source = (qrySource as table) => let #"01listGenerate" = Table.AddColumn(qrySource, "dRofusRecord", each List.Generate(() => [skip = 0, top = 1000 ], each [skip] <= rmSkip*10, // each room has less than 10,000 items. each [skip = [skip] + [top], top = [top] ] )), #"02expandList" = Table.ExpandListColumn(#"01listGenerate", "dRofusRecord"), #"03dRofusRecord" = Table.ExpandRecordColumn(#"02expandList", "dRofusRecord", {"skip", "top"}, {"skip", "top"}), #"04changeType" = Table.TransformColumnTypes(#"03dRofusRecord",{{"skip", Int64.Type}, {"top", Int64.Type}}), #"10getProjURI" = Table.AddColumn(#"04changeType", "dRofusRecord", each getProjectURIpaginated([projNo], "occurrences", #"105APImapOccurrences", [top], [skip])), #"11expandProjURI" = Table.ExpandListColumn(#"10getProjURI", "dRofusRecord"), #"12expandDrofusRecord" = Table.ExpandRecordColumn(#"11expandProjURI", "dRofusRecord ....getProjectURIpaginated()
(projectNo as text, objectType as text, selectColumnTable as table, topVal as number, skipVal as number)=> let #"FilterColumnTable" = Table.SelectRows(selectColumnTable, each ([ObjectType] = objectType)), GetWebContents = Web.Contents( "https://api-us.drofus.com/api/company/", [ RelativePath= projectNo & "/" & objectType & "?$select=" & Text.Combine(#"FilterColumnTable"[dRofus.ID], ",") & "&$top=" & Number.ToText(topVal) & "&$skip=" & Number.ToText(skipVal) ] ), GetBody = Json.Document(GetWebContents) in GetBody
getItems()
1) start with the project list.
2) List.Generate() to paginate every 1000 records.
3) getProjectURIpaginated([projectNo])
4) append all the queries
let
Source = (qrySource as table) => let
#"01listGenerate" = Table.AddColumn(qrySource, "dRofusRecord", each List.Generate(() =>
[skip = 0, top = 1000 ],
each [skip] <= rmSkip*10, // each room has less than 10,000 items.
each [skip = [skip] + [top],
top = [top] ]
)),
#"02expandList" = Table.ExpandListColumn(#"01listGenerate", "dRofusRecord"),
#"03dRofusRecord" = Table.ExpandRecordColumn(#"02expandList", "dRofusRecord", {"skip", "top"}, {"skip", "top"}),
#"04changeType" = Table.TransformColumnTypes(#"03dRofusRecord",{{"skip", Int64.Type}, {"top", Int64.Type}}),
#"10getProjURI" = Table.AddColumn(#"04changeType", "dRofusRecord", each getProjectURIpaginated([projNo], "occurrences", #"105APImapOccurrences", [top], [skip])),
#"11expandProjURI" = Table.ExpandListColumn(#"10getProjURI", "dRofusRecord"),
#"12expandDrofusRecord" = Table.ExpandRecordColumn(#"11expandProjURI", "dRofusRecord
....
getProjectURIpaginated()
(projectNo as text, objectType as text, selectColumnTable as table, topVal as number, skipVal as number)=>
let
#"FilterColumnTable" = Table.SelectRows(selectColumnTable, each ([ObjectType] = objectType)),
GetWebContents =
Web.Contents(
"https://api-us.drofus.com/api/company/",
[
RelativePath= projectNo
& "/" & objectType
& "?$select=" & Text.Combine(#"FilterColumnTable"[dRofus.ID], ",")
& "&$top=" & Number.ToText(topVal)
& "&$skip=" & Number.ToText(skipVal)
]
),
GetBody = Json.Document(GetWebContents)
in
GetBody
Hi Anonymous,
I think the error message may be related to you send too many requests which may trigger the limits on the API.
For this scenario, you can try to add interval to your loop to reduce the concurrent request amount and increase processing times.
Regards,
Xiaoxin Sheng
- Anonymous2 years agoNot applicable
Thank you Xiaoxin. When I change the interval to 100 (top = 100), the getProjectURIpaginated() runs without 502 errors.