Forum Discussion

Anonymous's avatar
Anonymous
Not applicable
2 years ago
Solved

Will multiple queries help with 502 Bad Gateway errors?

I use pagination in my query, but I still receive a 502 Bad Gateway error. Each time it is a different record triggering the error.   The data source has 200+ projects with 220k records. When I li...
  • Anonymous's avatar
    Anonymous
    2 years ago

    getItems()

    1) start with the project list.

    2) List.Generate() to paginate every 1000 records.

    3) getProjectURIpaginated([projectNo])

    4) append all the queries

     

     

     

    let
        Source = (qrySource as table) => let
    
        #"01listGenerate" = Table.AddColumn(qrySource, "dRofusRecord", each List.Generate(() =>
    [skip = 0,  top = 1000 ],
    each [skip] <= rmSkip*10,   // each room has less than 10,000 items.
    each [skip = [skip] + [top], 
          top = [top] ]
    )),
    
        #"02expandList" = Table.ExpandListColumn(#"01listGenerate", "dRofusRecord"),
        #"03dRofusRecord" = Table.ExpandRecordColumn(#"02expandList", "dRofusRecord", {"skip", "top"}, {"skip", "top"}),
        #"04changeType" = Table.TransformColumnTypes(#"03dRofusRecord",{{"skip", Int64.Type}, {"top", Int64.Type}}),
        #"10getProjURI" = Table.AddColumn(#"04changeType", "dRofusRecord", each getProjectURIpaginated([projNo], "occurrences", #"105APImapOccurrences", [top], [skip])),
        #"11expandProjURI" = Table.ExpandListColumn(#"10getProjURI", "dRofusRecord"),
        #"12expandDrofusRecord" = Table.ExpandRecordColumn(#"11expandProjURI", "dRofusRecord
    ....

     

     

     

    getProjectURIpaginated()

     

     

    (projectNo as text, objectType as text, selectColumnTable as table, topVal as number, skipVal as number)=>
    let
        #"FilterColumnTable" = Table.SelectRows(selectColumnTable, each ([ObjectType] = objectType)),
        GetWebContents =
            Web.Contents(
                "https://api-us.drofus.com/api/company/",
                [
                    RelativePath= projectNo
                    & "/" & objectType
                    & "?$select=" & Text.Combine(#"FilterColumnTable"[dRofus.ID], ",")
                    & "&$top=" & Number.ToText(topVal)
                    & "&$skip=" & Number.ToText(skipVal) 
                ]
            ),            
        GetBody = Json.Document(GetWebContents)    
    in
        GetBody