Forum Discussion
mkubala
2 years agoRegular Visitor
SharePoint REST API not returning all results
Hello, I found the below code to use the SharePoint REST API to pull in large amounts of data from a SharePoint library. I have a document library with over 130,000 documents that I want to creat...
- 2 years ago
I found this blog post which has an updated script that works: Updated – Get SharePoint List Data … Fast – Hoosier BI
let siteurl = "https://<your tenant url>/sites/<your site name>", // use ... /sites/<your site name>/<your subsite name> if applicable listname = "<your list name>", itemcount = Json.Document(Web.Contents(siteurl, [RelativePath = "_api/web/lists/GetByTitle('" & listname & "')/items?$select=ID&$orderby=ID%20desc&$top=1", Headers = [Accept = "application/json"]]))[value]{0}[ID], StartIDs = List.Numbers(0, Number.RoundUp(itemcount / 5000), 5000), ConvertToTable = Table.FromList(StartIDs, Splitter.SplitByNothing(), null, null, ExtraValues.Error), Add_EndIDs = Table.AddColumn(ConvertToTable, "Addition", each [Column1] + 4999, type number), RenamedColumns = Table.RenameColumns(Add_EndIDs, {{"Column1", "StartID"}, {"Addition", "EndID"}}), #"Changed Type" = Table.TransformColumnTypes(RenamedColumns, {{"StartID", type text}, {"EndID", type text}}), //Comment in only one of the fieldselect lines below, defining your select and expand columns using the example syntax shown fieldselect = "&$top=5000", // all fields with no expansion //fieldselect = "&$top=5000&$select=ID,Title,Date,PersonColumn,ChoiceColumn,LookupColumn", // list desired fields (no expansion) -No Spaces! //fieldselect = "&$top=5000&$select=ID,Title,Date,PersonColumn/LastName,PersonColumn/FirstName,ChoiceColumn,LookupColumn/Title,LookupColumn/Project,LookupColumn/ProjectStatus&$expand=PersonColumn,LookupColumn", //expand list fields - No Spaces! GetData = Table.AddColumn(#"Changed Type", "Items", each Json.Document(Web.Contents(siteurl, [RelativePath = "_api/web/lists/GetByTitle('" & listname & "')/items?$filter=(ID ge " & [StartID] & ") and (ID le " & [EndID] & ")" & fieldselect, Headers = [Accept = "application/json"]]))[value]), #"Removed Other Columns" = Table.SelectColumns(GetData, {"Items"}), #"Expanded Items" = Table.ExpandListColumn(#"Removed Other Columns", "Items") in #"Expanded Items"or this one which accomplishes the same thing
let siteurl = "https://<your tenant url>/sites/<your site name>", // use ... /sites/<your site name>/<your subsite name> if applicable listname = "<your list name>", //Comment in only one of the fieldselect lines below, defining your select and expand columns using the example syntax shown fieldselect = "&$top=5000", // all fields with no expansion //fieldselect = "&$top=5000&$select=ID,Title,Date,PersonColumn,ChoiceColumn,LookupColumn", // list desired fields (no expansion) -No Spaces! //fieldselect = "&$top=5000&$select=ID,Title,Date,PersonColumn/LastName,PersonColumn/FirstName,ChoiceColumn,LookupColumn/Title,LookupColumn/Project,LookupColumn/ProjectStatus&$expand=PersonColumn,LookupColumn", //expand list fields - No Spaces! InitialWebCall = Json.Document(Web.Contents(siteurl, [RelativePath = "_api/web/lists/GetByTitle('" & listname & "')/items?$skipToken=Paged=TRUE" & fieldselect, Headers = [Accept = "application/json"]])), datalist = List.Generate(() => InitialWebCall, each List.Count([value]) > 0, each try Json.Document(Web.Contents(siteurl, [RelativePath = "_api" & Text.AfterDelimiter([odata.nextLink], "_api"), Headers = [Accept = "application/json"]])) otherwise [value = {}], each [value]), #"Converted to Table" = Table.FromList(datalist, Splitter.SplitByNothing(), null, null, ExtraValues.Error), #"Expanded Column1" = Table.ExpandListColumn(#"Converted to Table", "Column1") in #"Expanded Column1"