Forum Discussion
Not all data is coming in using Web with JSON
I have a set of 5 web sources where I get data from (ServiceNow APIs) all in JSON format.
However one of the is very inconsistence, according to ServiceNow and using other tools I get all rows, but with Power BI the number can be anything from 1000 OFF to 200-300 off the actual number of rows, here is my query, am I doing something wrong:
let
Source = Json.Document(Web.Contents("https://<myInstance>.service-now.com/api/now/table/incident?sysparm_display_value=true",[Timeout=#duration(0,4,0,0)])),
result = Source[result],
#"Converted to Table" = Table.FromList(result, Splitter.SplitByNothing(), null, null, ExtraValues.Ignore),
#"Expanded Column1" = Table.ExpandRecordColumn(#"Converted to Table", "Column1", {"parent", "made_sla", "u_notification_major_incident", "caused_by", "watch_list", "upon_reject", "sys_updated_on", "child_incidents", "hold_reason", "approval_history", "number", "resolved_by", "sys_updated_by", "opened_by", "user_input", "sys_created_on", "sys_domain", "state", "sys_created_by", "knowledge", "order", "calendar_stc", "closed_at", "cmdb_ci", "delivery_plan", "impact", "active", "work_notes_list", "business_service", "priority", "sys_domain_path", "rfc", "time_worked", "expected_start", "opened_at", "business_duration", "group_list", "work_end", "caller_id", "resolved_at", "approval_set", "subcategory", "work_notes", "short_description", "close_code", "correlation_display", "delivery_task", "work_start", "assignment_group", "additional_assignee_list", "business_stc", "description", "calendar_duration", "close_notes", "notify", "sys_class_name", "closed_by", "follow_up", "parent_incident", "sys_id", "contact_type", "incident_state", "urgency", "problem_id", "company", "reassignment_count", "activity_due", "assigned_to", "severity", "comments", "approval", "sla_due", "comments_and_work_notes", "due_date", "sys_mod_count", "u_affected_user", "reopen_count", "sys_tags", "escalation", "upon_approval", "correlation_id", "location", "category"}, {"Column1.parent", "Column1.made_sla", "Column1.u_notification_major_incident", "Column1.caused_by", "Column1.watch_list", "Column1.upon_reject", "Column1.sys_updated_on", "Column1.child_incidents", "Column1.hold_reason", "Column1.approval_history", "Column1.number", "Column1.resolved_by", "Column1.sys_updated_by", "Column1.opened_by", "Column1.user_input", "Column1.sys_created_on", "Column1.sys_domain", "Column1.state", "Column1.sys_created_by", "Column1.knowledge", "Column1.order", "Column1.calendar_stc", "Column1.closed_at", "Column1.cmdb_ci", "Column1.delivery_plan", "Column1.impact", "Column1.active", "Column1.work_notes_list", "Column1.business_service", "Column1.priority", "Column1.sys_domain_path", "Column1.rfc", "Column1.time_worked", "Column1.expected_start", "Column1.opened_at", "Column1.business_duration", "Column1.group_list", "Column1.work_end", "Column1.caller_id", "Column1.resolved_at", "Column1.approval_set", "Column1.subcategory", "Column1.work_notes", "Column1.short_description", "Column1.close_code", "Column1.correlation_display", "Column1.delivery_task", "Column1.work_start", "Column1.assignment_group", "Column1.additional_assignee_list", "Column1.business_stc", "Column1.description", "Column1.calendar_duration", "Column1.close_notes", "Column1.notify", "Column1.sys_class_name", "Column1.closed_by", "Column1.follow_up", "Column1.parent_incident", "Column1.sys_id", "Column1.contact_type", "Column1.incident_state", "Column1.urgency", "Column1.problem_id", "Column1.company", "Column1.reassignment_count", "Column1.activity_due", "Column1.assigned_to", "Column1.severity", "Column1.comments", "Column1.approval", "Column1.sla_due", "Column1.comments_and_work_notes", "Column1.due_date", "Column1.sys_mod_count", "Column1.u_affected_user", "Column1.reopen_count", "Column1.sys_tags", "Column1.escalation", "Column1.upon_approval", "Column1.correlation_id", "Column1.location", "Column1.category"}),
#"Removed Columns" = Table.RemoveColumns(#"Expanded Column1",{"Column1.made_sla", "Column1.parent", "Column1.watch_list", "Column1.upon_reject", "Column1.child_incidents", "Column1.hold_reason", "Column1.approval_history"}),
#"Renamed Columns" = Table.RenameColumns(#"Removed Columns",{{"Column1.u_notification_major_incident", "major incident"}}),
#"Removed Columns1" = Table.RemoveColumns(#"Renamed Columns",{"Column1.caused_by"}),
#"Renamed Columns1" = Table.RenameColumns(#"Removed Columns1",{{"Column1.sys_updated_on", "updated on"}, {"Column1.number", "number"}}),
#"Expanded Column1.resolved_by" = Table.ExpandRecordColumn(#"Renamed Columns1", "Column1.resolved_by", {"display_value"}, {"Column1.resolved_by.display_value"}),
#"Replaced Errors" = Table.ReplaceErrorValues(#"Expanded Column1.resolved_by", {{"Column1.resolved_by.display_value", ""}}),
#"Renamed Columns2" = Table.RenameColumns(#"Replaced Errors",{{"Column1.sys_updated_by", "updated by"}, {"Column1.resolved_by.display_value", "rewsolved by"}}),
#"Expanded Column1.opened_by" = Table.ExpandRecordColumn(#"Renamed Columns2", "Column1.opened_by", {"display_value"}, {"Column1.opened_by.display_value"}),
#"Replaced Errors1" = Table.ReplaceErrorValues(#"Expanded Column1.opened_by", {{"Column1.opened_by.display_value", ""}}),
#"Removed Columns2" = Table.RemoveColumns(#"Replaced Errors1",{"Column1.user_input"}),
#"Renamed Columns3" = Table.RenameColumns(#"Removed Columns2",{{"Column1.opened_by.display_value", "opened by"}, {"Column1.sys_created_on", "created on"}}),
#"Removed Columns3" = Table.RemoveColumns(#"Renamed Columns3",{"Column1.sys_domain"}),
#"Renamed Columns4" = Table.RenameColumns(#"Removed Columns3",{{"Column1.state", "state"}, {"Column1.sys_created_by", "created by"}}),
#"Removed Columns4" = Table.RemoveColumns(#"Renamed Columns4",{"Column1.knowledge", "Column1.order", "Column1.calendar_stc"}),
#"Renamed Columns5" = Table.RenameColumns(#"Removed Columns4",{{"Column1.closed_at", "closed at"}, {"Column1.cmdb_ci", "cmdb ci"}}),
#"Expanded cmdb ci" = Table.ExpandRecordColumn(#"Renamed Columns5", "cmdb ci", {"display_value"}, {"cmdb ci.display_value"}),
#"Replaced Errors2" = Table.ReplaceErrorValues(#"Expanded cmdb ci", {{"cmdb ci.display_value", ""}}),
#"Renamed Columns6" = Table.RenameColumns(#"Replaced Errors2",{{"cmdb ci.display_value", "cmdb ci"}}),
#"Removed Columns5" = Table.RemoveColumns(#"Renamed Columns6",{"Column1.delivery_plan"}),
#"Renamed Columns7" = Table.RenameColumns(#"Removed Columns5",{{"Column1.impact", "imapct"}, {"Column1.active", "active"}}),
#"Removed Columns6" = Table.RemoveColumns(#"Renamed Columns7",{"Column1.work_notes_list"}),
#"Expanded Column1.business_service" = Table.ExpandRecordColumn(#"Removed Columns6", "Column1.business_service", {"display_value"}, {"Column1.business_service.display_value"}),
#"Renamed Columns8" = Table.RenameColumns(#"Expanded Column1.business_service",{{"Column1.business_service.display_value", "business service"}, {"Column1.priority", "priority"}}),
#"Removed Columns7" = Table.RemoveColumns(#"Renamed Columns8",{"Column1.sys_domain_path", "Column1.rfc", "Column1.time_worked", "Column1.expected_start"}),
#"Renamed Columns9" = Table.RenameColumns(#"Removed Columns7",{{"Column1.opened_at", "opened at"}}),
#"Removed Columns8" = Table.RemoveColumns(#"Renamed Columns9",{"Column1.business_duration", "Column1.group_list", "Column1.work_end"}),
#"Expanded Column1.caller_id" = Table.ExpandRecordColumn(#"Removed Columns8", "Column1.caller_id", {"display_value"}, {"Column1.caller_id.display_value"}),
#"Replaced Errors3" = Table.ReplaceErrorValues(#"Expanded Column1.caller_id", {{"Column1.caller_id.display_value", ""}}),
#"Renamed Columns10" = Table.RenameColumns(#"Replaced Errors3",{{"Column1.caller_id.display_value", "caller"}, {"Column1.resolved_at", "resolved at"}}),
#"Removed Columns9" = Table.RemoveColumns(#"Renamed Columns10",{"Column1.approval_set", "Column1.subcategory"}),
#"Renamed Columns11" = Table.RenameColumns(#"Removed Columns9",{{"Column1.work_notes", "work notes"}, {"Column1.short_description", "short description"}, {"Column1.close_code", "close code"}}),
#"Removed Columns10" = Table.RemoveColumns(#"Renamed Columns11",{"Column1.correlation_display", "Column1.delivery_task", "Column1.work_start"}),
#"Expanded Column1.assignment_group" = Table.ExpandRecordColumn(#"Removed Columns10", "Column1.assignment_group", {"display_value"}, {"Column1.assignment_group.display_value"}),
#"Renamed Columns12" = Table.RenameColumns(#"Expanded Column1.assignment_group",{{"Column1.assignment_group.display_value", "assignment group"}}),
#"Removed Columns11" = Table.RemoveColumns(#"Renamed Columns12",{"Column1.additional_assignee_list", "Column1.business_stc"}),
#"Renamed Columns13" = Table.RenameColumns(#"Removed Columns11",{{"Column1.description", "description"}}),
#"Removed Columns12" = Table.RemoveColumns(#"Renamed Columns13",{"Column1.calendar_duration"}),
#"Renamed Columns14" = Table.RenameColumns(#"Removed Columns12",{{"Column1.close_notes", "close notes"}}),
#"Removed Columns13" = Table.RemoveColumns(#"Renamed Columns14",{"Column1.notify", "Column1.sys_class_name"}),
#"Expanded Column1.closed_by" = Table.ExpandRecordColumn(#"Removed Columns13", "Column1.closed_by", {"display_value"}, {"Column1.closed_by.display_value"}),
#"Replaced Errors4" = Table.ReplaceErrorValues(#"Expanded Column1.closed_by", {{"Column1.closed_by.display_value", ""}}),
#"Renamed Columns15" = Table.RenameColumns(#"Replaced Errors4",{{"Column1.closed_by.display_value", "closed by"}}),
#"Removed Columns14" = Table.RemoveColumns(#"Renamed Columns15",{"Column1.follow_up", "Column1.sys_id", "Column1.parent_incident"}),
#"Renamed Columns16" = Table.RenameColumns(#"Removed Columns14",{{"Column1.contact_type", "contact type"}}),
#"Removed Columns15" = Table.RemoveColumns(#"Renamed Columns16",{"Column1.incident_state"}),
#"Renamed Columns17" = Table.RenameColumns(#"Removed Columns15",{{"Column1.urgency", "urgency"}}),
#"Removed Columns16" = Table.RemoveColumns(#"Renamed Columns17",{"Column1.problem_id"}),
#"Expanded Column1.company" = Table.ExpandRecordColumn(#"Removed Columns16", "Column1.company", {"display_value"}, {"Column1.company.display_value"}),
#"Renamed Columns18" = Table.RenameColumns(#"Expanded Column1.company",{{"Column1.reassignment_count", "reassignment count"}}),
#"Removed Columns17" = Table.RemoveColumns(#"Renamed Columns18",{"Column1.activity_due"}),
#"Expanded Column1.assigned_to" = Table.ExpandRecordColumn(#"Removed Columns17", "Column1.assigned_to", {"display_value"}, {"Column1.assigned_to.display_value"}),
#"Replaced Errors5" = Table.ReplaceErrorValues(#"Expanded Column1.assigned_to", {{"Column1.assigned_to.display_value", ""}}),
#"Renamed Columns19" = Table.RenameColumns(#"Replaced Errors5",{{"Column1.company.display_value", "company"}}),
#"Replaced Errors6" = Table.ReplaceErrorValues(#"Renamed Columns19", {{"company", ""}}),
#"Renamed Columns20" = Table.RenameColumns(#"Replaced Errors6",{{"Column1.assigned_to.display_value", "assigned to"}, {"Column1.severity", "severity"}, {"Column1.comments", "comments"}}),
#"Removed Columns18" = Table.RemoveColumns(#"Renamed Columns20",{"Column1.approval", "Column1.sla_due", "Column1.comments_and_work_notes", "Column1.due_date", "Column1.sys_mod_count"}),
#"Expanded Column1.u_affected_user" = Table.ExpandRecordColumn(#"Removed Columns18", "Column1.u_affected_user", {"display_value"}, {"Column1.u_affected_user.display_value"}),
#"Replaced Errors7" = Table.ReplaceErrorValues(#"Expanded Column1.u_affected_user", {{"Column1.u_affected_user.display_value", ""}}),
#"Renamed Columns21" = Table.RenameColumns(#"Replaced Errors7",{{"Column1.u_affected_user.display_value", "affected user"}, {"Column1.reopen_count", "reopen count"}}),
#"Removed Columns19" = Table.RemoveColumns(#"Renamed Columns21",{"Column1.sys_tags", "Column1.escalation", "Column1.upon_approval", "Column1.correlation_id", "Column1.category"}),
#"Replaced Errors8" = Table.ReplaceErrorValues(#"Removed Columns19", {{"assignment group", ""}}),
#"Replaced Errors9" = Table.ReplaceErrorValues(#"Replaced Errors8", {{"business service", ""}}),
#"Changed Type" = Table.TransformColumnTypes(#"Replaced Errors9",{{"created on", type datetime}, {"updated on", type datetime}, {"reassignment count", Int64.Type}, {"reopen count", Int64.Type}}),
#"Renamed Columns22" = Table.RenameColumns(#"Changed Type",{{"close code", "close type"}})
in
#"Renamed Columns22"
3 Replies
- v-yulgu-msftMicrosoft Employee
Hi tordnordahl,
Please make sure that current account have full read access to incident table. And contact with your administrator to check if some permissions of your account is restricted in ServiceNow.
Regards,
Yuliana Gu
- tordnordahlNew Member
The current user have admin rights to the servicenow instance. So it cannot be access related. I also used postman to verify the url and the data, and with postman I get all the data.
- antonstorozhukHelper V
Hello tordnordahl
I would like to propose you to try out an alternative option connecting Power BI and Servicenow:
Power BI Connector for ServiceNow app
- https://store.servicenow.com/sn_appstore_store.do#!/store/application/87a42573879e0110fb5033773cbb35...
- https://www.alphaservesp.com/products/servicenow/power-bi/
We have a handy User and Admin Guides for your convenience, but feel free to reach out support team at [email protected] if you need any assistance.
Cheers,
Anton