Forum Discussion

tordnordahl's avatar
tordnordahl
New Member
8 years ago

Not all data is coming in using Web with JSON

I have a set of 5 web sources where I get data from (ServiceNow APIs) all in JSON format.

 

However one of the is very inconsistence, according to ServiceNow and using other tools I get all rows, but with Power BI the number can be anything from 1000 OFF to 200-300 off the actual number of rows, here is my query, am I doing something wrong:

let
    Source = Json.Document(Web.Contents("https://<myInstance>.service-now.com/api/now/table/incident?sysparm_display_value=true",[Timeout=#duration(0,4,0,0)])),
    result = Source[result],
    #"Converted to Table" = Table.FromList(result, Splitter.SplitByNothing(), null, null, ExtraValues.Ignore),
    #"Expanded Column1" = Table.ExpandRecordColumn(#"Converted to Table", "Column1", {"parent", "made_sla", "u_notification_major_incident", "caused_by", "watch_list", "upon_reject", "sys_updated_on", "child_incidents", "hold_reason", "approval_history", "number", "resolved_by", "sys_updated_by", "opened_by", "user_input", "sys_created_on", "sys_domain", "state", "sys_created_by", "knowledge", "order", "calendar_stc", "closed_at", "cmdb_ci", "delivery_plan", "impact", "active", "work_notes_list", "business_service", "priority", "sys_domain_path", "rfc", "time_worked", "expected_start", "opened_at", "business_duration", "group_list", "work_end", "caller_id", "resolved_at", "approval_set", "subcategory", "work_notes", "short_description", "close_code", "correlation_display", "delivery_task", "work_start", "assignment_group", "additional_assignee_list", "business_stc", "description", "calendar_duration", "close_notes", "notify", "sys_class_name", "closed_by", "follow_up", "parent_incident", "sys_id", "contact_type", "incident_state", "urgency", "problem_id", "company", "reassignment_count", "activity_due", "assigned_to", "severity", "comments", "approval", "sla_due", "comments_and_work_notes", "due_date", "sys_mod_count", "u_affected_user", "reopen_count", "sys_tags", "escalation", "upon_approval", "correlation_id", "location", "category"}, {"Column1.parent", "Column1.made_sla", "Column1.u_notification_major_incident", "Column1.caused_by", "Column1.watch_list", "Column1.upon_reject", "Column1.sys_updated_on", "Column1.child_incidents", "Column1.hold_reason", "Column1.approval_history", "Column1.number", "Column1.resolved_by", "Column1.sys_updated_by", "Column1.opened_by", "Column1.user_input", "Column1.sys_created_on", "Column1.sys_domain", "Column1.state", "Column1.sys_created_by", "Column1.knowledge", "Column1.order", "Column1.calendar_stc", "Column1.closed_at", "Column1.cmdb_ci", "Column1.delivery_plan", "Column1.impact", "Column1.active", "Column1.work_notes_list", "Column1.business_service", "Column1.priority", "Column1.sys_domain_path", "Column1.rfc", "Column1.time_worked", "Column1.expected_start", "Column1.opened_at", "Column1.business_duration", "Column1.group_list", "Column1.work_end", "Column1.caller_id", "Column1.resolved_at", "Column1.approval_set", "Column1.subcategory", "Column1.work_notes", "Column1.short_description", "Column1.close_code", "Column1.correlation_display", "Column1.delivery_task", "Column1.work_start", "Column1.assignment_group", "Column1.additional_assignee_list", "Column1.business_stc", "Column1.description", "Column1.calendar_duration", "Column1.close_notes", "Column1.notify", "Column1.sys_class_name", "Column1.closed_by", "Column1.follow_up", "Column1.parent_incident", "Column1.sys_id", "Column1.contact_type", "Column1.incident_state", "Column1.urgency", "Column1.problem_id", "Column1.company", "Column1.reassignment_count", "Column1.activity_due", "Column1.assigned_to", "Column1.severity", "Column1.comments", "Column1.approval", "Column1.sla_due", "Column1.comments_and_work_notes", "Column1.due_date", "Column1.sys_mod_count", "Column1.u_affected_user", "Column1.reopen_count", "Column1.sys_tags", "Column1.escalation", "Column1.upon_approval", "Column1.correlation_id", "Column1.location", "Column1.category"}),
    #"Removed Columns" = Table.RemoveColumns(#"Expanded Column1",{"Column1.made_sla", "Column1.parent", "Column1.watch_list", "Column1.upon_reject", "Column1.child_incidents", "Column1.hold_reason", "Column1.approval_history"}),
    #"Renamed Columns" = Table.RenameColumns(#"Removed Columns",{{"Column1.u_notification_major_incident", "major incident"}}),
    #"Removed Columns1" = Table.RemoveColumns(#"Renamed Columns",{"Column1.caused_by"}),
    #"Renamed Columns1" = Table.RenameColumns(#"Removed Columns1",{{"Column1.sys_updated_on", "updated on"}, {"Column1.number", "number"}}),
    #"Expanded Column1.resolved_by" = Table.ExpandRecordColumn(#"Renamed Columns1", "Column1.resolved_by", {"display_value"}, {"Column1.resolved_by.display_value"}),
    #"Replaced Errors" = Table.ReplaceErrorValues(#"Expanded Column1.resolved_by", {{"Column1.resolved_by.display_value", ""}}),
    #"Renamed Columns2" = Table.RenameColumns(#"Replaced Errors",{{"Column1.sys_updated_by", "updated by"}, {"Column1.resolved_by.display_value", "rewsolved by"}}),
    #"Expanded Column1.opened_by" = Table.ExpandRecordColumn(#"Renamed Columns2", "Column1.opened_by", {"display_value"}, {"Column1.opened_by.display_value"}),
    #"Replaced Errors1" = Table.ReplaceErrorValues(#"Expanded Column1.opened_by", {{"Column1.opened_by.display_value", ""}}),
    #"Removed Columns2" = Table.RemoveColumns(#"Replaced Errors1",{"Column1.user_input"}),
    #"Renamed Columns3" = Table.RenameColumns(#"Removed Columns2",{{"Column1.opened_by.display_value", "opened by"}, {"Column1.sys_created_on", "created on"}}),
    #"Removed Columns3" = Table.RemoveColumns(#"Renamed Columns3",{"Column1.sys_domain"}),
    #"Renamed Columns4" = Table.RenameColumns(#"Removed Columns3",{{"Column1.state", "state"}, {"Column1.sys_created_by", "created by"}}),
    #"Removed Columns4" = Table.RemoveColumns(#"Renamed Columns4",{"Column1.knowledge", "Column1.order", "Column1.calendar_stc"}),
    #"Renamed Columns5" = Table.RenameColumns(#"Removed Columns4",{{"Column1.closed_at", "closed at"}, {"Column1.cmdb_ci", "cmdb ci"}}),
    #"Expanded cmdb ci" = Table.ExpandRecordColumn(#"Renamed Columns5", "cmdb ci", {"display_value"}, {"cmdb ci.display_value"}),
    #"Replaced Errors2" = Table.ReplaceErrorValues(#"Expanded cmdb ci", {{"cmdb ci.display_value", ""}}),
    #"Renamed Columns6" = Table.RenameColumns(#"Replaced Errors2",{{"cmdb ci.display_value", "cmdb ci"}}),
    #"Removed Columns5" = Table.RemoveColumns(#"Renamed Columns6",{"Column1.delivery_plan"}),
    #"Renamed Columns7" = Table.RenameColumns(#"Removed Columns5",{{"Column1.impact", "imapct"}, {"Column1.active", "active"}}),
    #"Removed Columns6" = Table.RemoveColumns(#"Renamed Columns7",{"Column1.work_notes_list"}),
    #"Expanded Column1.business_service" = Table.ExpandRecordColumn(#"Removed Columns6", "Column1.business_service", {"display_value"}, {"Column1.business_service.display_value"}),
    #"Renamed Columns8" = Table.RenameColumns(#"Expanded Column1.business_service",{{"Column1.business_service.display_value", "business service"}, {"Column1.priority", "priority"}}),
    #"Removed Columns7" = Table.RemoveColumns(#"Renamed Columns8",{"Column1.sys_domain_path", "Column1.rfc", "Column1.time_worked", "Column1.expected_start"}),
    #"Renamed Columns9" = Table.RenameColumns(#"Removed Columns7",{{"Column1.opened_at", "opened at"}}),
    #"Removed Columns8" = Table.RemoveColumns(#"Renamed Columns9",{"Column1.business_duration", "Column1.group_list", "Column1.work_end"}),
    #"Expanded Column1.caller_id" = Table.ExpandRecordColumn(#"Removed Columns8", "Column1.caller_id", {"display_value"}, {"Column1.caller_id.display_value"}),
    #"Replaced Errors3" = Table.ReplaceErrorValues(#"Expanded Column1.caller_id", {{"Column1.caller_id.display_value", ""}}),
    #"Renamed Columns10" = Table.RenameColumns(#"Replaced Errors3",{{"Column1.caller_id.display_value", "caller"}, {"Column1.resolved_at", "resolved at"}}),
    #"Removed Columns9" = Table.RemoveColumns(#"Renamed Columns10",{"Column1.approval_set", "Column1.subcategory"}),
    #"Renamed Columns11" = Table.RenameColumns(#"Removed Columns9",{{"Column1.work_notes", "work notes"}, {"Column1.short_description", "short description"}, {"Column1.close_code", "close code"}}),
    #"Removed Columns10" = Table.RemoveColumns(#"Renamed Columns11",{"Column1.correlation_display", "Column1.delivery_task", "Column1.work_start"}),
    #"Expanded Column1.assignment_group" = Table.ExpandRecordColumn(#"Removed Columns10", "Column1.assignment_group", {"display_value"}, {"Column1.assignment_group.display_value"}),
    #"Renamed Columns12" = Table.RenameColumns(#"Expanded Column1.assignment_group",{{"Column1.assignment_group.display_value", "assignment group"}}),
    #"Removed Columns11" = Table.RemoveColumns(#"Renamed Columns12",{"Column1.additional_assignee_list", "Column1.business_stc"}),
    #"Renamed Columns13" = Table.RenameColumns(#"Removed Columns11",{{"Column1.description", "description"}}),
    #"Removed Columns12" = Table.RemoveColumns(#"Renamed Columns13",{"Column1.calendar_duration"}),
    #"Renamed Columns14" = Table.RenameColumns(#"Removed Columns12",{{"Column1.close_notes", "close notes"}}),
    #"Removed Columns13" = Table.RemoveColumns(#"Renamed Columns14",{"Column1.notify", "Column1.sys_class_name"}),
    #"Expanded Column1.closed_by" = Table.ExpandRecordColumn(#"Removed Columns13", "Column1.closed_by", {"display_value"}, {"Column1.closed_by.display_value"}),
    #"Replaced Errors4" = Table.ReplaceErrorValues(#"Expanded Column1.closed_by", {{"Column1.closed_by.display_value", ""}}),
    #"Renamed Columns15" = Table.RenameColumns(#"Replaced Errors4",{{"Column1.closed_by.display_value", "closed by"}}),
    #"Removed Columns14" = Table.RemoveColumns(#"Renamed Columns15",{"Column1.follow_up", "Column1.sys_id", "Column1.parent_incident"}),
    #"Renamed Columns16" = Table.RenameColumns(#"Removed Columns14",{{"Column1.contact_type", "contact type"}}),
    #"Removed Columns15" = Table.RemoveColumns(#"Renamed Columns16",{"Column1.incident_state"}),
    #"Renamed Columns17" = Table.RenameColumns(#"Removed Columns15",{{"Column1.urgency", "urgency"}}),
    #"Removed Columns16" = Table.RemoveColumns(#"Renamed Columns17",{"Column1.problem_id"}),
    #"Expanded Column1.company" = Table.ExpandRecordColumn(#"Removed Columns16", "Column1.company", {"display_value"}, {"Column1.company.display_value"}),
    #"Renamed Columns18" = Table.RenameColumns(#"Expanded Column1.company",{{"Column1.reassignment_count", "reassignment count"}}),
    #"Removed Columns17" = Table.RemoveColumns(#"Renamed Columns18",{"Column1.activity_due"}),
    #"Expanded Column1.assigned_to" = Table.ExpandRecordColumn(#"Removed Columns17", "Column1.assigned_to", {"display_value"}, {"Column1.assigned_to.display_value"}),
    #"Replaced Errors5" = Table.ReplaceErrorValues(#"Expanded Column1.assigned_to", {{"Column1.assigned_to.display_value", ""}}),
    #"Renamed Columns19" = Table.RenameColumns(#"Replaced Errors5",{{"Column1.company.display_value", "company"}}),
    #"Replaced Errors6" = Table.ReplaceErrorValues(#"Renamed Columns19", {{"company", ""}}),
    #"Renamed Columns20" = Table.RenameColumns(#"Replaced Errors6",{{"Column1.assigned_to.display_value", "assigned to"}, {"Column1.severity", "severity"}, {"Column1.comments", "comments"}}),
    #"Removed Columns18" = Table.RemoveColumns(#"Renamed Columns20",{"Column1.approval", "Column1.sla_due", "Column1.comments_and_work_notes", "Column1.due_date", "Column1.sys_mod_count"}),
    #"Expanded Column1.u_affected_user" = Table.ExpandRecordColumn(#"Removed Columns18", "Column1.u_affected_user", {"display_value"}, {"Column1.u_affected_user.display_value"}),
    #"Replaced Errors7" = Table.ReplaceErrorValues(#"Expanded Column1.u_affected_user", {{"Column1.u_affected_user.display_value", ""}}),
    #"Renamed Columns21" = Table.RenameColumns(#"Replaced Errors7",{{"Column1.u_affected_user.display_value", "affected user"}, {"Column1.reopen_count", "reopen count"}}),
    #"Removed Columns19" = Table.RemoveColumns(#"Renamed Columns21",{"Column1.sys_tags", "Column1.escalation", "Column1.upon_approval", "Column1.correlation_id", "Column1.category"}),
    #"Replaced Errors8" = Table.ReplaceErrorValues(#"Removed Columns19", {{"assignment group", ""}}),
    #"Replaced Errors9" = Table.ReplaceErrorValues(#"Replaced Errors8", {{"business service", ""}}),
    #"Changed Type" = Table.TransformColumnTypes(#"Replaced Errors9",{{"created on", type datetime}, {"updated on", type datetime}, {"reassignment count", Int64.Type}, {"reopen count", Int64.Type}}),
    #"Renamed Columns22" = Table.RenameColumns(#"Changed Type",{{"close code", "close type"}})
in
    #"Renamed Columns22"

3 Replies