Forum Discussion
PQ Matching data that doesn’t exactly match!
- Anonymous7 years ago
In that case, go through the following example based on your sample data set.
Modify it as necessary. But open a "BlankQuery" in power bi desktop, paste the following codes and run it to get some idea.
let //your sample data set of "Seats" table // Replace the following codes with your actual power query code that fetches the Seats table. Seats = #table( {"Train","Origin","Date","Time","Seats"}, { {"A", "X", #date(2018,4,1),#time(10,5,0), 186}, {"B", "Y", #date(2018,4,1),#time(11,40,0), 147}, {"C", "Z", #date(2018,4,1),#time(13,48,0), 167}, {"A", "X", #date(2018,4,1),#time(22,10,0), 191}, {"B", "Y", #date(2018,4,1),#time(23,55,0), 146}, {"C", "Z", #date(2018,4,1),#time(1,30,0), 170} } ), //your sample data set of "Passengers" table. // Replace the following codes with your actual power query code that fetches the Passengers table. Passengers = #table( {"Train","Origin","Date","Time","Passengers"}, { {"A", "X", #date(2018,4,1),#time(10,9,0), 149}, {"B", "Y", #date(2018,4,1),#time(11,51,0), 118}, {"C", "Z", #date(2018,4,1),#time(13,59,0), 134}, {"A", "X", #date(2018,4,1),#time(22,2,0), 153}, {"B", "Y", #date(2018,4,2),#time(0,2,0), 117}, {"C", "Z", #date(2018,4,1),#time(1,29,0), 136} } ), // Add the date field and time field together to form a datetime column for operations AddDateTimeColumnS = Table.AddColumn(Seats,"DateTimeColumn",each [Date] & [Time],type datetime), // Reduce 2 hours from the DateTimeColumn added in the previous step as a new column "TimeWindowFrom" AddTimeWindowFrom = Table.AddColumn(AddDateTimeColumnS,"TimeWindowFrom",each [DateTimeColumn] - #duration(0,2,0,0),type datetime), // Add 2 hours from the DateTimeColumn added in the previous step as a new column "TimeWindowTo" AddTimeWindowTo = Table.AddColumn(AddTimeWindowFrom,"TimeWindowTo",each [DateTimeColumn] + #duration(0,2,0,0),type datetime), // Add the date field and time field together to form a datetime column for operations - To Passengers Table. AddDateTimeColumnP = Table.AddColumn(Passengers,"DateTimeColumn",each [Date] & [Time],type datetime), // Custom Function // This function takes and Train Number, Origin, TimeWindowFrom, TimeWindowTo from Seats table and the entire Passengers table as input. // It matches the details and returns the number of passengers as the list. // IF there are two or more trains matching all the criteria (a rare scenario), it returns the first match from the list MatchTrain = (p as table,t as text,o as text,dtf as datetime,dtt as datetime) as number => List.First( Table.Column( Table.SelectRows(p, each [Train]=t and [Origin]=o and dtf <= [DateTimeColumn] and dtt >=[DateTimeColumn]), "Passengers") ), // Adds a column "Passengers" to the "Seats Table" after matching the details using the custom function "MatchTrain" MatchedTable = Table.AddColumn(AddTimeWindowTo, "Passengers", each MatchTrain(AddDateTimeColumnP,[Train],[Origin],[TimeWindowFrom],[TimeWindowTo]) ), // Removes the unnecessary columns. RemoveUnnecessaryColumns = Table.RemoveColumns(MatchedTable,{"DateTimeColumn","TimeWindowFrom","TimeWindowTo"}), Output=RemoveUnnecessaryColumns in OutputIf you want to see the step by step operations, change the "Output=RemoveUnnecessaryColumns" line to each of the following and execute it. You will be able to see step by step output.
Output = Seats
Output = Passengers
Output = AddDateTimeColumnS
Output = AddTimeWindowFrom
Output = AddTimeWindowTo
Output = AddDateTimeColumnP
Output = MatchedTable
Output = RemoveUnnecessaryColumns
In that case, go through the following example based on your sample data set.
Modify it as necessary. But open a "BlankQuery" in power bi desktop, paste the following codes and run it to get some idea.
let
//your sample data set of "Seats" table
// Replace the following codes with your actual power query code that fetches the Seats table.
Seats = #table(
{"Train","Origin","Date","Time","Seats"},
{
{"A", "X", #date(2018,4,1),#time(10,5,0), 186},
{"B", "Y", #date(2018,4,1),#time(11,40,0), 147},
{"C", "Z", #date(2018,4,1),#time(13,48,0), 167},
{"A", "X", #date(2018,4,1),#time(22,10,0), 191},
{"B", "Y", #date(2018,4,1),#time(23,55,0), 146},
{"C", "Z", #date(2018,4,1),#time(1,30,0), 170}
}
),
//your sample data set of "Passengers" table.
// Replace the following codes with your actual power query code that fetches the Passengers table.
Passengers = #table(
{"Train","Origin","Date","Time","Passengers"},
{
{"A", "X", #date(2018,4,1),#time(10,9,0), 149},
{"B", "Y", #date(2018,4,1),#time(11,51,0), 118},
{"C", "Z", #date(2018,4,1),#time(13,59,0), 134},
{"A", "X", #date(2018,4,1),#time(22,2,0), 153},
{"B", "Y", #date(2018,4,2),#time(0,2,0), 117},
{"C", "Z", #date(2018,4,1),#time(1,29,0), 136}
}
),
// Add the date field and time field together to form a datetime column for operations
AddDateTimeColumnS = Table.AddColumn(Seats,"DateTimeColumn",each [Date] & [Time],type datetime),
// Reduce 2 hours from the DateTimeColumn added in the previous step as a new column "TimeWindowFrom"
AddTimeWindowFrom = Table.AddColumn(AddDateTimeColumnS,"TimeWindowFrom",each [DateTimeColumn] - #duration(0,2,0,0),type datetime),
// Add 2 hours from the DateTimeColumn added in the previous step as a new column "TimeWindowTo"
AddTimeWindowTo = Table.AddColumn(AddTimeWindowFrom,"TimeWindowTo",each [DateTimeColumn] + #duration(0,2,0,0),type datetime),
// Add the date field and time field together to form a datetime column for operations - To Passengers Table.
AddDateTimeColumnP = Table.AddColumn(Passengers,"DateTimeColumn",each [Date] & [Time],type datetime),
// Custom Function
// This function takes and Train Number, Origin, TimeWindowFrom, TimeWindowTo from Seats table and the entire Passengers table as input.
// It matches the details and returns the number of passengers as the list.
// IF there are two or more trains matching all the criteria (a rare scenario), it returns the first match from the list
MatchTrain = (p as table,t as text,o as text,dtf as datetime,dtt as datetime) as number =>
List.First(
Table.Column(
Table.SelectRows(p, each [Train]=t and [Origin]=o and dtf <= [DateTimeColumn] and dtt >=[DateTimeColumn]),
"Passengers")
),
// Adds a column "Passengers" to the "Seats Table" after matching the details using the custom function "MatchTrain"
MatchedTable = Table.AddColumn(AddTimeWindowTo,
"Passengers",
each MatchTrain(AddDateTimeColumnP,[Train],[Origin],[TimeWindowFrom],[TimeWindowTo])
),
// Removes the unnecessary columns.
RemoveUnnecessaryColumns = Table.RemoveColumns(MatchedTable,{"DateTimeColumn","TimeWindowFrom","TimeWindowTo"}),
Output=RemoveUnnecessaryColumns
in
OutputIf you want to see the step by step operations, change the "Output=RemoveUnnecessaryColumns" line to each of the following and execute it. You will be able to see step by step output.
Output = Seats
Output = Passengers
Output = AddDateTimeColumnS
Output = AddTimeWindowFrom
Output = AddTimeWindowTo
Output = AddDateTimeColumnP
Output = MatchedTable
Output = RemoveUnnecessaryColumns
Anonymous don't follow what you have done but it looks like you have cracked it! You are far too clever for me, this looks bl00dy impressive. I will play around with it and see if it works in my 'real life' example but it is looking very promising. Thanks heaps for your help.
BTW, how did you get to know all this stuff? Is it your job, courses, own time??? Thanks,
dan