Forum Discussion
Subtract Values in consecutive rows
- 5 years ago
themistoklis - one more solution:
let Source = Excel.CurrentWorkbook(){[Name="Table1"]}[Content], #"Changed Type" = Table.TransformColumnTypes(Source,{{"DATETIME", type datetime}, {"CATEGORY", type text}, {"CUMULATIVE VIEWS", Int64.Type}}), #"Sorted Rows" = Table.Buffer(Table.Sort(#"Changed Type",{{"CATEGORY", Order.Ascending}, {"DATETIME", Order.Ascending}})), listCategory = List.Buffer(#"Sorted Rows"[CATEGORY]), listCumulative = List.Buffer(#"Sorted Rows"[CUMULATIVE VIEWS]), listCount = List.Count(listCategory), listCumulativeDifference = List.Skip( List.Generate( ()=> [varCategory = listCategory{0}, varDiff = 0, Counter = 0], each [Counter] <= listCount, each try if [Counter] = 0 then [varDiff = listCumulative{[Counter]}, Counter = [Counter]+ 1] else if listCategory{[Counter]} = listCategory{[Counter] - 1} then [varDiff = listCumulative{[Counter]} - listCumulative{[Counter] -1}, Counter = [Counter]+ 1] else [varDiff = listCumulative{[Counter]}, Counter = [Counter]+ 1] otherwise [Counter = [Counter] + 1], each [varDiff] ), 1 ), CombinedColumns = Table.ToColumns(#"Sorted Rows") & {listCumulativeDifference}, BackToTable = Table.FromColumns( CombinedColumns, Table.ColumnNames(#"Sorted Rows") & {"Difference"} ) in BackToTableThis will do 100,000 rows as fast as Excel can think about loading the .NET framework to process it. I suspect 1M rows would be equally fast.
this is in the original file link above so you can get the solution there. it has 100K rows for testing now.
themistoklis - one more solution:
let
Source = Excel.CurrentWorkbook(){[Name="Table1"]}[Content],
#"Changed Type" = Table.TransformColumnTypes(Source,{{"DATETIME", type datetime}, {"CATEGORY", type text}, {"CUMULATIVE VIEWS", Int64.Type}}),
#"Sorted Rows" = Table.Buffer(Table.Sort(#"Changed Type",{{"CATEGORY", Order.Ascending}, {"DATETIME", Order.Ascending}})),
listCategory = List.Buffer(#"Sorted Rows"[CATEGORY]),
listCumulative = List.Buffer(#"Sorted Rows"[CUMULATIVE VIEWS]),
listCount = List.Count(listCategory),
listCumulativeDifference =
List.Skip(
List.Generate(
()=> [varCategory = listCategory{0}, varDiff = 0, Counter = 0],
each [Counter] <= listCount,
each
try
if [Counter] = 0
then [varDiff = listCumulative{[Counter]}, Counter = [Counter]+ 1]
else if listCategory{[Counter]} = listCategory{[Counter] - 1}
then [varDiff = listCumulative{[Counter]} - listCumulative{[Counter] -1}, Counter = [Counter]+ 1]
else [varDiff = listCumulative{[Counter]}, Counter = [Counter]+ 1]
otherwise [Counter = [Counter] + 1],
each [varDiff]
),
1
),
CombinedColumns =
Table.ToColumns(#"Sorted Rows") & {listCumulativeDifference},
BackToTable =
Table.FromColumns(
CombinedColumns,
Table.ColumnNames(#"Sorted Rows") & {"Difference"}
)
in
BackToTable
This will do 100,000 rows as fast as Excel can think about loading the .NET framework to process it. I suspect 1M rows would be equally fast.
this is in the original file link above so you can get the solution there. it has 100K rows for testing now.
- themistoklis5 years agoCommunity Champion
edhans's query run faster and i noticed that the table.buffer function was the one that made a huge impact on the performance
- Anonymous5 years agoNot applicable
let me have some doubts about this statement. In any case, for those wishing to try the different times of the two different solutions, you can use this script that I got from Ben Gribaudo's blog.
Obviously the results can vary a lot depending on many things. But you can test the performance of these scripts for different dimensions of the dataset
let //Get the current time, which will be the query start time StartTime = DateTime.LocalNow(), //Make sure StartTime is evaluated just before we request data from Facebook trig = if StartTime<>null then #"toAndFromCols LDS" else null, //Find the number of rows returned NumberOfRows = Number.ToText(Table.RowCount(trig)), //Get the current time, which will be the query end time EndTime = DateTime.LocalNow(), //Make sure PQ evvaluates all expressions in the right order: //first, get the Number of rows, which ensure that //the Source expression is evaluated, and in turn StartTime is evaluated //second, the EndTime is evaluated as part of the duration calculation Output = "Query analized " & NumberOfRows & " rows and took "& Duration.ToText(EndTime - StartTime) in Outputlet //Get the current time, which will be the query start time StartTime = DateTime.LocalNow(), //Make sure StartTime is evaluated just before we request data from Facebook trig = if StartTime<>null then #"listgenerate LDS" else null, //Find the number of rows returned NumberOfRows = Number.ToText(Table.RowCount(trig)), //Get the current time, which will be the query end time EndTime = DateTime.LocalNow(), //Make sure PQ evvaluates all expressions in the right order: //first, get the Number of rows, which ensure that //the Source expression is evaluated, and in turn StartTime is evaluated //second, the EndTime is evaluated as part of the duration calculation Output = "Query analized " & NumberOfRows & " rows and took " & Duration.ToText(EndTime - StartTime) in OutputlistGenerate LDS:
let Source = LDS,// Excel.CurrentWorkbook(){[Name="Table1"]}[Content], #"Changed Type" = Table.TransformColumnTypes(Source,{{"DATETIME", type datetime}, {"CATEGORY", type text}, {"CUMULATIVE VIEWS", Int64.Type}}), #"Sorted Rows" = Table.Buffer(Table.Sort(#"Changed Type",{{"CATEGORY", Order.Ascending}, {"DATETIME", Order.Ascending}})), listCategory = List.Buffer(#"Sorted Rows"[CATEGORY]), listCumulative = List.Buffer(#"Sorted Rows"[CUMULATIVE VIEWS]), listCount = List.Count(listCategory), listCumulativeDifference = List.Skip( List.Generate( ()=> [varCategory = listCategory{0}, varDiff = 0, Counter = 0], each [Counter] <= listCount, each try if [Counter] = 0 then [varDiff = listCumulative{[Counter]}, Counter = [Counter]+ 1] else if listCategory{[Counter]} = listCategory{[Counter] - 1} then [varDiff = listCumulative{[Counter]} - listCumulative{[Counter] -1}, Counter = [Counter]+ 1] else [varDiff = listCumulative{[Counter]}, Counter = [Counter]+ 1] otherwise [Counter = [Counter] + 1], each [varDiff] ), 1 ), CombinedColumns = Table.ToColumns(#"Sorted Rows") & {listCumulativeDifference}, BackToTable = Table.FromColumns( CombinedColumns, Table.ColumnNames(#"Sorted Rows") & {"Difference"} ) in BackToTabletoAndFromCols LDS:
let Origine = LDS, decumul =(cumulist)=> let ttc=Table.ToColumns(cumulist), names=Table.ColumnNames(cumulist), tfc = Table.FromColumns(ttc&{{0}&cumulist[CUMULATIVE VIEWS]},names&{"shifted"}), ac = Table.AddColumn(tfc, "dec", each [CUMULATIVE VIEWS]-[shifted]), rl = Table.RemoveLastN(ac,1) in rl, #"Raggruppate righe" = Table.Group(Origine, {"CATEGORY"}, {"all", each decumul(_)}), #"Tabella all espansa" = Table.ExpandTableColumn(#"Raggruppate righe", "all", {"DATETIME", "CUMULATIVE VIEWS", "HOURLY VIEWS (POWER QUERY CALCULATION)", "dec"}, {"DATETIME", "CUMULATIVE VIEWS", "HOURLY VIEWS (POWER QUERY CALCULATION)", "dec"}) in #"Tabella all espansa"LDS:
let Origine = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("jZRBagMxDEWvEmYdiKWxR3Z2cY8RssgidFko3fT2HWtwodUXaGNIeBjP/9K73xdqF6ILJ2qnRFeSK/NyXt6eX6/3j8/v023/wdtxPM5/cXbwOg5Dr5heE6SzQ5f9KIYuSpOh235kQ2/47jz+WQ0tDi3wJRXTZd0Pm2DD9DYykf80JUzLoCkZ3GlTGAVOTplSUCjklCkwcHLKrDBwKg7dYChOmU1DMbRTZoNjRRWPFSWGT3HapAQz5OTcTmNWqsHJwRkOCzt9EsNPZadQWjd9kplcMFz9ICEOXqM4o5aQKpSGOSJVDFqjsisHVKF0hneD6epTWvZuMF1Kw41DqlC6RlXRp4ZaRBWDLgWNFjKF0hI1RZ/SCplC6Rw1hdISNcWgJUdN0afhQqYYdE1RUygtUVH06ayQJ5SuUU30XwmFNHHsMfxOpIljj+feP34A", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [DATETIME = _t, CATEGORY = _t, #"CUMULATIVE VIEWS" = _t, #"HOURLY VIEWS (POWER QUERY CALCULATION)" = _t]), mt= Table.TransformColumnTypes(Origine,{{"DATETIME", type datetime}, {"CATEGORY", type text}, {"CUMULATIVE VIEWS", Int64.Type}, {"HOURLY VIEWS (POWER QUERY CALCULATION)", Int64.Type}}), ttc=Table.ToColumns(mt), names=Table.ColumnNames(mt), largeDS= List.Transform(ttc, each List.Repeat(_,5000)), tfc=Table.FromColumns(largeDS,names) in tfc- themistoklis5 years agoCommunity Champion
Anonymous
Just to mention that all scripts ran successfully (a few amendments where necessary). So users can use/test any of the scripts posted on this thread, based on their needs.
I agree that perfornance on Power Query running times is dependent on various things.
In my case i tried various things to improve performance. I also consulted the information from this site, which is actually good.
https://www.thebiccountant.com/speedperformance-aspects/
I would expect a pretty straight forward methodology to process the data (it is just a sorting and subtraction of consecutive rows), and without trial and error noone will ever find the best approach.
Although the query ran successfully I think I will also try python scripting