Forum Discussion
MittenState
2 years agoRegular Visitor
M Code to remove blanks and nulls from rows in a single sweep
I have multiple data sets with 10+ million each rows from our ERP that are subject to name and format changes by our IT team. CSV dumps are provided daily starting at 3am running until 6am or so, th...
- 2 years ago
Here are all the transforms, including the changed types and column names. This replaces the entirety of your function (minus the removal of duplicates, but that's easy to add).
h/t to Mr. von Neumann
lbendlin
Super User
2 years agoGot it:
let
Source = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("pZXbTuMwEIZfJeJqV4JkPD5fllIOWqAIKrTAchHaqHS3Tao0ef/1OKmbVGwXCSlNx649h/+zpy8vR2zAANnR8dHgdOjeUrrX3XtarlJn6NiPkQGE6WlWV4tputy4mdOrC/dmzEoOgGR2PjxhkCAw7WyRyK0pY8XJJ4IAsLsJ2Jmj80nrA/xeu93rvWLw2rVp9W1WRRxCLffZvF6mZfRY1NP3rHQz43WWUyHFpspmNM6XizxzhmUKGAdF1d6klVtOPw/HN0B+h5fDH5Qds5Z7Qdhe8PC0IUma4s29fz49k3fJDZXMjdaGxaiOXo//ozxDLmLFPiO+lmAs3xOfYUIJMtnasB2glFvHbgIY9Ob6oz4HliA5UU2lzqnpBNjZX+KglQUjrdCHOAiJVii1qzQE/4DDeeldNCAk83UxhcJJFmvWgHBaS3I3PBv5HHzwzR9/SmMkxxI8hHHl8o9ushkxiH7V7tCr6KEu59FVla12UBA4NxbNHhRsFNTeNOHkAotFQ9o4ef01Uk1Yiskbs2GxncSEYefwtbB13/SSEAsPeXR29SkWzToNSqBCqfEQCcUtShGq4wfuw6CuisDBWM9BoQIbw78pPKyLfE7ngtm4xUD5nC0262KTvi0zYpFcp2/RXVnM6mlFIUc348mogSCNkf5mdK+HEz40I5uI0JeMjYVHrpzpixLxVm9HaAfBywn93Y3fwLRrfw2DRiu1VeogBg0Wm7uwF/sDEAECcw3P+k3IUMVa7zDQ1vOLy4ChStdLnyXXGBtPwjUdyjKfFats+p7mdCP21LfW+EXbtuA/rhExys7fDdamSgMlZOMahVDMW2EKuoO9rgQJDw7bANgNoMPgSxwsWA3SHmxM2t16zjuVYiexfQ5Dpyi1mOengT/g7qz6QCBM70rssZjUb4t8TqrGBJsLpAXD1HWmyvWmtieFVT0ekovtZe3yUJ0sofk6abz7gx5LDKqfhLPvpJWdfbr5k7YthJ7U49tragCz3/WmWmV5tfHVFxsvYRD6PqvqMo8mRfSYuVNVRt8e6vV6ucjK7xQS0LVgJjSPTiIAzYUCcT95pDhFFQ1o5ZQ6wkdUaG1bbz/Pw0zoeX39Cw==", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [#"Dept ID" = _t, #"Supplier Name" = _t, #"Supplier ID" = _t, #"Line Description" = _t, #"Sum of monetary amount" = _t, Account = _t, #"Account Description" = _t, #"Business Unit" = _t, #"PO Number" = _t, #"Line Number" = _t, #"Schedule Number" = _t, #"PO Distribution Line Number" = _t, #"Invoice Date" = _t, #"Payment Date" = _t, #"Payment Amount" = _t, #"Payment ID" = _t, #"Merchandise Amt" = _t, #"Sum Freight" = _t, #"Unit Price" = _t, #"Payment Method" = _t, Quantity = _t, #"Discount Amount" = _t, #"Due Date" = _t, #"Discount Due Date" = _t, #"Voucher Entered Date" = _t, #"Matched Date" = _t, #"Payment Terms ID" = _t, #"Payment Terms Description" = _t, Origin = _t, #"Voucher Style" = _t, #"Close Status" = _t, #"Post Status" = _t, #"Voucher Source" = _t, #"Invoice Number" = _t, #"Match Status" = _t, #"Bank Code" = _t, #"Bank Account" = _t, #"Voucher ID" = _t, #"Voucher Line Number" = _t, #"Accounting Date" = _t, #"Match Line Status" = _t, #"Voucher Approval Status" = _t, #"Voucher Approval Date" = _t, #"Supplier Persistence" = _t, #"Entered By" = _t, #"Last User to Update" = _t, #"Check Number" = _t, #"Check Amount" = _t]),
xp = "Table.SelectRows(Source, each not List.Contains({"""",null},[" & Text.Combine(Table.SelectRows(TransformReference, each ([Remove Empty] = "Y"))[Old Name],"]) and not List.Contains({"""",null},[") & "]))",
#"Filtered Rows" = Expression.Evaluate(xp,[Table.SelectRows=Table.SelectRows,Source=Source,List.Contains=List.Contains])
in
#"Filtered Rows"
"TransformReference" is your instruction table.
let
Source = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("nVZNb6MwEP0rFudqT1XvCbRSpKbNimSlKurBNbPBCtjIHlfLv68JmEBasFmJi8dvZt58muMxSqBCskmiu6j9sK6AIPzDXhK93x2j1FRVwUGRF1pCMNhv95kLIAlopniFXIr2aoNQ3kgnXZVE/iWlFIBU1YSW0ogO80callsWKyeLjVIgWP1rb42N7awYuypOu+tg3xlPq6yNtjFqTQ6Ce+3vXsmLKT9ABeXtBroR+HD/Q2ypzUJmimC8JZFwjYp/mCZAssTXRnxKzmxJKd60SdZK3oaOaF1Ck80l4NWgwlcZkq7a82V2Cq4x3XmXPu2vIU7kfAuK5VRkXEPjsb+c8de055MCfso7eHcgcU7VCfS8dtMxZKdsPkN8uVi2gLnMfA3021CBHOsxToyrfEHaXmibfpj5eSqJGXbAHpSd5qtoqsK9o7H6jIab8EeBoCALU9pSbAZiAHZmLjc+li7NTVQ6YMON8QsWx6viJ+4WYjdWqTSKwbSOiyTFuvCu6biQtpNTpGi0NwqpMRDac3BcZ9EusrC111YojMeaijOJZealcAEGPgAuOH/dHXLJ9uxIcHFa0Mqth2W1WVWVkp+0+F+1IHb9b8AOlLYPil0W3lq4SV53i8me7aseF6DO35XenNYztd150NYVSnKosp5de3ERTBnppyEHdh4VavRYTLlu1X5+lDy/He9f", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [#"Old Name" = _t, #"Remove Col" = _t, #"New Name" = _t, #"New Type" = _t, #"Remove Errors" = _t, #"Remove Empty" = _t, Distinct = _t])
in
Source
You can do the other steps the same way.
lbendlin
Super User
2 years agolike so:
...
xp_remove_empty = "Table.SelectRows(Source, each not List.Contains({"""",null},[" & Text.Combine(Table.SelectRows(TransformReference, each ([Remove Empty] = "Y"))[Old Name],"]) and not List.Contains({"""",null},[") & "]))",
xp_remove_errors = "Table.RemoveRowsWithErrors(#""Removed Empty"", {""" & Text.Combine(Table.SelectRows(TransformReference, each ([Remove Errors] = "Y"))[Old Name],""", """) & """})",
#"Removed Empty" = Expression.Evaluate(xp_remove_empty,[Table.SelectRows=Table.SelectRows,Source=Source,List.Contains=List.Contains]),
#"Removed Errors" = Expression.Evaluate(xp_remove_errors,[Table.RemoveRowsWithErrors=Table.RemoveRowsWithErrors,#"Removed Empty"=#"Removed Empty"])
in
#"Removed Errors"