Forum Discussion
Matching Data that could be in different columns
- 5 years ago
Great - I'm so glad we're making progress! This line of the script determines which column number should be returned.
else List.Last ( d{t} )To get the column's position dynamically (instead of specifying the last column), add a step before add_dbAccount:
column_dbPosition = List.PositionOf ( Table.ColumnNames ( dbData ), "Account ID" ),And then reference it instead, like this:
else d{t}{column_dbPosition}Here is the complete new script with the column number dynamically referenced.
let // database data dbData = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("dY7BDoIwDIZfZdnZmKggcFs4ePCKN8JhaIUhY4bB+9uOgZBosnb9/q7/muf8aiy8a77DAjCnrakq65B0tnBjQGCUjvd3o1Gzo5XdpIhKS9V6/XA8BbzY5TyjByhk48bbyV/rfza/vlysb4pwyplWQz0Rm+tBaVGbYWXHMMIgStx4Cl0jtaLlsKTcg3xM5BpsFkovlMSb/bAjMDwFURw660sPNOeveR/CZbmnfAGQk4CVH54kjs68KD4=", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [#"First Name" = _t, Nickname = _t, Surname = _t, #"Full Name" = _t, #"email 1" = _t, #"email 2" = _t, #"Account ID" = _t]), dbData_ChangeTypes = Table.TransformColumnTypes(dbData,{{"First Name", type text}, {"Nickname", type text}, {"Surname", type text}, {"Full Name", type text}, {"email 1", type text}, {"email 2", type text}, {"Account ID", Int64.Type}}), // transform the table to a list of lists. dbData_List = Table.ToRows ( dbData_ChangeTypes ), // form data formData = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("bY3BCoMwEER/ZclZ+g0hBwte7S14iHUaY90sxPT/K1sLFnqYy7zHjPdmEdg94yoxbpe7sGlMJyCnhRkab0Zku+eADplcQZiU1cR2lsohrQe/JaaeU52V/5vvX1vI54MIKRGLxWnmqh11kvGRHuEJFX60tmCi9ns4vAE=", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [Email = _t, Name = _t]), formData_ChangeTypes = Table.TransformColumnTypes(formData,{{"Email", type text}, {"Name", type text}}), // Extract delimited parts of the name, so they can be evaluated separately. // Take caution when using Text.Split as extra delimiters can throw off the results. // Ideally, the form fields would be similar in structure to the db to be matched, but that is not the case here. // Edit/add steps if additional delimited parts need to be accomodated, like name suffixes. formData_NamePart1 = Table.AddColumn(formData_ChangeTypes, "Name Part 1", each Text.BeforeDelimiter([Name], " "), type text), formData_NamePart2 = Table.AddColumn(formData_NamePart1, "Name Part 2", each Text.AfterDelimiter([Name], " ", {0, RelativePosition.FromEnd}), type text), // Add a list object to each row, containing each record's values for specific columns in the table. // If steps were added for additonal name parts, also add them to the list of fields (r) in this step. // Match the form data to the db data. column_dbPosition = List.PositionOf ( Table.ColumnNames ( dbData ), "Account ID" ), add_dbAccount = Table.AddColumn ( formData_NamePart2, "Account ID", each let // Variables r = Record.ToList ( // List of specific fields Record.SelectFields ( _, { "Email", "Name Part 1", "Name Part 2" } ) ), // Current record values as a list d = dbData_List, // db list of lists t = List.PositionOf ( // Position of match in the db List.Transform ( // --> list of d, each List.ContainsAll ( _, // db lists r, // containing form data elements Comparer.FromCulture ( // Apply current culture Culture.Current, true // with case insensitivity ) ) ), true ), i = if t = -1 // = -1 if no matches were found then null //else List.Last ( d{t} ) else d{t}{column_dbPosition} in i, Int64.Type ), remove_columns = Table.RemoveColumns ( add_dbAccount, { "Name Part 1", "Name Part 2" } ) in remove_columnsAlternatively, you could use a function, like this: (I named the function fnGetDataFromExternalTable )
let fn = ( formsTable as table, dbTable as table, dbColumnName as text ) => let // database data dbData = dbTable, dbData_ChangeTypes = Table.TransformColumnTypes(dbData,{{"First Name", type text}, {"Nickname", type text}, {"Surname", type text}, {"Full Name", type text}, {"email 1", type text}, {"email 2", type text}, {"Account ID", Int64.Type}}), // transform the table to a list of lists. dbData_List = Table.ToRows ( dbData_ChangeTypes ), // form data formData = formsTable, formData_ChangeTypes = Table.TransformColumnTypes(formData,{{"Email", type text}, {"Name", type text}}), // Extract delimited parts of the name, so they can be evaluated separately. // Take caution when using Text.Split as extra delimiters can throw off the results. // Ideally, the form fields would be similar in structure to the db to be matched, but that is not the case here. // Edit/add steps if additional delimited parts need to be accomodated, like name suffixes. formData_NamePart1 = Table.AddColumn(formData_ChangeTypes, "Name Part 1", each Text.BeforeDelimiter([Name], " "), type text), formData_NamePart2 = Table.AddColumn(formData_NamePart1, "Name Part 2", each Text.AfterDelimiter([Name], " ", {0, RelativePosition.FromEnd}), type text), // Add a list object to each row, containing each record's values for specific columns in the table. // If steps were added for additonal name parts, also add them to the list of fields (r) in this step. // Match the form data to the db data. column_dbPosition = List.PositionOf ( Table.ColumnNames ( dbData ), dbColumnName ), add_dbAccount = Table.AddColumn ( formData_NamePart2, dbColumnName, each let // Variables r = Record.ToList ( // List of specific fields Record.SelectFields ( _, { "Email", "Name Part 1", "Name Part 2" } ) ), // Current record values as a list d = dbData_List, // db list of lists t = List.PositionOf ( // Position of match in the db List.Transform ( // --> list of d, each List.ContainsAll ( _, // db lists r, // containing form data elements Comparer.FromCulture ( // Apply current culture Culture.Current, true // with case insensitivity ) ) ), true ), i = if t = -1 // = -1 if no matches were found then null //else List.Last ( d{t} ) else d{t}{column_dbPosition} in i, Int64.Type ), remove_columns = Table.RemoveColumns ( add_dbAccount, { "Name Part 1", "Name Part 2" } ) in remove_columns in fnInvoke the function:
fnGetDataFromExternalTable ( formsDataTable, dbDataTable, "Account ID" )
Jakinta's solution is another way of handling this scenario for sure. It will work as long as all records in the form data actually exist in the database and the names entered into the form data only include one space (meaning the names in the form data must not include any middle initials, last name suffixes, etc.), otherwise errors will be returned. See the screensnips below. I have added two records to the form data. The first, George Jones, does not exist in the database. The second, Fred F Smith, I also added to the database, but without the middle initial.
I have used a few lines from Jankita's to my script and have tweaked the original error handling I included.
The error -1 is returned by List.PostionOf when it cannot find a match.
New Result without Errors
New Script
let
// database data
dbData = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("dY7BDoIwDIZfZdnZmKggcFs4ePCKN8JhaIUhY4bB+9uOgZBosnb9/q7/muf8aiy8a77DAjCnrakq65B0tnBjQGCUjvd3o1Gzo5XdpIhKS9V6/XA8BbzY5TyjByhk48bbyV/rfza/vlysb4pwyplWQz0Rm+tBaVGbYWXHMMIgStx4Cl0jtaLlsKTcg3xM5BpsFkovlMSb/bAjMDwFURw660sPNOeveR/CZbmnfAGQk4CVH54kjs68KD4=", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [#"First Name" = _t, Nickname = _t, Surname = _t, #"Full Name" = _t, #"email 1" = _t, #"email 2" = _t, #"Account ID" = _t]),
dbData_ChangeTypes = Table.TransformColumnTypes(dbData,{{"First Name", type text}, {"Nickname", type text}, {"Surname", type text}, {"Full Name", type text}, {"email 1", type text}, {"email 2", type text}, {"Account ID", Int64.Type}}),
// transform the table to a list of lists.
dbData_List = Table.ToRows ( dbData_ChangeTypes ),
// form data
formData = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("bY3BCoMwEER/ZclZ+g0hBwte7S14iHUaY90sxPT/K1sLFnqYy7zHjPdmEdg94yoxbpe7sGlMJyCnhRkab0Zku+eADplcQZiU1cR2lsohrQe/JaaeU52V/5vvX1vI54MIKRGLxWnmqh11kvGRHuEJFX60tmCi9ns4vAE=", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [Email = _t, Name = _t]),
formData_ChangeTypes = Table.TransformColumnTypes(formData,{{"Email", type text}, {"Name", type text}}),
// Extract delimited parts of the name, so they can be evaluated separately.
// Take caution when using Text.Split as extra delimiters can throw off the results.
// Ideally, the form fields would be similar in structure to the db to be matched, but that is not the case here.
// Edit/add steps if additional delimited parts need to be accomodated, like name suffixes.
formData_NamePart1 = Table.AddColumn(formData_ChangeTypes, "Name Part 1", each Text.BeforeDelimiter([Name], " "), type text),
formData_NamePart2 = Table.AddColumn(formData_NamePart1, "Name Part 2", each Text.AfterDelimiter([Name], " ", {0, RelativePosition.FromEnd}), type text),
// Add a list object to each row, containing each record's values for specific columns in the table.
// If steps were added for additonal name parts, also add them to the list of fields (r) in this step.
// Match the form data to the db data.
add_dbAccount = Table.AddColumn (
formData_NamePart2,
"Account ID",
each
let // Variables
r = Record.ToList ( // List of specific fields
Record.SelectFields (
_, { "Email", "Name Part 1", "Name Part 2" }
)
), // Current record values as a list
d = dbData_List, // db list of lists
t = List.PositionOf ( // Position of match in the db
List.Transform ( // --> list of
d, each List.ContainsAll (
_, // db lists
r, // containing form data elements
Comparer.FromCulture ( // Apply current culture
Culture.Current, true // with case insensitivity
)
)
), true
),
i = if t = -1 // = -1 if no matches were found
then null
else List.Last ( d{t} )
in
i,
Int64.Type
),
remove_columns = Table.RemoveColumns (
add_dbAccount,
{ "Name Part 1", "Name Part 2" }
)
in
remove_columns
Hi Jen
Thanks so much for this, the first version of the code you sent worked. I even figured out how to get it to pull other matching columns into the form table as wel but it is very slow to load (even slower than my original multiple merge method).
This new version also works but it doesnt load the data in the Account ID column of the database table when it matches, it loads the last column in the database table which isn't the Account ID column. How can i change this?
there are 3 other columns that i want to load when it finds a match.
Joanne
- jennratten5 years agoSuper User
Great - I'm so glad we're making progress! This line of the script determines which column number should be returned.
else List.Last ( d{t} )To get the column's position dynamically (instead of specifying the last column), add a step before add_dbAccount:
column_dbPosition = List.PositionOf ( Table.ColumnNames ( dbData ), "Account ID" ),And then reference it instead, like this:
else d{t}{column_dbPosition}Here is the complete new script with the column number dynamically referenced.
let // database data dbData = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("dY7BDoIwDIZfZdnZmKggcFs4ePCKN8JhaIUhY4bB+9uOgZBosnb9/q7/muf8aiy8a77DAjCnrakq65B0tnBjQGCUjvd3o1Gzo5XdpIhKS9V6/XA8BbzY5TyjByhk48bbyV/rfza/vlysb4pwyplWQz0Rm+tBaVGbYWXHMMIgStx4Cl0jtaLlsKTcg3xM5BpsFkovlMSb/bAjMDwFURw660sPNOeveR/CZbmnfAGQk4CVH54kjs68KD4=", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [#"First Name" = _t, Nickname = _t, Surname = _t, #"Full Name" = _t, #"email 1" = _t, #"email 2" = _t, #"Account ID" = _t]), dbData_ChangeTypes = Table.TransformColumnTypes(dbData,{{"First Name", type text}, {"Nickname", type text}, {"Surname", type text}, {"Full Name", type text}, {"email 1", type text}, {"email 2", type text}, {"Account ID", Int64.Type}}), // transform the table to a list of lists. dbData_List = Table.ToRows ( dbData_ChangeTypes ), // form data formData = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("bY3BCoMwEER/ZclZ+g0hBwte7S14iHUaY90sxPT/K1sLFnqYy7zHjPdmEdg94yoxbpe7sGlMJyCnhRkab0Zku+eADplcQZiU1cR2lsohrQe/JaaeU52V/5vvX1vI54MIKRGLxWnmqh11kvGRHuEJFX60tmCi9ns4vAE=", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [Email = _t, Name = _t]), formData_ChangeTypes = Table.TransformColumnTypes(formData,{{"Email", type text}, {"Name", type text}}), // Extract delimited parts of the name, so they can be evaluated separately. // Take caution when using Text.Split as extra delimiters can throw off the results. // Ideally, the form fields would be similar in structure to the db to be matched, but that is not the case here. // Edit/add steps if additional delimited parts need to be accomodated, like name suffixes. formData_NamePart1 = Table.AddColumn(formData_ChangeTypes, "Name Part 1", each Text.BeforeDelimiter([Name], " "), type text), formData_NamePart2 = Table.AddColumn(formData_NamePart1, "Name Part 2", each Text.AfterDelimiter([Name], " ", {0, RelativePosition.FromEnd}), type text), // Add a list object to each row, containing each record's values for specific columns in the table. // If steps were added for additonal name parts, also add them to the list of fields (r) in this step. // Match the form data to the db data. column_dbPosition = List.PositionOf ( Table.ColumnNames ( dbData ), "Account ID" ), add_dbAccount = Table.AddColumn ( formData_NamePart2, "Account ID", each let // Variables r = Record.ToList ( // List of specific fields Record.SelectFields ( _, { "Email", "Name Part 1", "Name Part 2" } ) ), // Current record values as a list d = dbData_List, // db list of lists t = List.PositionOf ( // Position of match in the db List.Transform ( // --> list of d, each List.ContainsAll ( _, // db lists r, // containing form data elements Comparer.FromCulture ( // Apply current culture Culture.Current, true // with case insensitivity ) ) ), true ), i = if t = -1 // = -1 if no matches were found then null //else List.Last ( d{t} ) else d{t}{column_dbPosition} in i, Int64.Type ), remove_columns = Table.RemoveColumns ( add_dbAccount, { "Name Part 1", "Name Part 2" } ) in remove_columnsAlternatively, you could use a function, like this: (I named the function fnGetDataFromExternalTable )
let fn = ( formsTable as table, dbTable as table, dbColumnName as text ) => let // database data dbData = dbTable, dbData_ChangeTypes = Table.TransformColumnTypes(dbData,{{"First Name", type text}, {"Nickname", type text}, {"Surname", type text}, {"Full Name", type text}, {"email 1", type text}, {"email 2", type text}, {"Account ID", Int64.Type}}), // transform the table to a list of lists. dbData_List = Table.ToRows ( dbData_ChangeTypes ), // form data formData = formsTable, formData_ChangeTypes = Table.TransformColumnTypes(formData,{{"Email", type text}, {"Name", type text}}), // Extract delimited parts of the name, so they can be evaluated separately. // Take caution when using Text.Split as extra delimiters can throw off the results. // Ideally, the form fields would be similar in structure to the db to be matched, but that is not the case here. // Edit/add steps if additional delimited parts need to be accomodated, like name suffixes. formData_NamePart1 = Table.AddColumn(formData_ChangeTypes, "Name Part 1", each Text.BeforeDelimiter([Name], " "), type text), formData_NamePart2 = Table.AddColumn(formData_NamePart1, "Name Part 2", each Text.AfterDelimiter([Name], " ", {0, RelativePosition.FromEnd}), type text), // Add a list object to each row, containing each record's values for specific columns in the table. // If steps were added for additonal name parts, also add them to the list of fields (r) in this step. // Match the form data to the db data. column_dbPosition = List.PositionOf ( Table.ColumnNames ( dbData ), dbColumnName ), add_dbAccount = Table.AddColumn ( formData_NamePart2, dbColumnName, each let // Variables r = Record.ToList ( // List of specific fields Record.SelectFields ( _, { "Email", "Name Part 1", "Name Part 2" } ) ), // Current record values as a list d = dbData_List, // db list of lists t = List.PositionOf ( // Position of match in the db List.Transform ( // --> list of d, each List.ContainsAll ( _, // db lists r, // containing form data elements Comparer.FromCulture ( // Apply current culture Culture.Current, true // with case insensitivity ) ) ), true ), i = if t = -1 // = -1 if no matches were found then null //else List.Last ( d{t} ) else d{t}{column_dbPosition} in i, Int64.Type ), remove_columns = Table.RemoveColumns ( add_dbAccount, { "Name Part 1", "Name Part 2" } ) in remove_columns in fnInvoke the function:
fnGetDataFromExternalTable ( formsDataTable, dbDataTable, "Account ID" ) - Jakinta5 years agoSolution Sage
DataBase
Form
Result
M-code
let Form = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("i45WyspPdQDipJz89PRiveT8XCUdJa/8VAUnsIBSrE60UlJqngMQQyWdUvMUnIpSE1PAciWZuQ4Z+SW5iZk5UPmQzFyF4NzMkgywPDbjg0uLE/OQLaioAIpWKFQqxcYCAA==", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [Email = _t, Name = _t]), FormSplit = Table.SplitColumn(Form, "Name", Splitter.SplitTextByDelimiter(" ", QuoteStyle.Csv), {"Name.1", "Name.2"}), DataBase = Table.FromRows(Json.Document(Binary.Decompress(Binary.FromText("dY67DsIgFIZfhTA3JipNdSMdXXFrOnBLoSnFSPv+Hi5Vm+hwLt8H+aHr8M0H/TC4gkVDbyc/DCFh9OjNo9cUSiQ+SO/AhTXwORs6OG6n4o+nM4HBOe6rDrN4C5CtuweS/uT/y/r1bskXIuXfbXS5M2cXkwlt+2IdNX75ykRQNWmuMKRMGa2eR+5s/CassT81V5nSAdqEKEJE3v0UTihUIdJcahhK4b5/AQ==", BinaryEncoding.Base64), Compression.Deflate)), let _t = ((type nullable text) meta [Serialized.Text = true]) in type table [#"First Name" = _t, Nickname = _t, Surname = _t, #"Full Name" = _t, #"email 1" = _t, #"email 2" = _t, #"Account ID" = _t, Column1 = _t]), #"Added Custom" = Table.AddColumn(FormSplit, "Account ID", each let r=Record.ToList(_), d=Table.ToRows(DataBase), t=List.PositionOf(List.Transform(d, each List.ContainsAll(_,r)), true),c=Table.ColumnNames(DataBase), i=List.PositionOf( Table.ColumnNames(DataBase), "Account ID") in if t<0 then null else (d{t}{i})), FINAL = Table.CombineColumns(#"Added Custom",{"Name.1", "Name.2"},Combiner.CombineTextByDelimiter(" ", QuoteStyle.None),"Name") in FINAL