Forum Discussion

Centaur's avatar
Centaur
Helper V
1 year ago
Solved

Duplicate Invoice No and Use Data from 2nd record

Hello Experts,

 

I am going to try and explain simply. 

First of all I am a novice user of PQ.  Not a programmer. 

 

I have a data set

There are duplicates on [invoiceNo]

This is ok (I delete dupes in a later step)

I need to grab the [GoodsReceipt] and put it in the first record (prior to deleting the dupes)

 

A little more detail:

I have a sort on [invoiceNo] and [Source]

[Source] = 1-PC or 2-Stampli (the 1 or 2 governs the sort)

the sort is ascending in both cases.

 

In every case the [GoodsReceipt] is in the [Source] = 2-Stampli (the 2nd record)

so it will be in the 2nd record.

How can I move the [Goods Receipt] to the first record? 

Noting this is only when there is a dupe on [Invoice No Stripped] AND [Invoice amount].  

 

here is a screen shot:

 

thank you and grateful for the help.

Let me know if it is not clear enough and I will provide more info.  

 

fwiw here are the first couple lines of code up to the sort line:

let
Source = Table.Combine({StampliALLPaid, #"Project Costs"}),
#"Uppercased Text" = Table.TransformColumns(Source,{{"Invoice No Stripped", Text.Upper, type text}}),
#"Reordered Columns3" = Table.ReorderColumns(#"Uppercased Text",{"Processing Began", "Vendor", "WDDate", "Invoice #", "Invoice No Stripped", "Source", "Invoice amount", "Currency", "Invoice date", "Invoice due date", "Company Code", "Budget Category", "Invoice Status", "Goods Receipt", "Project ", "Funding Date", "DDNo", "USD Amount", "Account", "TypeDraw"}),
#"Sorted Rows" = Table.Sort(#"Reordered Columns3",{{"Invoice No Stripped", Order.Ascending}, {"Source", Order.Ascending}}),

 

  • ronrsnfld's avatar
    ronrsnfld
    1 year ago

    Here is another pair of code blocks that might work on what you have:

    to Copy

    let
        Source = Excel.CurrentWorkbook(){[Name="Table1"]}[Content],
        #"Changed Type" = Table.TransformColumnTypes(Source,{
                            {"Processing Began", type date}, {"Vendor", type text}, {"WDDate", type date}, 
                            {"Invoice #", type text}, {"Invoice No Stripped", type text}, {"Invoice amount", type number}, 
                            {"Currency", type text}, {"Invoice date", type date}, {"Invoice due date", type date}, 
                            {"Company Code", type text}, {"Budget Category", type text}, {"Invoice Status", type text}, 
                            {"Goods Receipt", Int64.Type}, {"Project ", type text}, {"Source", type text}, 
                            {"Funding Date", type any}, {"DDNo", type any}, {"USD Amount", type any}, 
                            {"Account", type any}, {"TypeDraw", type any}}),
        
        #"Grouped Rows" = Table.Group(#"Changed Type", {"Invoice No Stripped", "Invoice amount"}, {
            {"Count", (t)=>Table.ReplaceValue(
                            t,
                            List.RemoveNulls(List.Distinct(t[Goods Receipt])){0}?,
                            null,
                            (x,y,z)=>y,
                            {"Goods Receipt"}
            ), 
                type table [Processing Began=nullable date, Vendor=nullable text, WDDate=nullable date, #"Invoice #"=nullable text, 
                    Invoice No Stripped=nullable text, Invoice amount=nullable number, Currency=nullable text, 
                    Invoice date=nullable date, Invoice due date=nullable date, Company Code=nullable text, 
                    Budget Category=nullable text, Invoice Status=nullable text, Goods Receipt=nullable number, 
                    #"Project "=nullable text, Source=nullable text, Funding Date=nullable text, DDNo=nullable text, 
                    USD Amount=nullable text, Account=nullable text, TypeDraw=nullable text]}}),
        
        #"Removed Columns" = Table.RemoveColumns(#"Grouped Rows",{"Invoice No Stripped", "Invoice amount"}),
        #"Expanded Count" = Table.ExpandTableColumn(#"Removed Columns", "Count", 
            {"Processing Began", "Vendor", "WDDate", "Invoice #", "Invoice No Stripped", "Invoice amount", "Currency", 
            "Invoice date", "Invoice due date", "Company Code", "Budget Category", "Invoice Status", "Goods Receipt", 
            "Project ", "Source", "Funding Date", "DDNo", "USD Amount", "Account", "TypeDraw"})
    in
        #"Expanded Count"

    to Move to 1-PC

    let
        Source = Excel.CurrentWorkbook(){[Name="Table1"]}[Content],
        #"Changed Type" = Table.TransformColumnTypes(Source,{
                            {"Processing Began", type date}, {"Vendor", type text}, {"WDDate", type date}, 
                            {"Invoice #", type text}, {"Invoice No Stripped", type text}, {"Invoice amount", type number}, 
                            {"Currency", type text}, {"Invoice date", type date}, {"Invoice due date", type date}, 
                            {"Company Code", type text}, {"Budget Category", type text}, {"Invoice Status", type text}, 
                            {"Goods Receipt", Int64.Type}, {"Project ", type text}, {"Source", type text}, 
                            {"Funding Date", type any}, {"DDNo", type any}, {"USD Amount", type any}, 
                            {"Account", type any}, {"TypeDraw", type any}}),
        
        #"Grouped Rows" = Table.Group(#"Changed Type", {"Invoice No Stripped", "Invoice amount"}, {
            {"Count", (t)=>Table.ReplaceValue(
                            t,
                            each [Source],
                            List.RemoveNulls(List.Distinct(t[Goods Receipt])){0}?,
                            (x,y,z)=> if y = "1-PC" then z else null,
                            {"Goods Receipt"}
            ), 
                type table [Processing Began=nullable date, Vendor=nullable text, WDDate=nullable date, #"Invoice #"=nullable text, 
                    Invoice No Stripped=nullable text, Invoice amount=nullable number, Currency=nullable text, 
                    Invoice date=nullable date, Invoice due date=nullable date, Company Code=nullable text, 
                    Budget Category=nullable text, Invoice Status=nullable text, Goods Receipt=nullable number, 
                    #"Project "=nullable text, Source=nullable text, Funding Date=nullable text, DDNo=nullable text, 
                    USD Amount=nullable text, Account=nullable text, TypeDraw=nullable text]}}),
        
        #"Removed Columns" = Table.RemoveColumns(#"Grouped Rows",{"Invoice No Stripped", "Invoice amount"}),
        #"Expanded Count" = Table.ExpandTableColumn(#"Removed Columns", "Count", 
            {"Processing Began", "Vendor", "WDDate", "Invoice #", "Invoice No Stripped", "Invoice amount", "Currency", 
            "Invoice date", "Invoice due date", "Company Code", "Budget Category", "Invoice Status", "Goods Receipt", 
            "Project ", "Source", "Funding Date", "DDNo", "USD Amount", "Account", "TypeDraw"})
    in
        #"Expanded Count"

     

    Again both of the above work without a problem on the sample data you provided.

     

33 Replies

  • Looking at only the data you present, why not just 

    Table.FillUp(#"Previous_Step",{"Goods Receipt"})

     

    You can also generate that step from the UI by selecting the column and then select FillUp from the dropdown menu.

    If that doesn't do what you want, present a more comprehensive example.

    • Centaur's avatar
      Centaur
      Helper V

      Hi Ronrsnfld,

      that is a great idea.  If I could add a condition to it since I need to only fill up when there is a duplicate on [Invoice No Stripped] and [Invoice Amount].  If that can be done, I think it would work.  If you have an idea how that can be done I am all ears!  

      thank you very much. 

      • ronrsnfld's avatar
        ronrsnfld
        Super User

        Use the Table.GroupBy function, and create a custom aggregation to fill up that column.

        Note:

        1. Table.GroupBy will not necessarily preserve your sort order. 
          If that is an issue:
        2. Work arounds: (after the sort and before the GroupBy)
          1. Use Table.Buffer (preferred)
          2. Add an Index column

         

        Simple way of doing this:

        1.    Select the Invoice column and GroupBy
        2.    For "Operation" choose "All Rows"
        3.    Open the Advanced Editor
          1. Look for a line, in the "Grouped Rows" step, that looks like:
            {"Count", each _, type table [...]}
          2. Replace each _, with each Table.FillUp(_, {"Goods Receipt"}
          3. Close the Advanced Editor
        4. Re-Expand the table column.

         

        Please note that had you posted a more comprehensive data sample, and/or included the information that only some lines had duplicate invoice numbers, I would have suggested this solution initially.

  • Please provide sample data that covers your issue or question completely, in a usable format (not as a screenshot).
    Do not include sensitive information. Do not include anything that is unrelated to the issue or question.
    Please show the expected outcome based on the sample data you provided.

    Need help uploading data? https://community.fabric.microsoft.com/t5/Community-Blog/How-to-provide-sample-data-in-the-Power-BI-Forum/ba-p/963216
    Want faster answers? https://community.fabric.microsoft.com/t5/Desktop/How-to-Get-Your-Question-Answered-Quickly/m-p/1447523

  • Hi Centaur ,


    For each duplicate [InvoiceNo] (with matching [Invoice amount]), copy the [GoodsReceipt] from the [Source]=2-StampIt row to the [Source]=1-PC row.

     

    Sort your table by [Invoice No Stripped], [Invoice amount], and [Source] (ascending so 1-PC comes before 2-StampIt). Group your table by [Invoice No Stripped] and [Invoice amount]. Inside each group, check if you have both 1-PC and 2-StampIt. If yes, take the [GoodsReceipt] value from the 2-StampIt row and update the 1-PC row with it. Expand back to a flat table and remove the “AllRows” helper column.

     

    Here’s an example M script for the core part:

     

    GroupedRows = Table.Group(#"Sorted Rows", {"Invoice No Stripped", "Invoice amount"}, {
        {"AllData", each _, type table [Invoice No Stripped=nullable text, Source=nullable text, Invoice amount=nullable number, Goods Receipt=nullable text]}
    }),
    AddGRColumn = Table.AddColumn(GroupedRows, "GoodsReceiptToKeep", each 
        let
            groupTable = [AllData],
            grValue = try Record.Field(Table.SelectRows(groupTable, each Text.Contains([Source], "Stampli")){0}, "Goods Receipt") otherwise null
        in grValue
    ),
    Expanded = Table.ExpandTableColumn(AddGRColumn, "AllData", {"Invoice No Stripped", "Source", "Invoice amount", "Goods Receipt"}, {"Invoice No Stripped", "Source", "Invoice amount", "Goods Receipt"}),
    UpdatedGR = Table.AddColumn(Expanded, "Final Goods Receipt", each if Text.Contains([Source], "1-PC") then [GoodsReceiptToKeep] else [Goods Receipt]),
    RemoveHelper = Table.RemoveColumns(UpdatedGR, {"GoodsReceiptToKeep", "Goods Receipt"})
    

     

    • Centaur's avatar
      Centaur
      Helper V

      wow Rohit, that is amazing.  I am sorry to say that it looks like I have an error in my first line of code ("RemovedDuplicates") after your last line ("RemoveHelper").  I did refer to the previous step ("RemoveHelper") however that did not fix it as there is still a "Token expected" error and higlights my first line of code ("RemovedDuplicates") after the ("RemoveHelper") line.  

       

      Please see my entire code with your code.

      I am showing a blank row to better offset your code (starting with "GroupedRows" and ending on "RemoveHelper". 

      Do you happen to see the reason I have the expected token error on the "RemovedDuplicates" row?

       

      NOTE:  I confirm if I remove all the code after your last line the token error is removed and no syntax errors either (and also changed the "In" line to refer to #RemoveHelper#) however I do have an error on the Expanded step once I step out of the Advanced Editor:

       

        

      let
          Source = Table.Combine({StampliALLPaid, #"Project Costs"}),
          #"Uppercased Text" = Table.TransformColumns(Source,{{"Invoice No Stripped", Text.Upper, type text}}),
          #"Reordered Columns3" = Table.ReorderColumns(#"Uppercased Text",{"Processing Began", "Vendor", "WDDate", "Invoice #", "Invoice No Stripped", "Source", "Invoice amount", "Currency", "Invoice date", "Invoice due date", "Company Code", "Budget Category", "Invoice Status", "Goods Receipt", "Project ", "Funding Date", "DDNo", "USD Amount", "Account", "TypeDraw"}),
          #"Sorted Rows" = Table.Sort(#"Reordered Columns3",{{"Invoice No Stripped", Order.Ascending}, {"Source", Order.Ascending}}),
       
          GroupedRows = Table.Group(#"Sorted Rows", {"Invoice No Stripped", "Invoice amount"}, {{"AllData", each _, type table [Invoice No Stripped=nullable text, Source=nullable text, Invoice amount=nullable number, Goods Receipt=nullable text]}}),
      AddGRColumn = Table.AddColumn(GroupedRows, "GoodsReceiptToKeep", each 
          let
              groupTable = [AllData],
              grValue = try Record.Field(Table.SelectRows(groupTable, each Text.Contains([Source], "Stampli")){0}, "Goods Receipt") otherwise null
          in grValue),
      Expanded = Table.ExpandTableColumn(AddGRColumn, "AllData", {"Invoice No Stripped", "Source", "Invoice amount", "Goods Receipt"}, {"Invoice No Stripped", "Source", "Invoice amount", "Goods Receipt"}),
      UpdatedGR = Table.AddColumn(Expanded, "Final Goods Receipt", each if Text.Contains([Source], "1-PC") then [GoodsReceiptToKeep] else [Goods Receipt]),
      RemoveHelper = Table.RemoveColumns(UpdatedGR, {"GoodsReceiptToKeep", "Goods Receipt"})
      
          #"Removed Duplicates" = Table.Distinct(#"RemoveHelper", {"Invoice No Stripped", "Invoice amount"}),
          #"Add Column" = Table.AddColumn(#"Removed Duplicates", "Custom", each if [WDDate] = null then [Invoice due date] else [WDDate]),
          #"Reordered Columns" = Table.ReorderColumns(#"Add Column",{"Processing Began", "Vendor", "WDDate", "Custom", "Invoice #", "Invoice No Stripped", "Invoice amount", "Currency", "Invoice date", "Invoice due date", "Company Code", "Budget Category", "Invoice Status", "Goods Receipt", "Project ", "Funding Date", "DDNo", "USD Amount", "Account", "TypeDraw"}),
          Custom1 = Table.AddColumn(#"Reordered Columns", "Custom2", each if [Custom] = null then [Funding Date] else [Custom]),
          #"Reordered Columns1" = Table.ReorderColumns(Custom1,{"Processing Began", "Vendor", "WDDate", "Custom2", "Custom", "Invoice #", "Invoice No Stripped", "Invoice amount", "Currency", "Invoice date", "Invoice due date", "Company Code", "Budget Category", "Invoice Status", "Goods Receipt", "Project ", "Funding Date", "DDNo", "USD Amount", "Account", "TypeDraw"}),
          #"Removed Columns" = Table.RemoveColumns(#"Reordered Columns1",{"WDDate", "Custom"}),
          #"Renamed Columns" = Table.RenameColumns(#"Removed Columns",{{"Custom2", "WDDate"}}),
          #"Changed Type" = Table.TransformColumnTypes(#"Renamed Columns",{{"WDDate", type date}}),
          #"Reordered Columns2" = Table.ReorderColumns(#"Changed Type",{"Vendor", "WDDate", "Invoice #", "Invoice No Stripped", "Invoice amount", "Currency", "Invoice date", "Invoice due date", "Company Code", "Budget Category", "Invoice Status", "Goods Receipt", "Project ", "Funding Date", "DDNo", "USD Amount", "Account", "TypeDraw", "Processing Began"}),
          #"Renamed Columns1" = Table.RenameColumns(#"Reordered Columns2",{{"WDDate", "WDDate_UseThis"}})
      in
          #"Renamed Columns1"

       

      thank you very much!

      Greatly appreciate your expert advice.  

      • rohit1991's avatar
        rohit1991
        Super User

        Hi Centaur ,

        You're very close! The issue you're running into—"Token expected"—is due to a missing comma at the end of the previous step. In Power Query M code, each step must be separated by a comma, except for the final one before the in keyword.

        In your code, you ended the RemoveHelper step like this:

        RemoveHelper = Table.RemoveColumns(UpdatedGR, {"GoodsReceiptToKeep", "Goods Receipt"})
        

         

        But then immediately started the next step with:

        #"Removed Duplicates" = Table.Distinct(#"RemoveHelper", {"Invoice No Stripped", "Invoice amount"}),
        

        Since "Removed Duplicates" is technically the next step after RemoveHelper, you need to put a comma at the end of the RemoveHelper line to separate the steps properly. Here’s the corrected version of that section:

        RemoveHelper = Table.RemoveColumns(UpdatedGR, {"GoodsReceiptToKeep", "Goods Receipt"}),  
        #"Removed Duplicates" = Table.Distinct(RemoveHelper, {"Invoice No Stripped", "Invoice amount"}),
        

         

        Also, you don't need the #"" syntax for RemoveHelper unless it includes spaces or special characters—RemoveHelper works fine as a reference on its own, and using #"RemoveHelper" can sometimes cause confusion if it was never defined with quotes.

         

        Here is what i believe would solve your issue:

        let
            Source = Table.Combine({StampliALLPaid, #"Project Costs"}),
        
            #"Uppercased Text" = Table.TransformColumns(Source,{{"Invoice No Stripped", Text.Upper, type text}}),
        
            #"Reordered Columns3" = Table.ReorderColumns(#"Uppercased Text",{
                "Processing Began", "Vendor", "WDDate", "Invoice #", "Invoice No Stripped", "Source", "Invoice amount", 
                "Currency", "Invoice date", "Invoice due date", "Company Code", "Budget Category", "Invoice Status", 
                "Goods Receipt", "Project ", "Funding Date", "DDNo", "USD Amount", "Account", "TypeDraw"}),
        
            #"Sorted Rows" = Table.Sort(#"Reordered Columns3",{
                {"Invoice No Stripped", Order.Ascending}, 
                {"Source", Order.Ascending}}),
        
            GroupedRows = Table.Group(#"Sorted Rows", {"Invoice No Stripped", "Invoice amount"}, {
                {"AllData", each _, type table [
                    Invoice No Stripped=nullable text, 
                    Source=nullable text, 
                    Invoice amount=nullable number, 
                    Goods Receipt=nullable text,
                    Processing Began=nullable datetime, Vendor=nullable text, WDDate=nullable date, 
                    Invoice #=nullable text, Currency=nullable text, Invoice date=nullable date, 
                    Invoice due date=nullable date, Company Code=nullable text, Budget Category=nullable text, 
                    Invoice Status=nullable text, Project = nullable text, Funding Date=nullable date, 
                    DDNo=nullable text, USD Amount=nullable number, Account=nullable text, TypeDraw=nullable text
                ]}
            }),
        
            AddGRColumn = Table.AddColumn(GroupedRows, "GoodsReceiptToKeep", each 
                let
                    groupTable = [AllData],
                    grValue = try Record.Field(
                        Table.SelectRows(groupTable, each Text.Contains([Source], "Stampli")){0}, 
                        "Goods Receipt"
                    ) otherwise null
                in grValue
            ),
        
            Expanded = Table.ExpandTableColumn(AddGRColumn, "AllData", {
                "Processing Began", "Vendor", "WDDate", "Invoice #", "Invoice No Stripped", "Source", 
                "Invoice amount", "Currency", "Invoice date", "Invoice due date", "Company Code", 
                "Budget Category", "Invoice Status", "Goods Receipt", "Project ", "Funding Date", "DDNo", 
                "USD Amount", "Account", "TypeDraw"
            }),
        
            UpdatedGR = Table.AddColumn(Expanded, "Final Goods Receipt", each 
                if Text.Contains([Source], "1-PC") then [GoodsReceiptToKeep] else [Goods Receipt]
            ),
        
            RemoveHelper = Table.RemoveColumns(UpdatedGR, {"GoodsReceiptToKeep", "Goods Receipt"}),
        
            #"Removed Duplicates" = Table.Distinct(RemoveHelper, {"Invoice No Stripped", "Invoice amount"}),
        
            #"Add Column" = Table.AddColumn(#"Removed Duplicates", "Custom", each 
                if [WDDate] = null then [Invoice due date] else [WDDate]
            ),
        
            #"Reordered Columns" = Table.ReorderColumns(#"Add Column",{
                "Processing Began", "Vendor", "WDDate", "Custom", "Invoice #", "Invoice No Stripped", 
                "Invoice amount", "Currency", "Invoice date", "Invoice due date", "Company Code", 
                "Budget Category", "Invoice Status", "Final Goods Receipt", "Project ", "Funding Date", 
                "DDNo", "USD Amount", "Account", "TypeDraw"
            }),
        
            Custom1 = Table.AddColumn(#"Reordered Columns", "Custom2", each 
                if [Custom] = null then [Funding Date] else [Custom]
            ),
        
            #"Reordered Columns1" = Table.ReorderColumns(Custom1,{
                "Processing Began", "Vendor", "WDDate", "Custom2", "Custom", "Invoice #", 
                "Invoice No Stripped", "Invoice amount", "Currency", "Invoice date", "Invoice due date", 
                "Company Code", "Budget Category", "Invoice Status", "Final Goods Receipt", "Project ", 
                "Funding Date", "DDNo", "USD Amount", "Account", "TypeDraw"
            }),
        
            #"Removed Columns" = Table.RemoveColumns(#"Reordered Columns1",{"WDDate", "Custom"}),
        
            #"Renamed Columns" = Table.RenameColumns(#"Removed Columns",{{"Custom2", "WDDate"}}),
        
            #"Changed Type" = Table.TransformColumnTypes(#"Renamed Columns",{{"WDDate", type date}}),
        
            #"Reordered Columns2" = Table.ReorderColumns(#"Changed Type",{
                "Vendor", "WDDate", "Invoice #", "Invoice No Stripped", "Invoice amount", "Currency", 
                "Invoice date", "Invoice due date", "Company Code", "Budget Category", "Invoice Status", 
                "Final Goods Receipt", "Project ", "Funding Date", "DDNo", "USD Amount", "Account", 
                "TypeDraw", "Processing Began"
            }),
        
            #"Renamed Columns1" = Table.RenameColumns(#"Reordered Columns2",{{"WDDate", "WDDate_UseThis"}})
        in
            #"Renamed Columns1"