Dyanmic Looping in List.Generate (Example-BoxOfficeMojo)

d_gosbellI picked up the cue from your suggestion "what PG2 does to loop through all the pages in order to loop through all the years" and many thanks to Gil Raviv for teaching the nested loop through - NestedLoop-GilRaviv

(page as number, year as number) as table =>
let
    Source = Web.Page(Web.Contents("http://boxofficemojo.com/yearly/chart/?page=" & Number.ToText(page) & "&view=releasedate&view2=domestic&yr=" & Number.ToText(year) & "&p=.htm")),
    Data1 = Source{1}[Data],
    RemoveBottom = Table.RemoveLastN(Data1,3)
in
    RemoveBottom

GD2

let
    maxYear = 3000,
    Source = Web.BrowserContents("https://www.boxofficemojo.com/yearly/chart/?page=1&view=releasedate&view2=domestic&yr=" & Number.ToText(maxYear) & "&p=.htm"),
    #"Extracted Table From Html" = Html.Table(Source, {{"Column1", "DIV[id='body'] > TABLE:nth-child(5) > TR > TD:not([colspan]):not([rowspan]):nth-child(1):nth-last-child(2), DIV[id='body'] > TABLE:nth-child(5) > * > TR > TD:not([colspan]):not([rowspan]):nth-child(1):nth-last-child(2)"}, {"Column2", "DIV[id='body'] > TABLE:nth-child(5) > TR > TD:not([colspan]):not([rowspan]):nth-child(1):nth-last-child(2) + TD:not([colspan]):not([rowspan]):nth-child(2):nth-last-child(1), DIV[id='body'] > TABLE:nth-child(5) > * > TR > TD:not([colspan]):not([rowspan]):nth-child(1):nth-last-child(2) + TD:not([colspan]):not([rowspan]):nth-child(2):nth-last-child(1)"}}, [RowSelector="DIV[id='body'] > TABLE:nth-child(5) > TR, DIV[id='body'] > TABLE:nth-child(5) > * > TR"]),
    #"Changed Type" = Table.TransformColumnTypes(#"Extracted Table From Html",{{"Column1", type text}, {"Column2", type text}}),
    #"Split Column by Delimiter" = Table.SplitColumn(#"Changed Type", "Column1", Splitter.SplitTextByEachDelimiter({" "}, QuoteStyle.Csv, false), {"Column1.1", "Column1.2"}),
    #"Changed Type1" = Table.TransformColumnTypes(#"Split Column by Delimiter",{{"Column1.1", Int64.Type}, {"Column1.2", type text}}),
    #"Removed Columns" = Table.RemoveColumns(#"Changed Type1",{"Column1.2", "Column2"}),
    pageYear = Table.RenameColumns(#"Removed Columns",{{"Column1.1", "Year"}}),
    Year = pageYear{0}[Year]
in
    Year

Max_Year

(x as number, y as number)=>
              let
                 Source = List.Generate(
                             ()=>
                                 [Page=1, Result=try GD2(y,x) otherwise null, year=x],
                                each [Result]<>null,
                                each [Page=[Page]+1, Result=try GD2([Page]+1,x) otherwise null, year=x]                                
                            )
in
    Source

fnInnerLoop

(year as number, fnXfunction)=>
              let
                 Source = List.Generate(
                             ()=>
                                 [Page=1, Year=year],
                                each [Year]<=Max_Year,
                                each [Page=1, Year=[Year]+1],
                                each fnXfunction([Year],[Page])
                                                               
                            )
in
    Source

fnOuterLoop

let
    Source = fnOuterLoop(1980,fnInnerLoop),
    #"Converted to Table" = Table.FromList(Source, Splitter.SplitByNothing(), null, null, ExtraValues.Error),
    #"Expanded Column1" = Table.ExpandListColumn(#"Converted to Table", "Column1"),
    #"Expanded Column2" = Table.ExpandRecordColumn(#"Expanded Column1", "Column1", {"Page", "Result", "year"}, {"Column1.Page", "Column1.Result", "Column1.year"})
in
    #"Expanded Column2"

Output

The main takeaway from this exercise is that if the Iteration Termination clause is not desired to be mentioned by brute force it must come dynamically from another query (Max_Year in this case, thanks to d_gosbell ).