Pass File list to notebook from copy data in a pipeline

This function will archive folders on the SFTP folder itself:
def sftp_archive_files(
        creds: dict
        ,src_path: str
        ,files: list
        ,archive_path: str
        ) -> Tuple [int, int, str]:
    """
    Connect to SFTP server and remove previously processed files using list of file names.

    Parameters:
        creds                      = Dictionary object that contains the endpoint, username, and password for this sftp site
        src_path                   = Path on sftp to search
        files                      = List of files (from previous handling function)
        archive_path               = Path to move file to on SFTP server
   
    Return:
        files_touched              = Number of files touched during the process
        files_moved                = Number of files moved on the SFTP server
        message                    = Message summarizing the action(s) taken
   
    Example usage:
        file_touched, files_deleted, msg = sftp_archive_files(sftp_creds, '/ven-directdivision', ['File1.txt','File2.txt'], '/ven-directdivision/archive')
    """

    # Connect to the SFTP server using paramiko
    transport = paramiko.Transport((creds['endpoint'],creds['port']))
    transport.connect(username=creds['username'], password=creds['password'])

    files_touched = 0
    files_moved = 0

    with paramiko.SFTPClient.from_transport(transport) as sftp:
        sftp.chdir(src_path)
       
        for file in files:
            files_touched += 1

            file_name = file
            file_path = f"{src_path}/{file_name}"

            archive_full_path = f"{archive_path}/{file_name}"
            print(f'Moving: {file_path} to {archive_full_path}')
           
            sftp.rename(file_path, archive_full_path)

            files_moved += 1

    transport.close()

    msg = f'Files touched: {files_touched}, Files moved: {files_moved}'
   
    return(files_touched, files_moved, msg)

    with paramiko.SFTPClient.from_transport(transport) as sftp:
        sftp.chdir(src_path)
        files = sftp.listdir_attr()

        for file_attr in files:
            files_touched += 1

            file_name = file_attr.filename
           
            # Check if the file matches the mask and date conditions
            if filename_mask in file_name:
                file_path = f"{src_path}/{file_name}"
                print(f'Downloading file: {file_path}...')

                # download file into memory, put file into dataframe, append dataframe to list of dataframes
                df = sftp_download_file(sftp, file_path, src_file_delim, src_file_encoding, src_file_head_row, transport)
                df['FileName'] = file_name
                lst_df.append(df)
                lst_files.append(file_name)
               
                files_downloaded += 1
               
       
        # Combine all DataFrames into a single result DataFrame
        if lst_df:
            df_result = pd.concat(lst_df, ignore_index=True)
            print("All files successfully combined into a single DataFrame.")
        else:
            df_result = pd.DataFrame()
            print("No files matched the conditions.")

    transport.close()
    return(files_touched, files_downloaded, df_result, lst_files)

Hopefully this helps!  We are using these functions to ingest hundreds of files daily.