# Specify the full path to WinSCPnet.dll
$assemblyPath = "C:\Program Files (x86)\WinSCP\WinSCPnet.dll"

# Load WinSCP .NET assembly using [System.Reflection.Assembly]::LoadFrom
[Reflection.Assembly]::LoadFrom($assemblyPath) | Out-Null

# Specify the path to winscp.exe
$winscpPath = "C:\Program Files (x86)\WinSCP\winscp.exe"

# Create a session instance
$sessionType = [WinSCP.Session].FullName
$session = New-Object $sessionType

# Set up session options
$sessionOptionsType = [WinSCP.SessionOptions].FullName
$sessionOptions = New-Object $sessionOptionsType

# Set properties for session options
$sessionOptions.Protocol = [WinSCP.Protocol]::Sftp
$sessionOptions.HostName = "65.109.140.15"
$sessionOptions.UserName = "root"
$sessionOptions.Password = "Tcft65rdx!"
$sessionOptions.SshHostKeyFingerprint = "ssh-ed25519 255 UeJ2ZrZQte4L0Inyewiu0BJ/PtPf+pc3OxF01xExWes"

# Specify the path for the local record file
$recordFilePath = "E:\CommonCrawl\transferred_files.txt"

try
{
    # Set the ExecutablePath property
    $session.GetType().GetProperty("ExecutablePath").SetValue($session, $winscpPath, $null)

    # Connect
    $session.Open($sessionOptions)

    # Specify the remote directory path
    $remotePath = "/opt/"

    # Check if the record file exists, if not, create it
    if (-not (Test-Path $recordFilePath)) {
        New-Item -ItemType File -Path $recordFilePath | Out-Null
    }

    do
    {
        # Get a list of files matching the pattern "_urls.txt.zst"
        $remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.zst", [WinSCP.EnumerationOptions]::AllDirectories)

        # Check if there are any files to download
        if ($remoteFiles.Count -eq 0)
        {
            Write-Host "No files found to download."
            break
        }

        # Iterate through each file and download/delete it
        foreach ($fileInfo in $remoteFiles)
        {
            # Check if the file has been transferred before
            if ((Get-Content $recordFilePath) -contains $fileInfo.FullName) {
                Write-Host "File already transferred: $($fileInfo.FullName)"
                continue
            }

            # Construct the full remote file path
            $remoteFilePath = [WinSCP.RemotePath]::Combine($remotePath, $fileInfo.FullName)

            # Construct the local file path
            $localFilePath = [System.IO.Path]::Combine("E:\CommonCrawl", $fileInfo.Name)

            # Download the file
            $transferResult = $session.GetFiles($remoteFilePath, $localFilePath, $True)

            # Check if the download was successful
            if ($transferResult.IsSuccess)
            {
                Write-Host "Download successful: $localFilePath"

                # Record the transferred file
                Add-Content -Path $recordFilePath -Value $fileInfo.FullName

                # Attempt to delete the remote file
                $removalResult = $session.RemoveFiles($remoteFilePath)

                # Check if the removal was successful
                if ($removalResult.IsSuccess)
                {
                    Write-Host "Deletion successful: $remoteFilePath"
                }
                else
                {
                    Write-Host "Failed to delete remote file: $remoteFilePath"
                }
            }
            else
            {
                Write-Host "Failed to download file: $remoteFilePath"
            }
        }

        # Get a new list of files after downloading
        $remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.zst", [WinSCP.EnumerationOptions]::AllDirectories)
    } while ($remoteFiles.Count -gt 0)
}
finally
{
    $session.Dispose()
}