CommonCrawl_URL_Processor/commoncrawl_transfer.ps1

# Specify the full path to WinSCPnet.dll
$assemblyPath = "C:\Program Files (x86)\WinSCP\WinSCPnet.dll"

# Load WinSCP .NET assembly using [System.Reflection.Assembly]::LoadFrom
[Reflection.Assembly]::LoadFrom($assemblyPath) | Out-Null

# Specify the path to winscp.exe
$winscpPath = "C:\Program Files (x86)\WinSCP\winscp.exe"

# Create a session instance
$sessionType = [WinSCP.Session].FullName
$session = New-Object $sessionType

# Set up session options
$sessionOptionsType = [WinSCP.SessionOptions].FullName
$sessionOptions = New-Object $sessionOptionsType

# Set properties for session options
$sessionOptions.Protocol = [WinSCP.Protocol]::Sftp
$sessionOptions.HostName = "37.27.11.121"
$sessionOptions.UserName = "root"
$sessionOptions.Password = "Tcft65rdx!"
$sessionOptions.SshHostKeyFingerprint = "ssh-ed25519 255 lM6ZIlmihQWkYz3iJONHYEkUPeJaUMUNOx7Av9CftuY"

try
{
    # Set the ExecutablePath property
    $session.GetType().GetProperty("ExecutablePath").SetValue($session, $winscpPath, $null)

    # Connect
    $session.Open($sessionOptions)

    # Specify the remote directory path
    $remotePath = "/opt/commoncrawl/"

    do
    {
        # Get a list of files matching the pattern "_urls.txt.zst"
        $remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.zst", [WinSCP.EnumerationOptions]::AllDirectories)

        # Check if there are any files to download
        if ($remoteFiles.Count -eq 0)
        {
            Write-Host "No files found to download."
            break
        }

        # Iterate through each file and download/delete it
        foreach ($fileInfo in $remoteFiles)
        {
            # Construct the full remote file path
            $remoteFilePath = [WinSCP.RemotePath]::Combine($remotePath, $fileInfo.FullName)

            # Construct the local file path
            $localFilePath = [System.IO.Path]::Combine("E:\CommonCrawl", $fileInfo.Name)

            # Check if lock file exists
            $lockFilePath = $remoteFilePath + ".lock"
            if ($session.FileExists($lockFilePath)) {
                Write-Host "Lock file exists. Skipping file: $remoteFilePath"
                continue
            }

            # Create lock file
            $session.PutFiles("/dev/null", $lockFilePath).Check()

            # Download the file
            $transferResult = $session.GetFiles($remoteFilePath, $localFilePath, $True)

            # Check if the download was successful
            if ($transferResult.IsSuccess)
            {
                Write-Host "Download successful: $localFilePath"

                # Attempt to delete the remote file
                $removalResult = $session.RemoveFiles($remoteFilePath)

                # Check if the removal was successful
                if ($removalResult.IsSuccess)
                {
                    Write-Host "Deletion successful: $remoteFilePath"
                }
                else
                {
                    Write-Host "Failed to delete remote file: $remoteFilePath"
                }
            }
            else
            {
                Write-Host "Failed to download file: $remoteFilePath"
            }

            # Remove lock file
            $session.RemoveFiles($lockFilePath).Check()
        }

        # Get a new list of files after downloading
        $remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.zst", [WinSCP.EnumerationOptions]::AllDirectories)
    } while ($remoteFiles.Count -gt 0)
}
finally
{
    $session.Dispose()
}