# Specify the full path to WinSCPnet.dll $assemblyPath = "C:\Program Files (x86)\WinSCP\WinSCPnet.dll" # Load WinSCP .NET assembly using [System.Reflection.Assembly]::LoadFrom [Reflection.Assembly]::LoadFrom($assemblyPath) | Out-Null # Specify the path to winscp.exe $winscpPath = "C:\Program Files (x86)\WinSCP\winscp.exe" # Create a session instance $sessionType = [WinSCP.Session].FullName $session = New-Object $sessionType # Set up session options $sessionOptionsType = [WinSCP.SessionOptions].FullName $sessionOptions = New-Object $sessionOptionsType # Set properties for session options $sessionOptions.Protocol = [WinSCP.Protocol]::Sftp $sessionOptions.HostName = "37.27.11.121" $sessionOptions.UserName = "root" $sessionOptions.Password = "Tcft65rdx!" $sessionOptions.SshHostKeyFingerprint = "ssh-ed25519 255 lM6ZIlmihQWkYz3iJONHYEkUPeJaUMUNOx7Av9CftuY" try { # Set the ExecutablePath property $session.GetType().GetProperty("ExecutablePath").SetValue($session, $winscpPath, $null) # Connect $session.Open($sessionOptions) # Specify the remote directory path $remotePath = "/opt/commoncrawl/" do { # Get a list of files matching the pattern "_urls.txt.gz" $remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.gz", [WinSCP.EnumerationOptions]::AllDirectories) # Check if there are any files to download if ($remoteFiles.Count -eq 0) { Write-Host "No files found to download." break } # Iterate through each file and download/delete it foreach ($fileInfo in $remoteFiles) { # Construct the full remote file path $remoteFilePath = [WinSCP.RemotePath]::Combine($remotePath, $fileInfo.FullName) # Construct the local file path $localFilePath = [System.IO.Path]::Combine("E:\CommonCrawl", $fileInfo.Name) # Download the file $transferResult = $session.GetFiles($remoteFilePath, $localFilePath, $True) # Check if the download was successful if ($transferResult.IsSuccess) { Write-Host "Download successful: $localFilePath" # Attempt to delete the remote file $removalResult = $session.RemoveFiles($remoteFilePath) # Check if the removal was successful if ($removalResult.IsSuccess) { Write-Host "Deletion successful: $remoteFilePath" } else { Write-Host "Failed to delete remote file: $remoteFilePath" } } else { Write-Host "Failed to download file: $remoteFilePath" } } # Get a new list of files after downloading $remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.gz", [WinSCP.EnumerationOptions]::AllDirectories) } while ($remoteFiles.Count -gt 0) } finally { $session.Dispose() }