108 lines
3.6 KiB
PowerShell
108 lines
3.6 KiB
PowerShell
# Specify the full path to WinSCPnet.dll
|
|
$assemblyPath = "C:\Program Files (x86)\WinSCP\WinSCPnet.dll"
|
|
|
|
# Load WinSCP .NET assembly using [System.Reflection.Assembly]::LoadFrom
|
|
[Reflection.Assembly]::LoadFrom($assemblyPath) | Out-Null
|
|
|
|
# Specify the path to winscp.exe
|
|
$winscpPath = "C:\Program Files (x86)\WinSCP\winscp.exe"
|
|
|
|
# Create a session instance
|
|
$sessionType = [WinSCP.Session].FullName
|
|
$session = New-Object $sessionType
|
|
|
|
# Set up session options
|
|
$sessionOptionsType = [WinSCP.SessionOptions].FullName
|
|
$sessionOptions = New-Object $sessionOptionsType
|
|
|
|
# Set properties for session options
|
|
$sessionOptions.Protocol = [WinSCP.Protocol]::Sftp
|
|
$sessionOptions.HostName = "65.109.140.15"
|
|
$sessionOptions.UserName = "root"
|
|
$sessionOptions.Password = "Tcft65rdx!"
|
|
$sessionOptions.SshHostKeyFingerprint = "ssh-ed25519 255 UeJ2ZrZQte4L0Inyewiu0BJ/PtPf+pc3OxF01xExWes"
|
|
|
|
# Specify the path for the local record file
|
|
$recordFilePath = "E:\CommonCrawl\transferred_files.txt"
|
|
|
|
try
|
|
{
|
|
# Set the ExecutablePath property
|
|
$session.GetType().GetProperty("ExecutablePath").SetValue($session, $winscpPath, $null)
|
|
|
|
# Connect
|
|
$session.Open($sessionOptions)
|
|
|
|
# Specify the remote directory path
|
|
$remotePath = "/opt/"
|
|
|
|
# Check if the record file exists, if not, create it
|
|
if (-not (Test-Path $recordFilePath)) {
|
|
New-Item -ItemType File -Path $recordFilePath | Out-Null
|
|
}
|
|
|
|
do
|
|
{
|
|
# Get a list of files matching the pattern "_urls.txt.zst"
|
|
$remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.zst", [WinSCP.EnumerationOptions]::AllDirectories)
|
|
|
|
# Check if there are any files to download
|
|
if ($remoteFiles.Count -eq 0)
|
|
{
|
|
Write-Host "No files found to download."
|
|
break
|
|
}
|
|
|
|
# Iterate through each file and download/delete it
|
|
foreach ($fileInfo in $remoteFiles)
|
|
{
|
|
# Check if the file has been transferred before
|
|
if ((Get-Content $recordFilePath) -contains $fileInfo.FullName) {
|
|
Write-Host "File already transferred: $($fileInfo.FullName)"
|
|
continue
|
|
}
|
|
|
|
# Construct the full remote file path
|
|
$remoteFilePath = [WinSCP.RemotePath]::Combine($remotePath, $fileInfo.FullName)
|
|
|
|
# Construct the local file path
|
|
$localFilePath = [System.IO.Path]::Combine("E:\CommonCrawl", $fileInfo.Name)
|
|
|
|
# Download the file
|
|
$transferResult = $session.GetFiles($remoteFilePath, $localFilePath, $True)
|
|
|
|
# Check if the download was successful
|
|
if ($transferResult.IsSuccess)
|
|
{
|
|
Write-Host "Download successful: $localFilePath"
|
|
|
|
# Record the transferred file
|
|
Add-Content -Path $recordFilePath -Value $fileInfo.FullName
|
|
|
|
# Attempt to delete the remote file
|
|
$removalResult = $session.RemoveFiles($remoteFilePath)
|
|
|
|
# Check if the removal was successful
|
|
if ($removalResult.IsSuccess)
|
|
{
|
|
Write-Host "Deletion successful: $remoteFilePath"
|
|
}
|
|
else
|
|
{
|
|
Write-Host "Failed to delete remote file: $remoteFilePath"
|
|
}
|
|
}
|
|
else
|
|
{
|
|
Write-Host "Failed to download file: $remoteFilePath"
|
|
}
|
|
}
|
|
|
|
# Get a new list of files after downloading
|
|
$remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.zst", [WinSCP.EnumerationOptions]::AllDirectories)
|
|
} while ($remoteFiles.Count -gt 0)
|
|
}
|
|
finally
|
|
{
|
|
$session.Dispose()
|
|
} |