Added in Lock file support

This commit is contained in:
datechnoman 2024-01-27 02:42:11 +00:00
parent 674a7ae450
commit 488571b4c0

View File

@ -54,6 +54,16 @@ try
# Construct the local file path # Construct the local file path
$localFilePath = [System.IO.Path]::Combine("E:\CommonCrawl", $fileInfo.Name) $localFilePath = [System.IO.Path]::Combine("E:\CommonCrawl", $fileInfo.Name)
# Check if lock file exists
$lockFilePath = $remoteFilePath + ".lock"
if ($session.FileExists($lockFilePath)) {
Write-Host "Lock file exists. Skipping file: $remoteFilePath"
continue
}
# Create lock file
$session.PutFiles("/dev/null", $lockFilePath).Check()
# Download the file # Download the file
$transferResult = $session.GetFiles($remoteFilePath, $localFilePath, $True) $transferResult = $session.GetFiles($remoteFilePath, $localFilePath, $True)
@ -79,6 +89,9 @@ try
{ {
Write-Host "Failed to download file: $remoteFilePath" Write-Host "Failed to download file: $remoteFilePath"
} }
# Remove lock file
$session.RemoveFiles($lockFilePath).Check()
} }
# Get a new list of files after downloading # Get a new list of files after downloading