diff --git a/commoncrawl_transfer.ps1 b/commoncrawl_transfer.ps1 index f91ec68..fc76bde 100644 --- a/commoncrawl_transfer.ps1 +++ b/commoncrawl_transfer.ps1 @@ -54,6 +54,16 @@ try # Construct the local file path $localFilePath = [System.IO.Path]::Combine("E:\CommonCrawl", $fileInfo.Name) + # Check if lock file exists + $lockFilePath = $remoteFilePath + ".lock" + if ($session.FileExists($lockFilePath)) { + Write-Host "Lock file exists. Skipping file: $remoteFilePath" + continue + } + + # Create lock file + $session.PutFiles("/dev/null", $lockFilePath).Check() + # Download the file $transferResult = $session.GetFiles($remoteFilePath, $localFilePath, $True) @@ -79,6 +89,9 @@ try { Write-Host "Failed to download file: $remoteFilePath" } + + # Remove lock file + $session.RemoveFiles($lockFilePath).Check() } # Get a new list of files after downloading