Added in Lock file support

commoncrawl_transfer.ps1

@@ -54,6 +54,16 @@ try
             # Construct the local file path
             $localFilePath = [System.IO.Path]::Combine("E:\CommonCrawl", $fileInfo.Name)
 
+            # Check if lock file exists
+            $lockFilePath = $remoteFilePath + ".lock"
+            if ($session.FileExists($lockFilePath)) {
+                Write-Host "Lock file exists. Skipping file: $remoteFilePath"
+                continue
+            }
+
+            # Create lock file
+            $session.PutFiles("/dev/null", $lockFilePath).Check()
+
             # Download the file
             $transferResult = $session.GetFiles($remoteFilePath, $localFilePath, $True)
 
@@ -79,6 +89,9 @@ try
             {
                 Write-Host "Failed to download file: $remoteFilePath"
             }
+
+            # Remove lock file
+            $session.RemoveFiles($lockFilePath).Check()
         }
 
         # Get a new list of files after downloading