Add in checking for new files once list is depleted

commoncrawl_local_to_share_move.ps1

@@ -7,76 +7,74 @@ $assemblyPath = "C:\Program Files (x86)\WinSCP\WinSCPnet.dll"
 # Specify the path to winscp.exe
 $winscpPath = "C:\Program Files (x86)\WinSCP\winscp.exe"
 
-# Create a session instance
-$sessionType = [WinSCP.Session].FullName
-$session = New-Object $sessionType
+# Specify the local directory path
+$localPath = "E:\CommonCrawl"
 
-# Set up session options
-$sessionOptionsType = [WinSCP.SessionOptions].FullName
-$sessionOptions = New-Object $sessionOptionsType
+# Specify the remote directory path
+$remotePath = "/mnt/user/ArchiveTeam/CommonCrawl_Files/WAT_URLs/CC-MAIN-2023-14_March_April_2023"
 
-# Set properties for session options
-$sessionOptions.Protocol = [WinSCP.Protocol]::Sftp
-$sessionOptions.HostName = "192.168.1.248"
-$sessionOptions.UserName = "root"
-$sessionOptions.Password = "Tcft65rdx!"
-$sessionOptions.SshHostKeyFingerprint = "ssh-ed25519 255 xyTP5uPM3w6ebT7P7Mx1945YIYrK7NXWlZNNOGt2geY"
+while ($true) {
+    $session = $null
 
-try
-{
-    # Set the ExecutablePath property
-    $session.GetType().GetProperty("ExecutablePath").SetValue($session, $winscpPath, $null)
+    try {
+        # Create a session instance
+        $sessionType = [WinSCP.Session].FullName
+        $session = New-Object $sessionType
 
-    # Connect
-    $session.Open($sessionOptions)
+        # Set up session options
+        $sessionOptionsType = [WinSCP.SessionOptions].FullName
+        $sessionOptions = New-Object $sessionOptionsType
 
-    # Specify the local directory path
-    $localPath = "E:\CommonCrawl"
+        # Set properties for session options
+        $sessionOptions.Protocol = [WinSCP.Protocol]::Sftp
+        $sessionOptions.HostName = "192.168.1.248"
+        $sessionOptions.UserName = "root"
+        $sessionOptions.Password = "Tcft65rdx!"
+        $sessionOptions.SshHostKeyFingerprint = "ssh-ed25519 255 xyTP5uPM3w6ebT7P7Mx1945YIYrK7NXWlZNNOGt2geY"
 
-    # Specify the remote directory path
-    $remotePath = "/mnt/user/ArchiveTeam/CommonCrawl_Files/WAT_URLs/CC-MAIN-2023-23_May_June_2023"
+        # Set the ExecutablePath property
+        $session.GetType().GetProperty("ExecutablePath").SetValue($session, $winscpPath, $null)
 
-    # Get a list of local files matching the pattern "_urls.txt.gz"
-    $localFiles = Get-ChildItem -Path $localPath -Filter "*_urls.txt.gz" -File -Recurse
+        # Connect
+        $session.Open($sessionOptions)
 
-    # Shuffle the local files array
-    $localFiles = $localFiles | Get-Random -Count $localFiles.Count
+        # Get a list of local files matching the pattern "_urls.txt.gz"
+        $localFiles = Get-ChildItem -Path $localPath -Filter "*_urls.txt.gz" -File -Recurse
 
-    # Check if there are any files to upload
-    if ($localFiles.Count -eq 0)
-    {
-        Write-Host "No files found to upload."
-    }
-    else
-    {
-        # Iterate through each file and upload/delete it
-        foreach ($fileInfo in $localFiles)
-        {
-            # Construct the full local file path
-            $localFilePath = $fileInfo.FullName
+        # Shuffle the local files array
+        $localFiles = $localFiles | Get-Random -Count $localFiles.Count
 
-            # Construct the remote file path
-            $remoteFilePath = [WinSCP.RemotePath]::Combine($remotePath, $fileInfo.Name)
+        # Check if there are any files to upload
+        if ($localFiles.Count -eq 0) {
+            Write-Host "No files found to upload."
+            break  # Break out of the loop if no files are found
+        } else {
+            # Iterate through each file and upload/delete it
+            foreach ($fileInfo in $localFiles) {
+                # Construct the full local file path
+                $localFilePath = $fileInfo.FullName
 
-            # Upload the file
-            $transferResult = $session.PutFiles($localFilePath, $remoteFilePath, $False)
+                # Construct the remote file path
+                $remoteFilePath = [WinSCP.RemotePath]::Combine($remotePath, $fileInfo.Name)
 
-            # Check if the upload was successful
-            if ($transferResult.IsSuccess)
-            {
-                Write-Host "Upload successful: $localFilePath to $remoteFilePath"
+                # Upload the file
+                $transferResult = $session.PutFiles($localFilePath, $remoteFilePath, $False)
 
-                # Attempt to delete the local file
-                Remove-Item -Path $localFilePath -Force
-            }
-            else
-            {
-                Write-Host "Failed to upload file: $localFilePath"
+                # Check if the upload was successful
+                if ($transferResult.IsSuccess) {
+                    Write-Host "Upload successful: $localFilePath to $remoteFilePath"
+
+                    # Attempt to delete the local file
+                    Remove-Item -Path $localFilePath -Force
+                } else {
+                    Write-Host "Failed to upload file: $localFilePath"
+                }
             }
         }
+    } finally {
+        # Dispose of the session if it's not null
+        if ($session -ne $null) {
+            $session.Dispose()
+        }
     }
 }
-finally
-{
-    $session.Dispose()
-}