From 7eafadc2fe99773a62424d5b63eb75c655c1387a Mon Sep 17 00:00:00 2001
From: datechnoman <datechnoman@hotmail.com>
Date: Sat, 27 Jan 2024 02:24:36 +0000
Subject: [PATCH] Update to zst file extension

---
 commoncrawl_transfer.ps1 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/commoncrawl_transfer.ps1 b/commoncrawl_transfer.ps1
index 5568b2c..f91ec68 100644
--- a/commoncrawl_transfer.ps1
+++ b/commoncrawl_transfer.ps1
@@ -35,8 +35,8 @@ try
 
     do
     {
-        # Get a list of files matching the pattern "_urls.txt.gz"
-        $remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.gz", [WinSCP.EnumerationOptions]::AllDirectories)
+        # Get a list of files matching the pattern "_urls.txt.zst"
+        $remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.zst", [WinSCP.EnumerationOptions]::AllDirectories)
 
         # Check if there are any files to download
         if ($remoteFiles.Count -eq 0)
@@ -82,7 +82,7 @@ try
         }
 
         # Get a new list of files after downloading
-        $remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.gz", [WinSCP.EnumerationOptions]::AllDirectories)
+        $remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.zst", [WinSCP.EnumerationOptions]::AllDirectories)
     } while ($remoteFiles.Count -gt 0)
 }
 finally