ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update to zst file extension

Browse Source
This commit is contained in:
datechnoman 2024-01-27 02:24:36 +00:00
parent 8244b9241a
commit 7eafadc2fe
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
commoncrawl_transfer.ps1
View File

@ -35,8 +35,8 @@ try
do
{
# Get a list of files matching the pattern "_urls.txt.gz"
$remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.gz", [WinSCP.EnumerationOptions]::AllDirectories)
# Get a list of files matching the pattern "_urls.txt.zst"
$remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.zst", [WinSCP.EnumerationOptions]::AllDirectories)
# Check if there are any files to download
if ($remoteFiles.Count -eq 0)
@ -82,7 +82,7 @@ try
}
# Get a new list of files after downloading
$remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.gz", [WinSCP.EnumerationOptions]::AllDirectories)
$remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.zst", [WinSCP.EnumerationOptions]::AllDirectories)
} while ($remoteFiles.Count -gt 0)
}
finally