ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update to zst file extension

Browse Source
This commit is contained in:
datechnoman 2024-01-27 02:24:36 +00:00
parent 8244b9241a
commit 7eafadc2fe
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
commoncrawl_transfer.ps1
View File

@ -35,8 +35,8 @@ try
do do
{ {
# Get a list of files matching the pattern "_urls.txt.gz" # Get a list of files matching the pattern "_urls.txt.zst"
$remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.gz", [WinSCP.EnumerationOptions]::AllDirectories) $remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.zst", [WinSCP.EnumerationOptions]::AllDirectories)
# Check if there are any files to download # Check if there are any files to download
if ($remoteFiles.Count -eq 0) if ($remoteFiles.Count -eq 0)
@ -82,7 +82,7 @@ try
} }
# Get a new list of files after downloading # Get a new list of files after downloading
$remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.gz", [WinSCP.EnumerationOptions]::AllDirectories) $remoteFiles = $session.EnumerateRemoteFiles($remotePath, "*_urls.txt.zst", [WinSCP.EnumerationOptions]::AllDirectories)
} while ($remoteFiles.Count -gt 0) } while ($remoteFiles.Count -gt 0)
} }
finally finally