ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update to zst extension

Browse Source
This commit is contained in:
datechnoman 2024-01-27 02:26:27 +00:00
parent 7eafadc2fe
commit 674a7ae450
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
commoncrawl_local_to_share_move.ps1
View File

@ -38,8 +38,8 @@ while ($true) {
# Connect
$session.Open($sessionOptions)
# Get a list of local files matching the pattern "_urls.txt.gz"
$localFiles = Get-ChildItem -Path $localPath -Filter "*_urls.txt.gz" -File -Recurse
# Get a list of local files matching the pattern "_urls.txt.zst"
$localFiles = Get-ChildItem -Path $localPath -Filter "*_urls.txt.zst" -File -Recurse
# Shuffle the local files array
$localFiles = $localFiles | Get-Random -Count $localFiles.Count