ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update urlextractor_archiveteam.sh

Browse Source
This commit is contained in:
datechnoman 2023-12-10 05:10:10 +00:00
parent 6a90fc7f5e
commit d97491b4f0
1 changed files with 9 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
urlextractor_archiveteam.sh
View File

@ -32,6 +32,15 @@ for file in "$directory"/*_urls.txt; do
if [[ $filename != "mediafire_urls.txt" && $filename != "t.me_urls.txt" && $filename != "telegram.me_urls.txt" && $filename != "sitemap_urls.txt" ]]; then if [[ $filename != "mediafire_urls.txt" && $filename != "t.me_urls.txt" && $filename != "telegram.me_urls.txt" && $filename != "sitemap_urls.txt" ]]; then
parallel gzip_file ::: "$file" & parallel gzip_file ::: "$file" &
# Remove the line containing the filename (without "_urls.txt") from urls_to_download.txt
if sed -i "/$(echo "$filename" | sed 's/_urls.txt//')/d" "/opt/commoncrawl/urls_to_download.txt"; then
echo "File $(echo "$filename" | sed 's/_urls.txt//') has been successfully removed from urls_to_download.txt"
remaining_count=$(wc -l "/opt/commoncrawl/urls_to_download.txt" | awk '{print $1}')
echo "WAT files remaining to be processed $remaining_count"
else
echo "Failed to remove $filename from urls_to_download.txt"
fi
fi fi
done done