diff --git a/urlextractor_archiveteam.sh b/urlextractor_archiveteam.sh index 0c4c06a..0ea7c28 100644 --- a/urlextractor_archiveteam.sh +++ b/urlextractor_archiveteam.sh @@ -12,10 +12,7 @@ export -f gzip_file for file in "$directory"/*_urls.txt; do filename=$(basename "$file") grep -E "http(s)?://(www\.)?mediafire.com" "$file" | sort -u >> "/opt/commoncrawl/export/mediafire_urls.txt" - grep "https://t.me/" "$file" | sort -u >> "/opt/commoncrawl/export/t.me_urls.txt" - grep "https://telegram.me/" "$file" | sort -u >> "/opt/commoncrawl/export/telegram.me_urls.txt" grep -E "http(s)?://(www\.)?i.imgur.com" "$file" | sort -u >> "/opt/commoncrawl/export/imgur_urls.txt" - grep "sitemap.xml" "$file" | sort -u >> "/opt/commoncrawl/export/sitemap_urls.txt" grep "https://cdn.discordapp.com/" "$file" | sort -u >> "/opt/commoncrawl/export/discord_urls.txt" if [[ $filename != "mediafire_urls.txt" && $filename != "t.me_urls.txt" && $filename != "telegram.me_urls.txt" && $filename != "sitemap_urls.txt" ]]; then @@ -24,5 +21,4 @@ for file in "$directory"/*_urls.txt; do done # Wait for all gzip processes to finish -wait - +wait \ No newline at end of file