From fd9376cbe0b9dbf67e9685cf3f2081719bfac57e Mon Sep 17 00:00:00 2001 From: datechnoman Date: Tue, 19 Dec 2023 00:23:55 +0000 Subject: [PATCH] Updated to extract Pastebin URL's --- urlextractor_archiveteam.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/urlextractor_archiveteam.sh b/urlextractor_archiveteam.sh index 03bb0fd..5878834 100644 --- a/urlextractor_archiveteam.sh +++ b/urlextractor_archiveteam.sh @@ -28,6 +28,7 @@ for file in "$directory"/*_urls.txt; do filename=$(basename "$file") grep -E "http(s)?://(www\.)?mediafire.com" "$file" | sort -u >> "/opt/CommonCrawl_URL_Processor/export/mediafire_urls.txt" grep -E "http(s)?://(www\.)?i.imgur.com" "$file" | sort -u >> "/opt/CommonCrawl_URL_Processor/export/imgur_urls.txt" + grep -E "http(s)?://(www\.)?pastebin.com" "$file" | sort -u >> "/opt/CommonCrawl_URL_Processor/export/pastebin_urls.txt" grep "https://cdn.discordapp.com/" "$file" | sort -u >> "/opt/CommonCrawl_URL_Processor/export/discord_urls.txt" if [[ $filename != "mediafire_urls.txt" && $filename != "t.me_urls.txt" && $filename != "telegram.me_urls.txt" && $filename != "sitemap_urls.txt" ]]; then