ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update urlextractor_archiveteam.sh

Browse Source
This commit is contained in:
datechnoman 2023-12-05 02:41:43 +00:00
parent d95927c230
commit bd939a62de
1 changed files with 1 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
urlextractor_archiveteam.sh
View File

@ -12,10 +12,7 @@ export -f gzip_file
for file in "$directory"/*_urls.txt; do for file in "$directory"/*_urls.txt; do
filename=$(basename "$file") filename=$(basename "$file")
grep -E "http(s)?://(www\.)?mediafire.com" "$file" | sort -u >> "/opt/commoncrawl/export/mediafire_urls.txt" grep -E "http(s)?://(www\.)?mediafire.com" "$file" | sort -u >> "/opt/commoncrawl/export/mediafire_urls.txt"
grep "https://t.me/" "$file" | sort -u >> "/opt/commoncrawl/export/t.me_urls.txt"
grep "https://telegram.me/" "$file" | sort -u >> "/opt/commoncrawl/export/telegram.me_urls.txt"
grep -E "http(s)?://(www\.)?i.imgur.com" "$file" | sort -u >> "/opt/commoncrawl/export/imgur_urls.txt" grep -E "http(s)?://(www\.)?i.imgur.com" "$file" | sort -u >> "/opt/commoncrawl/export/imgur_urls.txt"
grep "sitemap.xml" "$file" | sort -u >> "/opt/commoncrawl/export/sitemap_urls.txt"
grep "https://cdn.discordapp.com/" "$file" | sort -u >> "/opt/commoncrawl/export/discord_urls.txt" grep "https://cdn.discordapp.com/" "$file" | sort -u >> "/opt/commoncrawl/export/discord_urls.txt"
if [[ $filename != "mediafire_urls.txt" && $filename != "t.me_urls.txt" && $filename != "telegram.me_urls.txt" && $filename != "sitemap_urls.txt" ]]; then if [[ $filename != "mediafire_urls.txt" && $filename != "t.me_urls.txt" && $filename != "telegram.me_urls.txt" && $filename != "sitemap_urls.txt" ]]; then
@ -24,5 +21,4 @@ for file in "$directory"/*_urls.txt; do
done done
# Wait for all gzip processes to finish # Wait for all gzip processes to finish
wait wait