ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated with higher compression

Browse Source
This commit is contained in:
datechnoman 2024-01-20 11:25:59 +00:00
parent d1cfd0178f
commit 78f6b69cdf
1 changed files with 4 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
warc_wat_url_processor.py
View File

@ -3,7 +3,7 @@ import os
import gzip import gzip
import zstandard as zstd import zstandard as zstd
import re import re
import traceback # Import traceback module import traceback
from multiprocessing import Pool from multiprocessing import Pool
def extract_urls_from_file(file_path): def extract_urls_from_file(file_path):
@ -11,7 +11,6 @@ def extract_urls_from_file(file_path):
try: try:
with gzip.open(file_path, 'rt', encoding='latin-1') as file: with gzip.open(file_path, 'rt', encoding='latin-1') as file:
for line in file: for line in file:
# Extract URLs using regular expression
url_pattern = re.compile(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+') url_pattern = re.compile(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')
line_urls = re.findall(url_pattern, line) line_urls = re.findall(url_pattern, line)
urls.extend(line_urls) urls.extend(line_urls)
@ -20,7 +19,7 @@ def extract_urls_from_file(file_path):
except Exception as e: except Exception as e:
print(f"An unexpected error occurred while processing '{file_path}': {e}") print(f"An unexpected error occurred while processing '{file_path}': {e}")
print("Full traceback:") print("Full traceback:")
traceback.print_exc() # Print the full traceback traceback.print_exc()
return urls return urls
@ -39,9 +38,9 @@ def process_file(file_path):
output_file.write('\n'.join(urls)) output_file.write('\n'.join(urls))
print(f"URLs written to {output_file_path}") print(f"URLs written to {output_file_path}")
# Compress the output file using zstd with compression level -10 # Compress the output file using zstd with compression level -18
with open(output_file_path, 'rb') as input_file, open(output_file_path + '.zst', 'wb') as output_zstd_file: with open(output_file_path, 'rb') as input_file, open(output_file_path + '.zst', 'wb') as output_zstd_file:
cctx = zstd.ZstdCompressor(level=-10) cctx = zstd.ZstdCompressor(level=18)
output_zstd_file.write(cctx.compress(input_file.read())) output_zstd_file.write(cctx.compress(input_file.read()))
print(f"Compressed file saved as '{output_file_path}.zst'") print(f"Compressed file saved as '{output_file_path}.zst'")
@ -50,53 +49,29 @@ def process_file(file_path):
print(f"Original file removed: {file_path}") print(f"Original file removed: {file_path}")
def extract_urls_from_directory(directory_path): def extract_urls_from_directory(directory_path):
# Get the list of files in the directory and sort them
file_list = sorted(os.listdir(directory_path)) file_list = sorted(os.listdir(directory_path))
# Create a multiprocessing Pool with the number of processes
# based on the available CPU cores
pool = Pool(processes=7) pool = Pool(processes=7)
# Map the file processing function to the list of files
pool.map(process_file, [os.path.join(directory_path, filename) for filename in file_list if filename.endswith('.warc.gz')]) pool.map(process_file, [os.path.join(directory_path, filename) for filename in file_list if filename.endswith('.warc.gz')])
# Close the pool to free up resources
pool.close() pool.close()
pool.join() pool.join()
# Read the URLs from the file
with open('urls_to_download.txt', 'r') as file: with open('urls_to_download.txt', 'r') as file:
urls = file.readlines() urls = file.readlines()
# Remove any leading/trailing whitespace from the URLs
urls = [url.strip() for url in urls] urls = [url.strip() for url in urls]
# Define the batch size
batch_size = 48 batch_size = 48
# Define the concurrency level (number of download processes running concurrently)
concurrency_level = 4 concurrency_level = 4
# Split the URLs into batches
batches = [urls[i:i+batch_size] for i in range(0, len(urls), batch_size)] batches = [urls[i:i+batch_size] for i in range(0, len(urls), batch_size)]
# Iterate over the batches and download URLs
for batch in batches: for batch in batches:
# Create a multiprocessing Pool with the specified concurrency level
pool = Pool(processes=concurrency_level) pool = Pool(processes=concurrency_level)
for url in batch: for url in batch:
# Create the command to download the URL using axel with 3 connections
command = f'axel -n 4 {url}' command = f'axel -n 4 {url}'
# Start the subprocess in the background
pool.apply_async(subprocess.run, args=(command,), kwds={'shell': True}) pool.apply_async(subprocess.run, args=(command,), kwds={'shell': True})
# Close the pool to indicate that no more tasks will be added
pool.close() pool.close()
# Wait for all processes in the pool to finish
pool.join() pool.join()
# Extract URLs from the downloaded files
extract_urls_from_directory(os.getcwd()) extract_urls_from_directory(os.getcwd())