diff --git a/warc_wat_url_processor.py b/warc_wat_url_processor.py index 3cad8a8..6aade3b 100644 --- a/warc_wat_url_processor.py +++ b/warc_wat_url_processor.py @@ -3,7 +3,7 @@ import os import gzip import zstandard as zstd import re -import traceback # Import traceback module +import traceback from multiprocessing import Pool def extract_urls_from_file(file_path): @@ -11,7 +11,6 @@ def extract_urls_from_file(file_path): try: with gzip.open(file_path, 'rt', encoding='latin-1') as file: for line in file: - # Extract URLs using regular expression url_pattern = re.compile(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+') line_urls = re.findall(url_pattern, line) urls.extend(line_urls) @@ -20,7 +19,7 @@ def extract_urls_from_file(file_path): except Exception as e: print(f"An unexpected error occurred while processing '{file_path}': {e}") print("Full traceback:") - traceback.print_exc() # Print the full traceback + traceback.print_exc() return urls @@ -39,9 +38,9 @@ def process_file(file_path): output_file.write('\n'.join(urls)) print(f"URLs written to {output_file_path}") - # Compress the output file using zstd with compression level -10 + # Compress the output file using zstd with compression level -18 with open(output_file_path, 'rb') as input_file, open(output_file_path + '.zst', 'wb') as output_zstd_file: - cctx = zstd.ZstdCompressor(level=-10) + cctx = zstd.ZstdCompressor(level=18) output_zstd_file.write(cctx.compress(input_file.read())) print(f"Compressed file saved as '{output_file_path}.zst'") @@ -50,53 +49,29 @@ def process_file(file_path): print(f"Original file removed: {file_path}") def extract_urls_from_directory(directory_path): - # Get the list of files in the directory and sort them file_list = sorted(os.listdir(directory_path)) - - # Create a multiprocessing Pool with the number of processes - # based on the available CPU cores pool = Pool(processes=7) - - # Map the file processing function to the list of files pool.map(process_file, [os.path.join(directory_path, filename) for filename in file_list if filename.endswith('.warc.gz')]) - - # Close the pool to free up resources pool.close() pool.join() -# Read the URLs from the file with open('urls_to_download.txt', 'r') as file: urls = file.readlines() -# Remove any leading/trailing whitespace from the URLs urls = [url.strip() for url in urls] -# Define the batch size batch_size = 48 - -# Define the concurrency level (number of download processes running concurrently) concurrency_level = 4 - -# Split the URLs into batches batches = [urls[i:i+batch_size] for i in range(0, len(urls), batch_size)] -# Iterate over the batches and download URLs for batch in batches: - # Create a multiprocessing Pool with the specified concurrency level pool = Pool(processes=concurrency_level) for url in batch: - # Create the command to download the URL using axel with 3 connections command = f'axel -n 4 {url}' - - # Start the subprocess in the background pool.apply_async(subprocess.run, args=(command,), kwds={'shell': True}) - # Close the pool to indicate that no more tasks will be added pool.close() - - # Wait for all processes in the pool to finish pool.join() -# Extract URLs from the downloaded files extract_urls_from_directory(os.getcwd())