Updated with higher compression

warc_wat_url_processor.py

@@ -3,7 +3,7 @@ import os
 import gzip
 import zstandard as zstd
 import re
-import traceback  # Import traceback module
+import traceback
 from multiprocessing import Pool
 
 def extract_urls_from_file(file_path):
@@ -11,7 +11,6 @@ def extract_urls_from_file(file_path):
     try:
         with gzip.open(file_path, 'rt', encoding='latin-1') as file:
             for line in file:
-                # Extract URLs using regular expression
                 url_pattern = re.compile(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')
                 line_urls = re.findall(url_pattern, line)
                 urls.extend(line_urls)
@@ -20,7 +19,7 @@ def extract_urls_from_file(file_path):
     except Exception as e:
         print(f"An unexpected error occurred while processing '{file_path}': {e}")
         print("Full traceback:")
-        traceback.print_exc()  # Print the full traceback
+        traceback.print_exc()
     
     return urls
 
@@ -39,9 +38,9 @@ def process_file(file_path):
         output_file.write('\n'.join(urls))
     print(f"URLs written to {output_file_path}")
 
-    # Compress the output file using zstd with compression level -10
+    # Compress the output file using zstd with compression level -18
     with open(output_file_path, 'rb') as input_file, open(output_file_path + '.zst', 'wb') as output_zstd_file:
-        cctx = zstd.ZstdCompressor(level=-10)
+        cctx = zstd.ZstdCompressor(level=18)
         output_zstd_file.write(cctx.compress(input_file.read()))
     print(f"Compressed file saved as '{output_file_path}.zst'")
 
@@ -50,53 +49,29 @@ def process_file(file_path):
     print(f"Original file removed: {file_path}")
 
 def extract_urls_from_directory(directory_path):
-    # Get the list of files in the directory and sort them
     file_list = sorted(os.listdir(directory_path))
-
-    # Create a multiprocessing Pool with the number of processes
-    # based on the available CPU cores
     pool = Pool(processes=7)
-
-    # Map the file processing function to the list of files
     pool.map(process_file, [os.path.join(directory_path, filename) for filename in file_list if filename.endswith('.warc.gz')])
-
-    # Close the pool to free up resources
     pool.close()
     pool.join()
 
-# Read the URLs from the file
 with open('urls_to_download.txt', 'r') as file:
     urls = file.readlines()
 
-# Remove any leading/trailing whitespace from the URLs
 urls = [url.strip() for url in urls]
 
-# Define the batch size
 batch_size = 48
-
-# Define the concurrency level (number of download processes running concurrently)
 concurrency_level = 4
-
-# Split the URLs into batches
 batches = [urls[i:i+batch_size] for i in range(0, len(urls), batch_size)]
 
-# Iterate over the batches and download URLs
 for batch in batches:
-    # Create a multiprocessing Pool with the specified concurrency level
     pool = Pool(processes=concurrency_level)
 
     for url in batch:
-        # Create the command to download the URL using axel with 3 connections
         command = f'axel -n 4 {url}'
-
-        # Start the subprocess in the background
         pool.apply_async(subprocess.run, args=(command,), kwds={'shell': True})
 
-    # Close the pool to indicate that no more tasks will be added
     pool.close()
-
-    # Wait for all processes in the pool to finish
     pool.join()
 
-# Extract URLs from the downloaded files
 extract_urls_from_directory(os.getcwd())