From 805b133dbaf5b0adc79eecbfe270a654ced69e85 Mon Sep 17 00:00:00 2001 From: datechnoman Date: Tue, 5 Dec 2023 10:20:23 +0000 Subject: [PATCH] Update multithread_script_5.0.py --- multithread_script_5.0.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multithread_script_5.0.py b/multithread_script_5.0.py index 938eb5f..b9e4717 100644 --- a/multithread_script_5.0.py +++ b/multithread_script_5.0.py @@ -43,7 +43,7 @@ def extract_urls_from_directory(directory_path): # Create a multiprocessing Pool with the number of processes # based on the available CPU cores - pool = Pool(processes=16) + pool = Pool(processes=15) # Map the file processing function to the list of files pool.map(process_file, [os.path.join(directory_path, filename) for filename in file_list if filename.endswith('.wat.gz')]) @@ -60,7 +60,7 @@ with open('urls_to_download.txt', 'r') as file: urls = [url.strip() for url in urls] # Define the batch size -batch_size = 100 +batch_size = 90 # Define the concurrency level (number of download processes running concurrently) concurrency_level = 8