Update multithread_script_5.0.py

multithread_script_5.0.py

@@ -43,7 +43,7 @@ def extract_urls_from_directory(directory_path):
 
     # Create a multiprocessing Pool with the number of processes
     # based on the available CPU cores
-    pool = Pool(processes=16)
+    pool = Pool(processes=15)
 
     # Map the file processing function to the list of files
     pool.map(process_file, [os.path.join(directory_path, filename) for filename in file_list if filename.endswith('.wat.gz')])
@@ -60,7 +60,7 @@ with open('urls_to_download.txt', 'r') as file:
 urls = [url.strip() for url in urls]
 
 # Define the batch size
-batch_size = 100
+batch_size = 90
 
 # Define the concurrency level (number of download processes running concurrently)
 concurrency_level = 8