ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update multithread_script_5.0.py

Browse Source
This commit is contained in:
datechnoman 2023-12-06 10:59:41 +00:00
parent adc15ec579
commit 6a90fc7f5e
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
multithread_script_5.0.py
View File

@ -43,7 +43,7 @@ def extract_urls_from_directory(directory_path):
# Create a multiprocessing Pool with the number of processes # Create a multiprocessing Pool with the number of processes
# based on the available CPU cores # based on the available CPU cores
pool = Pool(processes=15) pool = Pool(processes=7)
# Map the file processing function to the list of files # Map the file processing function to the list of files
pool.map(process_file, [os.path.join(directory_path, filename) for filename in file_list if filename.endswith('.wat.gz')]) pool.map(process_file, [os.path.join(directory_path, filename) for filename in file_list if filename.endswith('.wat.gz')])
@ -60,7 +60,7 @@ with open('urls_to_download.txt', 'r') as file:
urls = [url.strip() for url in urls] urls = [url.strip() for url in urls]
# Define the batch size # Define the batch size
batch_size = 90 batch_size = 84
# Define the concurrency level (number of download processes running concurrently) # Define the concurrency level (number of download processes running concurrently)
concurrency_level = 8 concurrency_level = 8