ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update multithread_script_5.0.py

Browse Source
This commit is contained in:
datechnoman 2023-12-04 12:00:27 +00:00
parent d326a35340
commit 00c0678135
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
multithread_script_5.0.py
View File

@ -43,7 +43,7 @@ def extract_urls_from_directory(directory_path):
# Create a multiprocessing Pool with the number of processes # Create a multiprocessing Pool with the number of processes
# based on the available CPU cores # based on the available CPU cores
pool = Pool(processes=12) pool = Pool(processes=16)
# Map the file processing function to the list of files # Map the file processing function to the list of files
pool.map(process_file, [os.path.join(directory_path, filename) for filename in file_list if filename.endswith('.wat.gz')]) pool.map(process_file, [os.path.join(directory_path, filename) for filename in file_list if filename.endswith('.wat.gz')])