ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update warc_wat_url_processor.py

Browse Source
This commit is contained in:
datechnoman 2024-02-08 00:39:09 +00:00
parent 486a68a796
commit 3151fce353
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
warc_wat_url_processor.py
View File

@ -117,7 +117,7 @@ def main():
urls = [url.strip() for url in urls] urls = [url.strip() for url in urls]
download_concurrency_level = 40 download_concurrency_level = 15
with ProcessPoolExecutor(max_workers=download_concurrency_level) as executor: with ProcessPoolExecutor(max_workers=download_concurrency_level) as executor:
print("Submitting tasks to the ProcessPoolExecutor...") print("Submitting tasks to the ProcessPoolExecutor...")