ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update warc_wat_url_processor.py

Browse Source
This commit is contained in:
datechnoman 2024-01-28 09:14:12 +00:00
parent d0fa7c84f4
commit b6a9c68140
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
warc_wat_url_processor.py
View File

@ -48,7 +48,6 @@ def process_file(file_path):
# Extract URLs from the gzipped file # Extract URLs from the gzipped file
urls = extract_urls_from_file(file_path) urls = extract_urls_from_file(file_path)
print(f"Extracted {len(urls)} URLs from {file_path}")
# Create the output file path with '_urls.txt' extension # Create the output file path with '_urls.txt' extension
output_file_path = os.path.splitext(file_path)[0] + '_urls.txt' output_file_path = os.path.splitext(file_path)[0] + '_urls.txt'
@ -121,13 +120,18 @@ def main():
download_concurrency_level = 40 download_concurrency_level = 40
with ProcessPoolExecutor(max_workers=download_concurrency_level) as executor: with ProcessPoolExecutor(max_workers=download_concurrency_level) as executor:
print("Submitting tasks to the ProcessPoolExecutor...")
futures = [executor.submit(download_and_process_file, url) for url in urls] futures = [executor.submit(download_and_process_file, url) for url in urls]
print(f"Submitted {len(futures)} tasks.")
print("Waiting for tasks to complete...")
completed_futures, _ = wait(futures) completed_futures, _ = wait(futures)
print(f"{len(completed_futures)} tasks completed.")
for completed_future in completed_futures: for completed_future in completed_futures:
try: try:
result = completed_future.result() result = completed_future.result()
print(f"Task result: {result}")
# Process the result if needed # Process the result if needed
except Exception as e: except Exception as e:
print(f"Error in processing future: {e}") print(f"Error in processing future: {e}")