diff --git a/warc_wat_url_processor.py b/warc_wat_url_processor.py index cedf98c..089f8a5 100644 --- a/warc_wat_url_processor.py +++ b/warc_wat_url_processor.py @@ -102,7 +102,15 @@ def main(): futures = [executor.submit(download_and_process_file, url) for url in urls] # Wait for all downloads and processing to complete before starting the next iteration - as_completed(futures) + completed_futures, _ = wait(futures) + + # Process results from completed futures + for completed_future in completed_futures: + try: + result = completed_future.result() + # Process the result if needed + except Exception as e: + print(f"Error in processing future: {e}") if __name__ == "__main__": main()