ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated error processing for zstd compression errors

Browse Source
This commit is contained in:
datechnoman 2024-01-25 01:33:11 +00:00
parent 097ec759ab
commit 2bee6db85a
1 changed files with 10 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
warc_wat_url_processor.py
View File

@ -42,12 +42,15 @@ def process_file(file_path):
compressed_file_path = f'{output_file_path}.zst' compressed_file_path = f'{output_file_path}.zst'
command_compress = f'zstd -T0 -12 --long {output_file_path} -o {compressed_file_path}' command_compress = f'zstd -T0 -12 --long {output_file_path} -o {compressed_file_path}'
try:
# Run the compression command synchronously and wait for it to complete # Run the compression command synchronously and wait for it to complete
result = subprocess.run(command_compress, shell=True, check=True) result = subprocess.run(command_compress, shell=True, check=True)
if result.returncode == 0: if result.returncode == 0:
print(f"Compressed file saved as '{compressed_file_path}'") print(f"Compressed file saved as '{compressed_file_path}'")
else: else:
print(f"Compression failed for '{output_file_path}'") print(f"Compression failed for '{output_file_path}'")
except subprocess.CalledProcessError as compression_error:
print(f"Compression failed for '{output_file_path}': {compression_error}")
# Remove the original gzipped file # Remove the original gzipped file
os.remove(file_path) os.remove(file_path)
@ -92,7 +95,7 @@ def main():
urls = [url.strip() for url in urls] urls = [url.strip() for url in urls]
download_concurrency_level = 4 download_concurrency_level = 40
# Start downloading and processing files in parallel # Start downloading and processing files in parallel
with ProcessPoolExecutor(max_workers=download_concurrency_level) as executor: with ProcessPoolExecutor(max_workers=download_concurrency_level) as executor: