ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated to use command line zstd

Browse Source
This commit is contained in:
datechnoman 2024-01-20 11:29:57 +00:00
parent 78f6b69cdf
commit e98f80aec4
1 changed files with 4 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
warc_wat_url_processor.py
View File

@ -1,7 +1,6 @@
import subprocess
import os
import gzip
import zstandard as zstd
import re
import traceback
from multiprocessing import Pool
@ -20,7 +19,7 @@ def extract_urls_from_file(file_path):
print(f"An unexpected error occurred while processing '{file_path}': {e}")
print("Full traceback:")
traceback.print_exc()
return urls
def process_file(file_path):
@ -38,10 +37,9 @@ def process_file(file_path):
output_file.write('\n'.join(urls))
print(f"URLs written to {output_file_path}")
# Compress the output file using zstd with compression level -18
with open(output_file_path, 'rb') as input_file, open(output_file_path + '.zst', 'wb') as output_zstd_file:
cctx = zstd.ZstdCompressor(level=18)
output_zstd_file.write(cctx.compress(input_file.read()))
# Use zstd command-line tool for compression
command = f'zstd -T0 -18 --long {output_file_path} -o {output_file_path}.zst'
subprocess.run(command, shell=True)
print(f"Compressed file saved as '{output_file_path}.zst'")
# Remove the original gzipped file