ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated to use command line zstd

Browse Source
This commit is contained in:
datechnoman 2024-01-20 11:29:57 +00:00
parent 78f6b69cdf
commit e98f80aec4
1 changed files with 4 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
warc_wat_url_processor.py
View File

@ -1,7 +1,6 @@
import subprocess import subprocess
import os import os
import gzip import gzip
import zstandard as zstd
import re import re
import traceback import traceback
from multiprocessing import Pool from multiprocessing import Pool
@ -38,10 +37,9 @@ def process_file(file_path):
output_file.write('\n'.join(urls)) output_file.write('\n'.join(urls))
print(f"URLs written to {output_file_path}") print(f"URLs written to {output_file_path}")
# Compress the output file using zstd with compression level -18 # Use zstd command-line tool for compression
with open(output_file_path, 'rb') as input_file, open(output_file_path + '.zst', 'wb') as output_zstd_file: command = f'zstd -T0 -18 --long {output_file_path} -o {output_file_path}.zst'
cctx = zstd.ZstdCompressor(level=18) subprocess.run(command, shell=True)
output_zstd_file.write(cctx.compress(input_file.read()))
print(f"Compressed file saved as '{output_file_path}.zst'") print(f"Compressed file saved as '{output_file_path}.zst'")
# Remove the original gzipped file # Remove the original gzipped file