ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update warc_wat_url_processor.py

Browse Source
This commit is contained in:
datechnoman 2024-01-20 12:33:28 +00:00
parent 06e3399861
commit 513b32e80a
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
warc_wat_url_processor.py
View File

@ -39,7 +39,7 @@ def process_file(file_path):
# Use zstd command-line tool for compression # Use zstd command-line tool for compression
compressed_file_path = f'{output_file_path}.zst' compressed_file_path = f'{output_file_path}.zst'
command_compress = f'zstd -T0 -14 --long {output_file_path} -o {compressed_file_path}' command_compress = f'zstd -T0 -12 --long {output_file_path} -o {compressed_file_path}'
subprocess.run(command_compress, shell=True) subprocess.run(command_compress, shell=True)
print(f"Compressed file saved as '{compressed_file_path}'") print(f"Compressed file saved as '{compressed_file_path}'")