ArchiveTeam/Migrated_CommonCrawl_WAT_Path_Comparer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated script to support .zst instead of .gz

Browse Source
This commit is contained in:
datechnoman 2024-02-02 04:07:17 +00:00
parent 10c2658bff
commit 0fd62a7391
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
commoncrawl_wat_path_comparer.py
View File

@ -9,7 +9,7 @@ def read_paths_from_file(file_path):
return [line.strip() for line in file]
def list_files_in_directory(directory):
return [f.replace("_urls.txt", "") for f in os.listdir(directory) if os.path.isfile(os.path.join(directory, f))]
return [f.replace("_urls.txt.zst", ".gz") for f in os.listdir(directory) if os.path.isfile(os.path.join(directory, f))]
def main():
# Get the directory containing wat.paths file