ArchiveTeam/Migrated_ArchiveTeam_Project_URL_Extractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated to include .zst support

Browse Source
This commit is contained in:
datechnoman 2024-01-23 22:20:41 +00:00
parent 11a9f30a29
commit fd15770812
1 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
archiveteam_project_url_extractor.py
View File

@ -11,6 +11,8 @@ def process_file(file, directory_path, output_directory, patterns, counter_lock,
# Determine the appropriate command based on file extension # Determine the appropriate command based on file extension
if file.endswith(".gz"): if file.endswith(".gz"):
command = f"zcat {file_path}" command = f"zcat {file_path}"
elif file.endswith(".zst"):
command = f"zstdcat {file_path}"
elif file.endswith(".txt"): elif file.endswith(".txt"):
command = f"cat {file_path}" command = f"cat {file_path}"
else: else:
@ -38,20 +40,20 @@ def process_file(file, directory_path, output_directory, patterns, counter_lock,
remaining_count = len(gzipped_files) - processed_counter[0] remaining_count = len(gzipped_files) - processed_counter[0]
print(f"{file_path} processed. Remaining files: {remaining_count}") print(f"{file_path} processed. Remaining files: {remaining_count}")
# Ask the user for the directory containing .txt and .txt.gz files # Ask the user for the directory containing .txt, .txt.gz, and .zst files
directory_path = input("Enter the directory path containing .txt and .txt.gz files: ") directory_path = input("Enter the directory path containing .txt, .txt.gz, and .zst files: ")
# Ensure the directory exists # Ensure the directory exists
if not os.path.exists(directory_path): if not os.path.exists(directory_path):
print(f"Error: The directory '{directory_path}' does not exist.") print(f"Error: The directory '{directory_path}' does not exist.")
exit() exit()
# List all files in the directory that end with .txt or .txt.gz # List all files in the directory that end with .txt, .txt.gz, or .zst
gzipped_files = [file for file in os.listdir(directory_path) if file.endswith(".txt") or file.endswith(".txt.gz")] gzipped_files = [file for file in os.listdir(directory_path) if file.endswith((".txt", ".txt.gz", ".zst"))]
# Check if there are any .txt or .txt.gz files in the directory # Check if there are any .txt, .txt.gz, or .zst files in the directory
if not gzipped_files: if not gzipped_files:
print("Error: No .txt or .txt.gz files found in the specified directory.") print("Error: No .txt, .txt.gz, or .zst files found in the specified directory.")
exit() exit()
# Ask the user for the output directory # Ask the user for the output directory