ArchiveTeam/Migrated_Keyword_URL_Extractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated script to stream compressed files

Browse Source
This commit is contained in:
datechnoman 2024-02-04 22:14:27 +00:00
parent 4930a9418d
commit 95abf80bd1
1 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
url_extractor.py
View File

@ -10,11 +10,14 @@ def process_file(file, directory_path, output_file_path, keyword, counter_lock,
# Run the command and append the output to the same output file # Run the command and append the output to the same output file
command = f"zcat {file_path} | grep '{keyword}'" command = f"zcat {file_path} | grep '{keyword}'"
result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, text=True) process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, text=True)
# Append the output to the same output file # Stream the output line by line and append to the output file
with open(output_file_path, "a") as output_file: with open(output_file_path, "a") as output_file:
output_file.write(result.stdout) for line in process.stdout:
output_file.write(line)
process.wait() # Wait for the process to finish
# Update the processed files count # Update the processed files count
with counter_lock: with counter_lock: