Updated script to stream compressed files

This commit is contained in:
datechnoman 2024-02-04 22:14:27 +00:00
parent 4930a9418d
commit 95abf80bd1

View File

@ -10,11 +10,14 @@ def process_file(file, directory_path, output_file_path, keyword, counter_lock,
# Run the command and append the output to the same output file
command = f"zcat {file_path} | grep '{keyword}'"
result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, text=True)
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, text=True)
# Append the output to the same output file
# Stream the output line by line and append to the output file
with open(output_file_path, "a") as output_file:
output_file.write(result.stdout)
for line in process.stdout:
output_file.write(line)
process.wait() # Wait for the process to finish
# Update the processed files count
with counter_lock: