ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update warc_wat_url_processor.py

Browse Source
This commit is contained in:
datechnoman 2024-01-28 09:10:17 +00:00
parent 875082e8d9
commit d0fa7c84f4
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
warc_wat_url_processor.py
View File

@ -48,6 +48,7 @@ def process_file(file_path):
# Extract URLs from the gzipped file # Extract URLs from the gzipped file
urls = extract_urls_from_file(file_path) urls = extract_urls_from_file(file_path)
print(f"Extracted {len(urls)} URLs from {file_path}")
# Create the output file path with '_urls.txt' extension # Create the output file path with '_urls.txt' extension
output_file_path = os.path.splitext(file_path)[0] + '_urls.txt' output_file_path = os.path.splitext(file_path)[0] + '_urls.txt'