ArchiveTeam/Migrated_ArchiveOrg_CDX_Stats_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added in removed cdxcount and cdxsummary

Browse Source
This commit is contained in:
datechnoman 2024-03-13 04:34:48 +00:00
parent 9793f0e9db
commit 9e94e1e108
1 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
urls_automated_cdx_processor.py
View File

@ -35,6 +35,19 @@ def run_cdxsummary(file_path, json_filepath):
# Parse the JSON output # Parse the JSON output
json_output = json.loads(result.stdout) json_output = json.loads(result.stdout)
# Add "cdxcount" entry with value 1
json_output["cdxcount"] = 1
# Add "cdxsize" entry with the size of the cdx.gz file in bytes
cdx_size_bytes = os.path.getsize(file_path)
json_output["cdxsize"] = cdx_size_bytes
# Remove "pathquery" and "samples" keys
if "pathquery" in json_output:
del json_output["pathquery"]
if "samples" in json_output:
del json_output["samples"]
# Process the tophosts section # Process the tophosts section
if "tophosts" in json_output: if "tophosts" in json_output:
processed_tophosts = process_tophosts(json_output["tophosts"]) processed_tophosts = process_tophosts(json_output["tophosts"])