ArchiveTeam/CommonCrawl_URL_Processor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add in checking for new files once list is depleted

Browse Source
This commit is contained in:
datechnoman 2023-12-21 01:58:55 +00:00
parent 0aad853966
commit 50e89b9de2
1 changed files with 54 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
commoncrawl_local_to_share_move.ps1
View File

@ -7,35 +7,37 @@ $assemblyPath = "C:\Program Files (x86)\WinSCP\WinSCPnet.dll"
# Specify the path to winscp.exe # Specify the path to winscp.exe
$winscpPath = "C:\Program Files (x86)\WinSCP\winscp.exe" $winscpPath = "C:\Program Files (x86)\WinSCP\winscp.exe"
# Create a session instance # Specify the local directory path
$sessionType = [WinSCP.Session].FullName $localPath = "E:\CommonCrawl"
$session = New-Object $sessionType
# Set up session options # Specify the remote directory path
$sessionOptionsType = [WinSCP.SessionOptions].FullName $remotePath = "/mnt/user/ArchiveTeam/CommonCrawl_Files/WAT_URLs/CC-MAIN-2023-14_March_April_2023"
$sessionOptions = New-Object $sessionOptionsType
# Set properties for session options while ($true) {
$sessionOptions.Protocol = [WinSCP.Protocol]::Sftp $session = $null
$sessionOptions.HostName = "192.168.1.248"
$sessionOptions.UserName = "root" try {
$sessionOptions.Password = "Tcft65rdx!" # Create a session instance
$sessionOptions.SshHostKeyFingerprint = "ssh-ed25519 255 xyTP5uPM3w6ebT7P7Mx1945YIYrK7NXWlZNNOGt2geY" $sessionType = [WinSCP.Session].FullName
$session = New-Object $sessionType
# Set up session options
$sessionOptionsType = [WinSCP.SessionOptions].FullName
$sessionOptions = New-Object $sessionOptionsType
# Set properties for session options
$sessionOptions.Protocol = [WinSCP.Protocol]::Sftp
$sessionOptions.HostName = "192.168.1.248"
$sessionOptions.UserName = "root"
$sessionOptions.Password = "Tcft65rdx!"
$sessionOptions.SshHostKeyFingerprint = "ssh-ed25519 255 xyTP5uPM3w6ebT7P7Mx1945YIYrK7NXWlZNNOGt2geY"
try
{
# Set the ExecutablePath property # Set the ExecutablePath property
$session.GetType().GetProperty("ExecutablePath").SetValue($session, $winscpPath, $null) $session.GetType().GetProperty("ExecutablePath").SetValue($session, $winscpPath, $null)
# Connect # Connect
$session.Open($sessionOptions) $session.Open($sessionOptions)
# Specify the local directory path
$localPath = "E:\CommonCrawl"
# Specify the remote directory path
$remotePath = "/mnt/user/ArchiveTeam/CommonCrawl_Files/WAT_URLs/CC-MAIN-2023-23_May_June_2023"
# Get a list of local files matching the pattern "_urls.txt.gz" # Get a list of local files matching the pattern "_urls.txt.gz"
$localFiles = Get-ChildItem -Path $localPath -Filter "*_urls.txt.gz" -File -Recurse $localFiles = Get-ChildItem -Path $localPath -Filter "*_urls.txt.gz" -File -Recurse
@ -43,15 +45,12 @@ try
$localFiles = $localFiles | Get-Random -Count $localFiles.Count $localFiles = $localFiles | Get-Random -Count $localFiles.Count
# Check if there are any files to upload # Check if there are any files to upload
if ($localFiles.Count -eq 0) if ($localFiles.Count -eq 0) {
{
Write-Host "No files found to upload." Write-Host "No files found to upload."
} break # Break out of the loop if no files are found
else } else {
{
# Iterate through each file and upload/delete it # Iterate through each file and upload/delete it
foreach ($fileInfo in $localFiles) foreach ($fileInfo in $localFiles) {
{
# Construct the full local file path # Construct the full local file path
$localFilePath = $fileInfo.FullName $localFilePath = $fileInfo.FullName
@ -62,21 +61,20 @@ try
$transferResult = $session.PutFiles($localFilePath, $remoteFilePath, $False) $transferResult = $session.PutFiles($localFilePath, $remoteFilePath, $False)
# Check if the upload was successful # Check if the upload was successful
if ($transferResult.IsSuccess) if ($transferResult.IsSuccess) {
{
Write-Host "Upload successful: $localFilePath to $remoteFilePath" Write-Host "Upload successful: $localFilePath to $remoteFilePath"
# Attempt to delete the local file # Attempt to delete the local file
Remove-Item -Path $localFilePath -Force Remove-Item -Path $localFilePath -Force
} } else {
else
{
Write-Host "Failed to upload file: $localFilePath" Write-Host "Failed to upload file: $localFilePath"
} }
} }
} }
} } finally {
finally # Dispose of the session if it's not null
{ if ($session -ne $null) {
$session.Dispose() $session.Dispose()
}
}
} }