From 06b924340c0d9589b982f85a2b4d997fa8837194 Mon Sep 17 00:00:00 2001 From: Deon George Date: Sat, 15 Jun 2024 15:12:09 +1000 Subject: [PATCH] Support nodelist archives with more than 1 file in it --- app/Jobs/EchoareaImport.php | 4 +--- app/Jobs/FileareaImport.php | 4 +--- app/Jobs/NodelistImport.php | 32 ++++++++++++++++++++++++++------ app/Traits/Import.php | 22 +++------------------- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/app/Jobs/EchoareaImport.php b/app/Jobs/EchoareaImport.php index 9469211..2a42aee 100644 --- a/app/Jobs/EchoareaImport.php +++ b/app/Jobs/EchoareaImport.php @@ -10,12 +10,10 @@ use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\Log; use App\Models\{Domain,Echoarea}; -use App\Traits\Import as ImportTrait; class EchoareaImport implements ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - use ImportTrait; protected const LOGKEY = 'JEI'; private const importkey = 'echoarea'; @@ -102,4 +100,4 @@ class EchoareaImport implements ShouldQueue Log::info(sprintf('%s:= Updated %d records',self::LOGKEY,$p)); } -} +} \ No newline at end of file diff --git a/app/Jobs/FileareaImport.php b/app/Jobs/FileareaImport.php index 5cfa049..96ba805 100644 --- a/app/Jobs/FileareaImport.php +++ b/app/Jobs/FileareaImport.php @@ -10,12 +10,10 @@ use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\Log; use App\Models\{Domain,Filearea}; -use App\Traits\Import as ImportTrait; class FileareaImport implements ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - use ImportTrait; protected const LOGKEY = 'JFI'; private const importkey = 'filearea'; @@ -102,4 +100,4 @@ class FileareaImport implements ShouldQueue Log::info(sprintf('%s:= Updated %d records',self::LOGKEY,$p)); } -} +} \ No newline at end of file diff --git a/app/Jobs/NodelistImport.php b/app/Jobs/NodelistImport.php index 62c236b..2e22c0a 100644 --- a/app/Jobs/NodelistImport.php +++ b/app/Jobs/NodelistImport.php @@ -80,11 +80,27 @@ class NodelistImport implements ShouldQueue // Get the file from the host $file = $this->getFileFromHost(self::importkey,$this->file); Log::debug(sprintf('%s:+ Loading file [%s].',static::LOGKEY,$file)); - $lines = $this->getFileLines($file); - Log::debug(sprintf('%s:- Processing [%d] lines.',static::LOGKEY,$lines)); + $z = $this->openFile($file); + $c = 0; $fh = NULL; - $z = $this->openFile($file,$fh); + while ($c < $z->count()) { + // Nodelist files have an extension of numbers, between 1-365 + if (preg_match('/^.+\.[0-3][0-9][0-9]$/',$z->getNameIndex($c))) { + $fh = $z->getStreamIndex($c); + break; + } + + $c++; + } + + if (is_null($fh)) + throw new \Exception('Couldnt find nodelist in file'); + + $lines = $this->getFileLines($fh); + Log::debug(sprintf('%s:- Processing [%d] lines.',static::LOGKEY,$lines)); + // Rewind + $fh = $z->getStreamIndex($c); // Line 1 tells us the nodelist and the CRC $line = stream_get_line($fh,0,"\r\n"); @@ -123,9 +139,9 @@ class NodelistImport implements ShouldQueue $no->addresses()->detach(); elseif ($no->addresses->count()) { - Log::error($x=sprintf('%s:! Nodelist [%s] for [%s] has existing records [%d]',self::LOGKEY,$date,$do->name,$no->addresses->count())); + Log::error(sprintf('%s:! Nodelist [%s] for [%s] has existing records [%d]',self::LOGKEY,$date,$do->name,$no->addresses->count())); - throw new \Exception($x); + return; } $mailer_binkp = Mailer::where('name','BINKP')->singleOrFail(); @@ -179,6 +195,7 @@ class NodelistImport implements ShouldQueue $region = 0; $host = 0; $ishub = FALSE; + $ho = NULL; break; @@ -186,18 +203,21 @@ class NodelistImport implements ShouldQueue $region = (int)$fields[1]; $host = (int)$fields[1]; $ishub = FALSE; + $ho = NULL; break; case 'Host': $host = (int)$fields[1]; $ishub = FALSE; + $ho = NULL; break; case 'Hub': $node = (int)$fields[1]; $ishub = TRUE; + $ho = NULL; break; @@ -543,7 +563,7 @@ class NodelistImport implements ShouldQueue ->diff(our_nodes($do)->pluck('id')); $remove = Address::whereIn('id',$remove)->get(); - Log::alert(sprintf('%s:%% Deleting [%d] addresses [%s]',self::LOGKEY,$remove->count(),$remove->pluck('ftn4d')->join(','))); + Log::alert(sprintf('%s:%% Deleting [%d] addresses [%s]',self::LOGKEY,$remove->count(),$remove->pluck('ftn2d')->join(','))); Address::whereIN('id',$remove->pluck('id'))->update(['active'=>FALSE]); Address::whereIN('id',$remove->pluck('id'))->delete(); diff --git a/app/Traits/Import.php b/app/Traits/Import.php index 51b7e38..cd64f4c 100644 --- a/app/Traits/Import.php +++ b/app/Traits/Import.php @@ -6,7 +6,6 @@ namespace App\Traits; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Storage; use App\Models\File; @@ -19,39 +18,24 @@ trait Import * Count the lines in a file * Assumes $file is compressed with ZIP */ - private function getFileLines(string $file): int + private function getFileLines(mixed $f): int { $c = 0; - $f = NULL; - $z = $this->openFile($file,$f); - while (! feof($f)) { fgets($f); $c++; } - fclose($f); - return $c; } - private function openFile(string $file,&$f): \ZipArchive + private function openFile(string $file): \ZipArchive { $z = new \ZipArchive; if ($z->open($file,\ZipArchive::RDONLY) === TRUE) { - if ($z->count() !== 1) - throw new \Exception(sprintf('%s:! File [%s] has more than 1 file (%d)', self::LOGKEY, $file, $z->count())); - - $zipfile = $z->statIndex(0, \ZipArchive::FL_UNCHANGED); - Log::debug(sprintf('%s:- Looking at [%s] in archive [%s]', self::LOGKEY,$zipfile['name'],$file)); - - $f = $z->getStream($zipfile['name']); - if (! $f) - throw new \Exception(sprintf('%s:! Failed getting ZipArchive::stream (%s)',self::LOGKEY,$z->getStatusString())); - else - return $z; + return $z; } else { throw new \Exception(sprintf('%s:! Failed opening ZipArchive [%s] (%s)',self::LOGKEY,$file,$z->getStatusString()));