From 64215ebceab2f8cc2523402857b9321f354ca9b8 Mon Sep 17 00:00:00 2001 From: Deon George Date: Fri, 25 Jun 2021 21:31:57 +1000 Subject: [PATCH] Import nodelists --- app/Console/Commands/ImportNodelist.php | 142 +---------- app/Jobs/ImportNodelist.php | 231 ++++++++++++++++++ app/Models/Address.php | 4 + app/Models/Nodelist.php | 20 ++ app/Models/System.php | 2 +- app/Traits/Import.php | 82 +++++++ .../2021_06_25_040417_create_nodelist.php | 48 ++++ public/oldschool/css/main.css | 4 +- resources/views/domain/view.blade.php | 16 +- resources/views/system/addedit.blade.php | 4 + resources/views/system/home.blade.php | 42 +++- 11 files changed, 446 insertions(+), 149 deletions(-) create mode 100644 app/Jobs/ImportNodelist.php create mode 100644 app/Models/Nodelist.php create mode 100644 app/Traits/Import.php create mode 100644 database/migrations/2021_06_25_040417_create_nodelist.php diff --git a/app/Console/Commands/ImportNodelist.php b/app/Console/Commands/ImportNodelist.php index 93358b0..ef67b17 100644 --- a/app/Console/Commands/ImportNodelist.php +++ b/app/Console/Commands/ImportNodelist.php @@ -2,8 +2,11 @@ namespace App\Console\Commands; +use Carbon\Carbon; use Illuminate\Console\Command; -use App\Models\{Flag,Node,Zone}; + +use App\Models\{Domain,Nodelist}; +use App\Jobs\ImportNodelist as Job; class ImportNodelist extends Command { @@ -12,7 +15,10 @@ class ImportNodelist extends Command * * @var string */ - protected $signature = 'import:nodelist {file : Nodelist File}'; + protected $signature = 'import:nodelist' + .' {domain : Domain Name}' + .' {file : Nodelist File}' + .' {--D|delete : Delete old data for the date}'; /** * The console command description. @@ -38,135 +44,9 @@ class ImportNodelist extends Command */ public function handle() { - $file = $this->argument('file'); - $lines = intval(exec("wc -l '$file'")); + $do = Domain::where('name',$this->argument('domain'))->singleOrFail(); + $o = Nodelist::firstOrCreate(['date'=>Carbon::now(),'domain_id'=>$do->id]); - $file = fopen($file,"r"); - $bar = $this->output->createProgressBar($lines); - $bar->setFormat("%current%/%max% [%bar%] %percent:3s%% (%memory%) (%remaining%) "); - $bar->setRedrawFrequency(100); - $bar->start(); - - $zone = $region = $host = NULL; - - while (! feof($file)) { - $line = stream_get_line($file, 0, "\r\n"); - - // Lines beginning with a semicolon(;) are comments - if (preg_match('/^;/',$line) OR ($line == chr(0x1A))) - continue; - - $fields = explode(',',$line); - - $o = new Node(); - - // First field is either zone,region,host,hub,down,pvt (or blank) - if ($fields[0] AND ! in_array($fields[0],['Zone','Region','Host','Hub','Pvt','Down'])) - { - $this->error(sprintf('Invalid field zero [%s] - IGNORING record (%s)',$fields[0],$line)); - $bar->advance(); - continue; - } - - $status = NULL; - $node = 0; - - switch ($fields[0]) - { - case 'Zone': $zone = $fields[1]; - $zo = Zone::firstOrCreate([ - 'id'=>$zone - ]); - break; - case 'Host': $host = $fields[1]; break; - case 'Region': $region = $fields[1]; break; - - case 'Down': - case 'Pvt': $status = $fields[0]; - case '': - $node = $fields[1]; - break; - - default: - $this->error(sprintf('Unhandled first field [%s]',$fields[0])); - $bar->advance(); - continue 2; - } - - if (! $zone) - { - $this->error('Zone NOT set, ignoring record...'); - $bar->advance(); - continue; - } - - if (! $host) - { - $host = $zone; - } - - if (! $region) - { - $region = 0 ; - } - - $o = Node::firstOrNew([ - 'zone_id'=>$zo->id, - 'host_id'=>$host, - 'node_id'=>$node, - ]); - - $o->region_id = $region; - $o->system = $fields[2]; - $o->location = $fields[3]; - $o->sysop = $fields[4]; - $o->active = TRUE; - $o->status = $status; - - if (! in_array($fields[5],['-Unpublished-'])) - $o->phone = $fields[5]; - - $o->baud = $fields[6]; - $o->save(); - - // Fields 7 onwards are flags - $flags = collect(); - for ($i=7;$i$flag] - ); - - if (! $flag) - { - $this->warn('Ignoring blank flag on: '.$line); - continue; - } - - if (! $fo->exists) - { - $fo->description = 'Auto Created on Import'; - $fo->save(); - } - - $flags->put($fo->id,['arguments'=>$value]); - } - - $o->flags()->sync($flags); - - $bar->advance(); - } - - fclose($file); - $bar->finish(); - echo "\n"; + return Job::dispatchSync($do,$o,$this->argument('file'),$this->option('delete')); } } \ No newline at end of file diff --git a/app/Jobs/ImportNodelist.php b/app/Jobs/ImportNodelist.php new file mode 100644 index 0000000..3a61015 --- /dev/null +++ b/app/Jobs/ImportNodelist.php @@ -0,0 +1,231 @@ +do = $do; + $this->no = $no; + $this->file = $file; + + if ($delete) + $no->addresses()->detach(); + } + + /** + * Execute the job. + * + * @return void + * @throws \Exception + */ + public function handle() + { + // Get the file from the host + $file = $this->getFileFromHost('',self::importkey,$this->file); + $lines = $this->getFileLines($file); + Log::debug(sprintf('%s:Processing [%d] lines.',static::LOGKEY,$lines)); + + $fh = fopen($file,'r'); + $c =0; + + $region = NULL; + $host = NULL; + $hub_id = NULL; + + while (! feof($fh)) { + $line = stream_get_line($fh, 0, "\r\n"); + + // Lines beginning with a semicolon(;) are comments + if (preg_match('/^;/',$line) OR ($line == chr(0x1a))) + continue; + + // Remove any embedded CR and BOM + $line = str_replace("\r",'',$line); + $line = preg_replace('/^\x{feff}/u','',$line); + $c++; + + $fields = str_getcsv(trim($line)); + + // First field is either zone,region,host,hub,down,pvt (or blank) + if ($fields[0] AND ! in_array($fields[0],Nodelist::definitions)) { + Log::error(sprintf('%s:Invalid field zero [%s] - IGNORING record (%s)',self::LOGKEY,$fields[0],$line)); + continue; + } + + $node = 0; + + switch ($fields[0]) { + case 'Zone': $zone = $fields[1]; + // We ignore the Zone entries, since they are dynamically created. + $zo = Zone::firstOrCreate([ + 'zone_id'=>$zone, + 'domain_id'=>$this->do->id, + ]); + + $region = 0; + $host = 0; + $hub_id = NULL; + $role = DomainController::NODE_ZC; + + continue 2; + + case 'Region': + $region = $fields[1]; + $role = DomainController::NODE_RC; + $host = 0; + $hub_id = NULL; + + break; + + case 'Host': + $host = $fields[1]; + $hub_id = NULL; + $role = DomainController::NODE_NC; + + // We ignore the Host entries, since they are dynamically created. + continue 2; + + case 'Hub': + $role = DomainController::NODE_HC; + + break; + + case 'Pvt': + $node = $fields[1]; + $role = DomainController::NODE_PVT; + + break; + + case 'Down': + $node = $fields[1]; + $role = DomainController::NODE_DOWN; + + break; + + case '': + $node = $fields[1]; + break; + + default: + Log::error(sprintf('%s:Unhandled first field [%s]',self::LOGKEY,$fields[0])); + continue 2; + } + + if (! $zone) { + Log::error(sprintf('%s:Zone NOT set, ignoring record...',self::LOGKEY)); + continue; + } + + Address::unguard(); + $ao = Address::firstOrNew([ + 'zone_id' => $zo->id, + 'region_id' => $region, + 'host_id' => $host, + 'node_id' => $node, + 'point_id' => 0, + ]); + Address::reguard(); + + $ao->active = TRUE; + $ao->role = $role; + $ao->hub_id = $hub_id; + + $role = NULL; + + // Get the System + if ($ao->system_id && (($ao->system->sysop === str_replace('_',' ',$fields[4])) || ($ao->system->name !== str_replace('_',' ',$fields[2])))) { + $so = $ao->system; + + // If the sysop name is different + if ($so->sysop !== str_replace('_',' ',$fields[4])) { + $so->sysop = str_replace('_',' ',$fields[4]); + $so->location = str_replace('_',' ',$fields[3]); + + // We have the same name has changed. + } else { + $so->name = str_replace('_',' ',$fields[2]); + $so->location = str_replace('_',' ',$fields[3]); + } + + // We'll search and see if we already have that system + } else { + $so = System::where('name',str_replace('_',' ',$fields[2])) + ->where('sysop',str_replace('_',' ',$fields[4])) + ->firstOrNew(); + + $so->name = str_replace('_',' ',$fields[2]); + $so->sysop = str_replace('_',' ',$fields[4]); + $so->location = str_replace('_',' ',$fields[3]); + $so->active = TRUE; + + if (! $so->exists) + $so->notes = sprintf('Created by Nodelist Import: %d',$this->no->id); + } + + /* + if (! in_array($fields[5],['-Unpublished-'])) + $so->phone = $fields[5]; + + $so->baud = $fields[6]; + */ + + // Save the system record + $so->save(); + + try { + $so->addresses()->save($ao); + if ($role == DomainController::NODE_HC) + $hub_id = $ao->id; + + $this->no->addresses()->attach($ao,['role'=>$role]); + + } catch (\Exception $e) { + Log::error(sprintf('%s:Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]); + throw new \Exception($e->getMessage()); + } + + if (! ($c % 100)) { + Log::notice(sprintf('%s:Processed [%s] records',self::LOGKEY,$c),['memory'=>memory_get_usage(TRUE)]); + } + } + + fclose($fh); + + if ($this->deletefile and $c) + unlink($file); + + Log::info(sprintf('%s:Records Updated: %d',self::LOGKEY,$c)); + } +} diff --git a/app/Models/Address.php b/app/Models/Address.php index 4be748f..6de94ca 100644 --- a/app/Models/Address.php +++ b/app/Models/Address.php @@ -59,6 +59,10 @@ class Address extends Model return 'Host'; case DomainController::NODE_HC; return 'Hub'; + case DomainController::NODE_PVT; + return 'PVT'; + case DomainController::NODE_DOWN; + return 'DOWN'; case NULL: return 'Node'; default: diff --git a/app/Models/Nodelist.php b/app/Models/Nodelist.php new file mode 100644 index 0000000..f4a98d9 --- /dev/null +++ b/app/Models/Nodelist.php @@ -0,0 +1,20 @@ +belongsToMany(Address::class); + } +} \ No newline at end of file diff --git a/app/Models/System.php b/app/Models/System.php index 527618d..523e7f1 100644 --- a/app/Models/System.php +++ b/app/Models/System.php @@ -31,7 +31,7 @@ class System extends Model * @param Address $o * @return string */ - public function name(Address $o): string + public function full_name(Address $o): string { switch ($o->attributes['role']) { case DomainController::NODE_ZC; diff --git a/app/Traits/Import.php b/app/Traits/Import.php new file mode 100644 index 0000000..526d71b --- /dev/null +++ b/app/Traits/Import.php @@ -0,0 +1,82 @@ +_columns = collect(explode(',',strtoupper($line)))->filter(); + return $this->_columns->intersect($this->columns); + } + + /** + * Get the index for the column in the file + * + * @param string $key + * @return int|null + */ + private function getColumnKey(string $key): ?int + { + return ($x=$this->_columns->search(strtoupper($this->columns->get($key)))) !== FALSE ? $x : NULL; + } + + private function getFileFromHost(string $host,string $key,string $file): string + { + $path = 'import/'.$key; + + // If we are doing it locally, we dont need to retrieve it via curl + if (! $host) { + return preg_match('/^data/',$file) ? $file : sprintf('storage/app/public/%s/%s',$path,basename($file)); + } + + // Get the file from the host + $srcfile = sprintf('http://%s%s',$host,$file); + $dstfile = '/tmp/'.basename($host); + Log::debug(sprintf('%s:Retrieving [%s] from [%s]',static::LOGKEY,$host,$file)); + + $src = fopen($srcfile,'r'); + $dst = fopen($dstfile,'w'); + stream_copy_to_stream($src,$dst); + fclose($src); + fclose($dst); + + // Store the file for later processing + Storage::putFileAs($path,$dstfile,basename($file)); + + return Storage::path($path.'/'.basename($file)); + } +} \ No newline at end of file diff --git a/database/migrations/2021_06_25_040417_create_nodelist.php b/database/migrations/2021_06_25_040417_create_nodelist.php new file mode 100644 index 0000000..5404368 --- /dev/null +++ b/database/migrations/2021_06_25_040417_create_nodelist.php @@ -0,0 +1,48 @@ +id(); + $table->timestamps(); + $table->date('date'); + $table->integer('domain_id'); + $table->foreign('domain_id')->references('id')->on('domains'); + + $table->unique(['date','domain_id']); + }); + + Schema::create('address_nodelist', function (Blueprint $table) { + $table->integer('role')->nullable(); + + $table->integer('address_id'); + $table->foreign('address_id')->references('id')->on('addresses'); + $table->integer('nodelist_id'); + $table->foreign('nodelist_id')->references('id')->on('nodelists'); + + $table->unique(['address_id','nodelist_id']); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('address_nodelist'); + Schema::dropIfExists('nodelists'); + } +} diff --git a/public/oldschool/css/main.css b/public/oldschool/css/main.css index 2da5c66..c7d9aff 100644 --- a/public/oldschool/css/main.css +++ b/public/oldschool/css/main.css @@ -341,13 +341,13 @@ ul#navlist-desktop { #content ul.ind { margin-right:2ch } -#content ul li { +#content ul:not(.pagination) li { margin:0 1ch; text-indent:-3ch; padding-left:3ch; display:block } -#content ul li:before { +#content ul:not(.pagination) li:before { content:"\2666\a0\A0"; color:#0a0 } diff --git a/resources/views/domain/view.blade.php b/resources/views/domain/view.blade.php index 791dd4a..c65d9c7 100644 --- a/resources/views/domain/view.blade.php +++ b/resources/views/domain/view.blade.php @@ -79,10 +79,20 @@ - - @foreach ($oz->addresses()->active()->FTNorder()->whereNull('hub_id')->with(['system','zone.domain'])->get() as $ao) + @if ($ao->role == 'Host') + + {{ sprintf('NC-%s-%05d',$oz->domain->name,$ao->host_id) }} + {{ $ao->system->sysop }} + {{ $ao->system->location }} + {{ $ao->role }} + {{ $oz->zone_id }}:{{ $ao->host_id }}/0.0@{{ $oz->domain->name }} + - + + @endif + - {{ $ao->system->name($ao) }} + {{ $ao->system->full_name($ao) }} {{ $ao->system->sysop }} {{ $ao->system->location }} {{ $ao->role }} @@ -93,7 +103,7 @@ @foreach ($oz->addresses()->active()->FTNorder()->where('hub_id',$ao->id)->with(['system','zone.domain'])->get() as $aoo) - {{ $aoo->system->name($aoo) }} + {{ $aoo->system->full_name($aoo) }} {{ $aoo->system->sysop }} {{ $ao->system->location }} {{ $aoo->role }} diff --git a/resources/views/system/addedit.blade.php b/resources/views/system/addedit.blade.php index f957d83..64b24a7 100644 --- a/resources/views/system/addedit.blade.php +++ b/resources/views/system/addedit.blade.php @@ -9,6 +9,10 @@ @endsection @section('content') + @if ($o->exists) +

{{ $o->name }}

+ @endif +
@if ($o->exists) diff --git a/resources/views/system/home.blade.php b/resources/views/system/home.blade.php index 689d84c..e68e561 100644 --- a/resources/views/system/home.blade.php +++ b/resources/views/system/home.blade.php @@ -12,7 +12,7 @@
-
+

This system is aware of the following systems:

@if (\App\Models\System::count() == 0) @@ -22,33 +22,30 @@

There are no systems - you need to ask an admin to create one for you.

@endcan @else - + @can('admin',(new \App\Models\System)) +

You can Add New System.

+ @endcan + +
- - + + - @can('admin',(new \App\Models\System)) - - - - @endcan - @foreach (\App\Models\System::orderBy('name')->cursor() as $oo) + @foreach (\App\Models\System::active()->orderBy('name')->cursor() as $oo) - - + + @endforeach @@ -67,6 +66,13 @@ @endsection @section('page-scripts') + + + + + + + @append
ID System Sysop LocationActiveZeroTier ID ConnectAddressZeroTier ID
Add New System
{{ $oo->id }} {{ $oo->name }} {{ $oo->sysop }} {{ $oo->location }}{{ $oo->active ? 'YES' : 'NO' }}- @switch($oo->method) @case(23)Telnet@break @@ -57,6 +54,8 @@ @default No details @endswitch {{ join(',',$oo->addresses->pluck('ftn')->toArray()) }}{{ $oo->zt_id }}