Enhance nodelist import

app/Console/Commands/NodelistImport.php

@@ -2,10 +2,8 @@
 
 namespace App\Console\Commands;
 
-use Carbon\Carbon;
 use Illuminate\Console\Command;
 
-use App\Models\{Domain,Nodelist};
 use App\Jobs\NodelistImport as Job;
 
 class NodelistImport extends Command
@@ -16,9 +14,9 @@ class NodelistImport extends Command
 	 * @var string
 	 */
 	protected $signature = 'nodelist:import'
-		.' {domain : Domain Name}'
 		.' {file : Nodelist File}'
-		.' {--D|delete : Delete old data for the date}';
+		.' {--D|delete : Delete old data for the date}'
+		.' {--U|unlink : Delete file after import}';
 
 	/**
 	 * The console command description.
@@ -27,16 +25,6 @@ class NodelistImport extends Command
 	 */
 	protected $description = 'Import Nodelist';
 
-	/**
-	 * Create a new command instance.
-	 *
-	 * @return void
-	 */
-	public function __construct()
-	{
-		parent::__construct();
-	}
-
 	/**
 	 * Execute the console command.
 	 *
@@ -44,9 +32,6 @@ class NodelistImport extends Command
 	 */
 	public function handle()
 	{
-		$do = Domain::where('name',$this->argument('domain'))->singleOrFail();
+		return Job::dispatchSync($this->argument('file'),$this->option('delete'),$this->option('unlink'));
-		$o = Nodelist::firstOrCreate(['date'=>Carbon::now(),'domain_id'=>$do->id]);
-
-		return Job::dispatchSync($do,$o,$this->argument('file'),$this->option('delete'));
 	}
 }

app/Http/Controllers/SystemController.php

@@ -257,6 +257,7 @@ class SystemController extends Controller
 				'name' => 'required|min:3',
 				'location' => 'required|min:3',
 				'sysop' => 'required|min:3',
+				'phone' => 'nullable|regex:/^([0-9-]+)$/',
 				'address' => 'nullable|regex:/^(?!:\/\/)(?=.{1,255}$)((.{1,63}\.){1,127}(?![0-9]*$)[a-z0-9-]+\.?)$/i',
 				'port' => 'nullable|digits_between:2,5',
 				'method' => 'nullable|numeric',
@@ -267,7 +268,7 @@ class SystemController extends Controller
 				'zt_id' => 'nullable|size:10|regex:/^([A-Fa-f0-9]){10}$/|unique:systems,zt_id,'.($o->exists ? $o->id : 0),
 			]);
 
-			foreach (['name','location','sysop','address','port','active','method','notes','mailer_type','mailer_address','mailer_port','zt_id'] as $key)
+			foreach (['name','location','sysop','phone','address','port','active','method','notes','mailer_type','mailer_address','mailer_port','zt_id'] as $key)
 				$o->{$key} = $request->post($key);
 
 			$o->save();

app/Jobs/NodelistImport.php

@@ -2,16 +2,18 @@
 
 namespace App\Jobs;
 
+use Carbon\Carbon;
 use Illuminate\Bus\Queueable;
-use Illuminate\Contracts\Queue\ShouldBeUnique;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Str;
 
 use App\Http\Controllers\DomainController;
-use App\Models\{Address,Domain,Nodelist,System,Zone};
+use App\Models\{Address,Domain,Nodelist,Setup,System,Zone};
 use App\Traits\Import as ImportTrait;
 
 class NodelistImport implements ShouldQueue
@@ -21,27 +23,23 @@ class NodelistImport implements ShouldQueue
 
 	protected const LOGKEY = 'JIN';
 	private const importkey = 'nodelist';
+
 	private string $file;
-	private Domain $do;
+	private bool $delete_file;
-	private Nodelist $no;
+	private bool $delete_recs;
-	private bool $deletefile = FALSE;
 
 	/**
 	 * Import Nodelist constructor.
 	 *
-	 * @param Domain $do
-	 * @param Nodelist $no
 	 * @param string $file
-	 * @param bool $delete
+	 * @param bool $delete_recs
+	 * @param bool $delete_file
 	 */
-	public function __construct(Domain $do,Nodelist $no,string $file,bool $delete=TRUE)
+	public function __construct(string $file,bool $delete_recs=FALSE,bool $delete_file=TRUE)
 	{
-		$this->do = $do;
-		$this->no = $no;
 		$this->file = $file;
-
+		$this->delete_file = $delete_file;
-		if ($delete)
+		$this->delete_recs = $delete_recs;
-			$no->addresses()->detach();
 	}
 
 	/**
@@ -52,21 +50,56 @@ class NodelistImport implements ShouldQueue
 	 */
 	public function handle()
 	{
+		$us = Setup::findOrFail(config('app.id'));
+
 		// Get the file from the host
 		$file = $this->getFileFromHost('',self::importkey,$this->file);
 		$lines = $this->getFileLines($file);
 		Log::debug(sprintf('%s:Processing [%d] lines.',static::LOGKEY,$lines));
 
 		$fh = fopen($file,'r');
+
+		// Line 1 tells us the nodelist and the CRC
+		$line = stream_get_line($fh, 0, "\r\n");
+
+		$matches = [];
+		if ((! preg_match('/^;A\ /',$line)) || (! preg_match('/^;A\ (.*)\ Nodelist for ([MTWFS][a-z]+,\ [JFMASOND][a-z]+\ [0-9]{1,2},\ [0-9]{4})\ --\ Day\ number\ ([0-9]+)\ :\ ([0-9a-f]+)$/',$line,$matches)))
+			abort(500,'Nodelist invalid');
+
+		$file_crc = $matches[4];
+		$do = Domain::where('name',strtolower($matches[1]))->single();
+
+		if (! $do) {
+			Log::error(sprintf('%s:! Domain not found [%s].',static::LOGKEY,strtolower($matches[1])));
+			return;
+		}
+
+		$date = Carbon::createFromFormat('D, M d, Y H:i',$matches[2].'0:00');
+
+		if ($date->dayOfYear != $matches[3]) {
+			Log::error(sprintf('%s:! Nodelist date doesnt match [%d] (%d:%s).',static::LOGKEY,$matches[3],$date->dayOfYear,$date->format('Y-m-d')));
+			return;
+		}
+
+		Log::debug(sprintf('%s:Importing nodelist for [%s] dated [%s] lines.',static::LOGKEY,$do->name,$date->format('Y-m-d')));
+
+		DB::beginTransaction();
+		$no = Nodelist::firstOrCreate(['date'=>$date,'domain_id'=>$do->id]);
+
+		if ($this->delete_recs)
+			$no->addresses()->detach();
+
 		$p = $c =0;
 
 		$region = NULL;
 		$host = NULL;
 		$hub_id = NULL;
 		$zo = NULL;
+		$tocrc = '';
 
 		while (! feof($fh)) {
 			$line = stream_get_line($fh, 0, "\r\n");
+			$tocrc .= $line."\r\n";
 
 			// Lines beginning with a semicolon(;) are comments
 			if (preg_match('/^;/',$line) OR ($line == chr(0x1a)))
@@ -92,7 +125,7 @@ class NodelistImport implements ShouldQueue
 					Zone::unguard();
 					$zo = Zone::firstOrNew([
 						'zone_id'=>$zone,
-						'domain_id'=>$this->do->id,
+						'domain_id'=>$do->id,
 						'active'=>TRUE,
 					]);
 					Zone::reguard();
@@ -159,6 +192,7 @@ class NodelistImport implements ShouldQueue
 
 			Address::unguard();
 			$ao = Address::firstOrNew([
+				'zone_id' => $zo->id,
 				'region_id' => $region,
 				'host_id' => $host,
 				'node_id' => $node,
@@ -177,25 +211,27 @@ class NodelistImport implements ShouldQueue
 
 			$sysop = trim(str_replace('_',' ',$fields[4]));
 			$system = trim(str_replace('_',' ',$fields[2]));
-			$location = trim(str_replace('_',' ',$fields[3]));
 
 			// Get the System
 			if ($ao->system_id && (($ao->system->sysop === $sysop) || ($ao->system->name === $system))) {
 				$so = $ao->system;
 
-				// If the sysop name is different
+				// Dont change the system details if a user exists here, or its us
-				if ($so->sysop !== $sysop) {
+				if ((! $so->users->count()) && ($so->id != $us->system_id)) {
-					Log::debug(sprintf('%s:Sysop Name changed for BBS [%s:%s] from [%s] to [%s]',
+					// If the sysop name is different
-						self::LOGKEY,$so->id,$so->name,$so->sysop,$sysop));
+					if ($so->sysop !== $sysop) {
+						Log::debug(sprintf('%s:Sysop Name changed for BBS [%s:%s] from [%s] to [%s]',
+							self::LOGKEY,$so->id,$so->name,$so->sysop,$sysop));
 
-					$so->sysop = $sysop;
+						$so->sysop = $sysop;
 
-				// We have the same name has changed.
+					// We have the same name has changed.
-				} elseif ($so->name !== $system) {
+					} elseif ($so->name !== $system) {
-					Log::debug(sprintf('%s:System Name changed for BBS [%s:%s] to [%s]',
+						Log::debug(sprintf('%s:System Name changed for BBS [%s:%s] to [%s]',
-						self::LOGKEY,$so->id,$so->name,$system));
+							self::LOGKEY,$so->id,$so->name,$system));
 
-					$so->name = $system;
+						$so->name = $system;
+					}
 				}
 
 			// We'll search and see if we already have that system
@@ -214,10 +250,11 @@ class NodelistImport implements ShouldQueue
 				$so->active = TRUE;
 
 				if (! $so->exists)
-					$so->notes = sprintf('Created by Nodelist Import: %d',$this->no->id);
+					$so->notes = sprintf('Created by Nodelist Import: %d',$no->id);
 			}
 
-			$so->location = $location;
+			$so->phone = $fields[5] != '-Unpublished-' ? $fields[5] : NULL;
+			$so->location = trim(str_replace('_',' ',$fields[3]));
 			/*
 			if (! in_array($fields[5],['-Unpublished-']))
 				$so->phone = $fields[5];
@@ -225,6 +262,70 @@ class NodelistImport implements ShouldQueue
 			$so->baud = $fields[6];
 			*/
 
+			// Flags
+			$method = NULL;
+			$address = '';
+			$port = '';
+			for ($i=7;$i<count($fields);$i++) {
+				$x = Str::of($fields[$i])->split('/:/');
+
+				switch ($x->first()) {
+					// address
+					case 'INA':
+						$address = $x->get(1);
+						break;
+
+					case 'IBN':
+						$method = Setup::O_BINKP;
+
+						switch ($x->count()) {
+							case 1:
+								$port = 24554;
+								break;
+							case 2:
+								$port = $x->get(1);
+								break;
+							case 3:
+								$address = $x->get(1);
+								$port = $x->get(2);
+						}
+						break;
+
+					case 'ITN':
+						$method = Setup::O_EMSI;
+
+						switch ($x->count()) {
+							case 1:
+								$port = 60179;
+								break;
+							case 2:
+								$port = $x->get(1);
+								break;
+							case 3:
+								$address = $x->get(1);
+								$port = $x->get(2);
+						}
+						break;
+
+					// Ignore
+					case 'ZEC':
+					case 'REC':
+					case 'MO':
+					case 'CM':
+						break;
+
+					default:
+						Log::debug(sprintf('%s: - Not configured to handle flag [%s]',self::LOGKEY,$x->first()));
+						continue 2;
+				}
+
+				if ($method && ($ao->role != DomainController::NODE_PVT)) {
+					$so->mailer_type = $method;
+					$so->mailer_address = $address;
+					$so->mailer_port = $port;
+				}
+			}
+
 			// Save the system record
 			$so->save();
 
@@ -244,7 +345,7 @@ class NodelistImport implements ShouldQueue
 				if ($ao->role == DomainController::NODE_HC)
 					$hub_id = $ao->id;
 
-				$this->no->addresses()->attach($ao,['role'=>$ao->role]);
+				$no->addresses()->attach($ao,['role'=>$ao->role]);
 
 			} catch (\Exception $e) {
 				Log::error(sprintf('%s:Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]);
@@ -256,9 +357,17 @@ class NodelistImport implements ShouldQueue
 			}
 		}
 
+		// Remove addresses not recorded;
+		Address::whereIN('id',$zo->addresses->except($us->system->addresses->pluck('id')->toArray())->diff($no->addresses)->pluck('id')->toArray())->delete();
+
+		if (crc16(substr($tocrc,0,-3)) == $file_crc)
+			DB::commit();
+		else
+			DB::rollBack();
+
 		fclose($fh);
 
-		if ($this->deletefile and $c)
+		if ($this->delete_file and $c)
 			unlink($file);
 
 		Log::info(sprintf('%s:Updated %d records from %d systems',self::LOGKEY,$p,$c));

app/Models/System.php

@@ -47,6 +47,11 @@ class System extends Model
 		return $this->hasOne(Setup::class);
 	}
 
+	public function users()
+	{
+		return $this->belongsToMany(User::class);
+	}
+
 	/**
 	 * This system is the ZC for the following zones
 	 */

app/Traits/CRC.php

@@ -4,17 +4,17 @@ namespace App\Traits;
 
 trait CRC
 {
-	private function CRC16USD_UPDATE($b,$crc): int
+	private function CRC16USD_UPDATE(int $b,int $crc): int
 	{
 		return (self::crc16usd_tab[(($crc >> 8) ^ $b) & 0xff] ^ (($crc & 0x00ff) << 8)) & 0xffff;
 	}
 
-	private function CRC32_FINISH($crc)
+	private function CRC32_FINISH(int $crc)
 	{
 		return ~$crc & 0xffffffff;
 	}
 
-	private function CRC32_UPDATE($b,$crc)
+	private function CRC32_UPDATE(int $b,int $crc)
 	{
 		return ((self::crc32_tab[($crc^$b) & 0xff] ^ (($crc>>8) & 0x00ffffff)) & 0xffffffff);
 	}
@@ -24,12 +24,12 @@ trait CRC
 		return ($this->ls_Protocol & self::LSZ_OPTCRC32) ? self::LSZ_INIT_CRC32 : self::LSZ_INIT_CRC16;
 	}
 
-	private function LSZ_FINISH_CRC($crc)
+	private function LSZ_FINISH_CRC(int $crc)
 	{
 		return ($this->ls_Protocol & self::LSZ_OPTCRC32) ? $this->CRC32_FINISH($crc) : $crc;
 	}
 
-	private function LSZ_UPDATE_CRC($b,$crc)
+	private function LSZ_UPDATE_CRC(int $b,int $crc)
 	{
 		return ($this->ls_Protocol & self::LSZ_OPTCRC32) ? $this->CRC32_UPDATE($b,$crc) : $this->CRC16USD_UPDATE($b,$crc);
 	}

database/migrations/2021_08_21_134156_add_phone.php

@@ -0,0 +1,32 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class AddPhone extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+		Schema::table('systems', function (Blueprint $table) {
+			$table->string('phone')->nullable();
+		});
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+		Schema::table('systems', function (Blueprint $table) {
+			$table->dropColumn('phone');
+		});
+    }
+}

resources/views/system/form-system.blade.php

@@ -11,6 +11,7 @@
 				<h2 class="cap">@if($o->exists) Update @else Add @endif System</h2>
 
 				<div class="row">
+					<!-- Name -->
 					<div class="col-4">
 						<label for="name" class="form-label">Name</label>
 						<div class="input-group has-validation">
@@ -26,6 +27,7 @@
 						</div>
 					</div>
 
+					<!-- Active -->
 					<div class="col-2">
 						<label for="active" class="form-label">Active</label>
 						<div class="input-group">
@@ -39,7 +41,8 @@
 						</div>
 					</div>
 
-					<div class="col-3">
+					<!-- ZeroTier ID -->
+					<div class="offset-3 col-3">
 						<label for="zt_id" class="form-label">ZeroTier ID</label>
 						<div class="input-group has-validation">
 							<span class="input-group-text"><i class="bi bi-shield-lock-fill"></i></span>
@@ -54,6 +57,7 @@
 				</div>
 
 				<div class="row">
+					<!-- Sysop -->
 					<div class="col-4">
 						<label for="sysop" class="form-label">Sysop</label>
 						<div class="input-group has-validation">
@@ -69,6 +73,7 @@
 						</div>
 					</div>
 
+					<!-- Location -->
 					<div class="col-8">
 						<label for="location" class="form-label">Location</label>
 						<div class="input-group has-validation">
@@ -86,6 +91,7 @@
 				</div>
 
 				<div class="row">
+					<!-- Mailer Details -->
 					<div class="col-12">
 						<h4>Mailer Details</h4>
 
@@ -119,10 +125,26 @@
 								</div>
 							</div>
 						</div>
+
+						<div class="pt-0 row">
+							<div class="col-4">
+								<label for="phone" class="form-label">Phone</label>
+								<div class="input-group has-validation">
+									<span class="input-group-text"><i class="bi bi-telephone-fill"></i></span>
+									<input type="text" class="form-control @error('phone') is-invalid @enderror" id="phone" placeholder="Phone" name="phone" value="{{ old('phone',$o->phone) }}" @cannot('admin',$o)disabled @endcannot>
+									<span class="invalid-feedback" role="alert">
+										@error('phone')
+											{{ $message }}
+										@enderror
+									</span>
+								</div>
+							</div>
+						</div>
 					</div>
 				</div>
 
 				<div class="row">
+					<!-- BBS Details -->
 					<div class="col-12">
 						<h4>BBS Details</h4>
 
@@ -161,6 +183,7 @@
 				</div>
 
 				<div class="row">
+					<!-- Notes -->
 					<div class="col-12">
 						<label for="notes" class="form-label">Notes</label>
 						<textarea class="form-control" rows=3 name="notes" placeholder="Notes..." @cannot('admin',$o)disabled @endcannot>{{ old('notes',$o->notes) }}</textarea>