clrghouz/app/Jobs/NodelistImport.php

<?php

namespace App\Jobs;

use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Str;

use App\Models\{Address,Domain,File,Mailer,Nodelist,System,Zone};
use App\Traits\Import as ImportTrait;

class NodelistImport implements ShouldQueue
{
	use Dispatchable,InteractsWithQueue,Queueable,SerializesModels,ImportTrait;

	protected const LOGKEY = 'JNI';
	private const importkey = 'nodelist';

	public const QUEUE = 'nodelist';

	private File|string $file;
	private ?string $domain;
	private bool $delete_file;
	private bool $delete_recs;
	private bool $testmode;
	private bool $ignore_crc;

	/**
	 * Import Nodelist from a file.
	 *
	 * A nodelist is treated as authoritative (it will add/update/delete details of existing entries), except where:
	 * + The system has a user owner (the user provides the authoritative information)
	 * + The system is a node of this instance (the admin provides the authoritative information)
	 *
	 * @param File|string $file
	 * @param string|null $domain
	 * @param bool $delete_recs
	 * @param bool $delete_file
	 * @param bool $test
	 * @param bool $ignore_crc
	 */
	public function __construct(File|string $file,string $domain=NULL,bool $delete_recs=FALSE,bool $delete_file=TRUE,bool $test=FALSE,bool $ignore_crc = FALSE)
	{
		$this->file = $file;
		$this->domain = $domain;
		$this->delete_file = $delete_file;
		$this->delete_recs = $delete_recs;
		$this->testmode = $test;
		$this->ignore_crc = $ignore_crc;

		$this->onQueue(self::QUEUE);
	}

	public function __get($key): mixed
	{
		switch ($key) {
			case 'jobname':
				return sprintf('%s-%s',$this->domain?->name,is_object($this->file) ? $this->file->name : $this->file);

			default:
				return NULL;
		}
	}

	/**
	 * Execute the job.
	 *
	 * @return void
	 * @throws \Exception
	 */
	public function handle()
	{
		// Get the file from the host
		$file = $this->getFileFromHost(self::importkey,$this->file);
		Log::debug(sprintf('%s:+ Loading file [%s].',static::LOGKEY,$file));
		$z = $this->openFile($file);

		$c = 0;
		$fh = NULL;
		while ($c < $z->count()) {
			// Nodelist files have an extension of numbers, between 1-365
			if (preg_match('/^.+\.[0-3][0-9][0-9]$/',$z->getNameIndex($c))) {
				$fh = $z->getStreamIndex($c);
				break;
			}

			$c++;
		}

		if (is_null($fh))
			throw new \Exception('Couldnt find nodelist in file');

		$lines = $this->getFileLines($fh);
		Log::debug(sprintf('%s:- Processing [%d] lines.',static::LOGKEY,$lines));
		// Rewind
		$fh = $z->getStreamIndex($c);

		// Line 1 tells us the nodelist and the CRC
		$line = stream_get_line($fh,0,"\r\n");

		$matches = [];
		if ((! preg_match('/^;A\ /',$line)) || (! preg_match('/^;A\ (.*)\ Nodelist for ([MTWFS][a-z]+,\ [JFMASOND][a-z]+\ [0-9]{1,2},\ [0-9]{4})\ --\ Day\ number\ ([0-9]+)\ :\ ([0-9a-f]+)$/',$line,$matches))) {
			Log::error(sprintf('%s:! Nodelist file [%d] is not valid?',self::LOGKEY,$this->file->id),['m'=>$matches,'l'=>$line]);

			throw new \Exception('Invalid nodelist for file: '.$this->file->id);
		}

		$file_crc = (int)$matches[4];
		$do = Domain::where('name',strtolower($matches[1] ?: $this->domain))->single();

		if (! $do) {
			Log::error(sprintf('%s:! Domain not found [%s].',static::LOGKEY,strtolower($matches[1] ?: $this->domain)));

			throw new \Exception('Nodelist Domain not found: '.$this->file);
		}

		$date = Carbon::createFromFormat('D, M d, Y H:i',$matches[2].'0:00');

		if ($date->dayOfYear !== (int)$matches[3]) {
			Log::error(sprintf('%s:! Nodelist date doesnt match [%d] (%d:%s).',static::LOGKEY,$matches[3],$date->dayOfYear,$date->format('Y-m-d')));

			throw new \Exception('Nodelist date doesnt match for file: '.$this->file->id);
		}

		Log::info(sprintf('%s:- Importing nodelist for [%s] dated [%s].',static::LOGKEY,$do->name,$date->format('Y-m-d')));

		// We'll only commit this if there were no errors
		DB::beginTransaction();
		$no = Nodelist::firstOrCreate(['date'=>$date,'domain_id'=>$do->id]);

		if ($this->delete_recs)
			$no->addresses()->detach();

		elseif ($no->addresses->count()) {
			Log::error(sprintf('%s:! Nodelist [%s] for [%s] has existing records [%d]',self::LOGKEY,$date,$do->name,$no->addresses->count()));

			return;
		}

		$mailer_binkp = Mailer::where('name','BINKP')->sole();
		$mailer_emsi = Mailer::where('name','EMSI')->sole();

		$p = $c = 0;

		$region = NULL;
		$host = NULL;
		$ishub = FALSE;
		$zo = NULL;
		$ho = NULL;
		$crc_check = '';

		while (! feof($fh)) {
			$line = stream_get_line($fh,0,"\r\n");
			$crc_check .= $line."\r\n";

			// Lines beginning with a semicolon(;) are comments
			if ((! $line) OR preg_match('/^;/',$line) OR ($line === chr(0x1a)))
				continue;

			// Remove any embedded CR and UTF-8 BOM
			$line = str_replace("\r",'',$line);
			$line = preg_replace('/^\x{feff}/u','',$line);
			$c++;

			Log::debug(sprintf('%s:| %s',self::LOGKEY,$line));

			$fields = str_getcsv(trim($line));

			// First field is either zone,region,host,hub,down,pvt (or blank)
			if ($fields[0] AND ! in_array($fields[0],Nodelist::definitions)) {
				Log::error(sprintf('%s:! Invalid field zero [%s] - IGNORING record (%s)',self::LOGKEY,$fields[0],$line));
				continue;
			}

			$node = 0;
			$role = NULL;

			switch ($fields[0]) {
				case 'Zone':
					Zone::unguard();
					$zo = Zone::firstOrNew([
						'zone_id'=>(int)$fields[1],
						'domain_id'=>$do->id,
						'active'=>TRUE,
					]);
					Zone::reguard();

					$region = 0;
					$host = 0;
					$ishub = FALSE;
					$ho = NULL;

					break;

				case 'Region':
					$region = (int)$fields[1];
					$host = (int)$fields[1];
					$ishub = FALSE;
					$ho = NULL;

					break;

				case 'Host':
					$host = (int)$fields[1];
					$ishub = FALSE;
					$ho = NULL;

					break;

				case 'Hub':
					$node = (int)$fields[1];
					$ishub = TRUE;
					$ho = NULL;

					break;

				case 'Pvt':
					$node = (int)$fields[1];
					$role = Address::NODE_PVT;

					break;

				case 'Hold':
					$node = (int)$fields[1];
					$role = Address::NODE_HOLD;

					break;

				case 'Down':
					$node = (int)$fields[1];
					$role = Address::NODE_DOWN;

					break;

				// Normal Node
				case '':
					$node = $fields[1];
					break;

				default:
					Log::error(sprintf('%s:! Unhandled first field [%s]',self::LOGKEY,$fields[0]));
					continue 2;
			}

			if (! $zo) {
				Log::error(sprintf('%s:! Zone NOT set, ignoring record...',self::LOGKEY));
				continue;
			}

			// Find or load an existing entry
			Address::unguard();
			$ao = Address::firstOrNew([
				'zone_id' => $zo?->id,
				'host_id' => $host,
				'node_id' => $node,
				'point_id' => 0,
				'active' => TRUE,
			]);
			Address::reguard();

			// If the address doesnt exist, we'll need to add in the region
			if (! $ao->exists)
				$ao->region_id = $region;

			// Address moved regions
			if ($ao->region_id && ($ao->region_id !== $region)) {
				Log::alert(sprintf('%s:%% Address [%s] changing regions [%d->%d]',self::LOGKEY,$ao->ftn,$ao->region_id,$region));
				$ao->region_id = $region;
			}

			// Hub details changed
			if ($ao->hub_id && ($ao->hub_id !== $ho?->id)) {
				Log::alert(sprintf('%s:%% Address [%s] changing hubs [%d->%s]',self::LOGKEY,$ao->ftn,$ao->hub_id,$ho?->id));
				$ao->hub_id = $ho?->id;
			}

			$sysop = trim(str_replace('_',' ',$fields[4]));
			$system = trim(str_replace('_',' ',$fields[2]));

			$protect = FALSE;
			if ($ao->exists) {
				Log::info(sprintf('%s:- Processing existing address [%s] (%d)',self::LOGKEY,$ao->ftn,$ao->id));

				// If the address is linked to a user's system, or our system, we'll not process it any further
				if (our_address()->contains($ao->id)) {
					Log::info(sprintf('%s:! Limiting update to an address belonging to me',self::LOGKEY));
					$protect = TRUE;

				} elseif ($ao->is_hosted) {
					Log::info(sprintf('%s:! Limiting update to a system managed by this site',self::LOGKEY));
					$protect = TRUE;

				} elseif ($ao->is_owned) {
					Log::info(sprintf('%s:! Limiting update to a system managed by a user',self::LOGKEY));
					$protect = TRUE;
				}

				$so = $ao->system;
			}

			// Flags
			$methods = collect();
			$address = '';
			for ($i=7;$i<count($fields);$i++) {
				$x = Str::of($fields[$i])->split('/:/');

				switch ($x->first()) {
					// Address
					case 'INA':
						$address = $x->get(1);
						break;

					// BINKP
					case 'IBN':
					case 'ITN':
						if ($x->first() === 'IBN') {
							$dport = 24554;
							$method = $mailer_binkp->id;

						} else {
							$dport = 60179;
							$method = $mailer_emsi->id;
						}

						switch ($x->count()) {
							case 1:
								$methods->put($method,['port'=>$dport]);
								break;

							case 2:
								$mp = is_numeric($x->get(1)) ? (int)$x->get(1) : $dport;
								$ma = is_numeric($xx=$x->get(1)) ? NULL : $xx;

								if ($ma && ($ma !== $address))
									$methods->put($method,['address'=>$ma,'port'=>$mp]);
								else
									$methods->put($method,['port'=>$mp]);

								break;

							case 3:
								$mp = (int)$x->get(2);
								$ma = $x->get(1);

								if ($ma && ($ma !== $address))
									$methods->put($method,['address'=>$ma,'port'=>$mp]);
								else
									$methods->put($method,['port'=>$mp]);

								break;
						}

						break;

					// Ignore
					case 'ZEC':
					case 'REC':
					case 'MO':
					case 'CM':
						break;

					default:
						Log::debug(sprintf('%s:! Not configured to handle flag [%s]',self::LOGKEY,$x->first()));
						continue 2;
				}
			}

			// If we are a zone/region/host record, then the system servicing that address may change
			switch ($ao->role_id) {
				case Address::NODE_ZC:
				case Address::NODE_RC:
				case Address::NODE_NC:
					// For new address, we'll need to add/link to a system
					if ($ao->exists) {
						if (($ao->system->address !== $address) || ($ao->system->name !== $system) || ($ao->system->sysop !== $sysop)) {
							Log::alert(sprintf('%s:! System has changed for [%s], no longer [%s]',self::LOGKEY,$ao->ftn,$ao->system->name));

							$ao->active = FALSE;
							$ao->save();

							$ao = $ao->replicate();
							$ao->active = TRUE;
							$ao->system_id = NULL;
						}
					}

					if (! $ao->system_id) {
						// When searching for a system, we prioritise in this order:
						// - the mailer address is correct
						// - the system name is correct
						$so = System::select('systems.*')
							->join('mailer_system',['mailer_system.system_id'=>'systems.id'])
							->when($address,
								fn($query)=>$query->where(
									fn($query)=>$query
										->where('systems.address',$address)
										->orWhere('mailer_system.address',$address)
								),
								fn($query)=>$query->where('systems.name',$system))
							->single();

						// If no system, we'll need to create one
						if (! $so) {
							Log::info(sprintf('%s:= New System for ZC/RC/NC [%s] - System [%s] Sysop [%s]',self::LOGKEY,$ao->ftn,$system,$sysop));
							$so = new System;
							$so->sysop = $sysop;
							$so->name = $system;
							$so->address = $address;
							$so->location = 'TBA';
							$so->notes = sprintf('Created by Nodelist Import: %d',$no->id);
							$so->active = TRUE;

							$so->save();
						}

						$ao->system_id = $so->id;
					}

					$ao->save();

					$no->addresses()->attach($ao,['role'=>$ao->role_id]);
					continue 2;
			}

			if (! $protect) {
				// Normal Node
				if ($ao->system_id && (
					(($ao->system->sysop === $sysop) || ($ao->system->name === $system))
					&& (($ao->system->address === $address) || $methods->pluck('address')->contains($address))))
				{
					Log::info(sprintf('%s:= Matched [%s] to existing system [%s] with address [%s]',self::LOGKEY,$ao->ftn,$ao->system->name,$ao->system->address));
					$so = $ao->system;

					// If the sysop name is different
					if ($so->sysop !== $sysop) {
						Log::alert(sprintf('%s:! Sysop Name changed for BBS [%s:%s] from [%s] to [%s]',self::LOGKEY,$so->id,$so->name,$so->sysop,$sysop));

						$so->sysop = $sysop;

					// We have the same name has changed (except for ZC/RC addresses)
					} elseif ($so->name !== $system) {
						Log::alert(sprintf('%s:! System Name changed for BBS [%s:%s] to [%s]',self::LOGKEY,$so->id,$so->name,$system));

						$so->name = $system;
					}

				// We'll search and see if we already have that system
				} else {
					Log::debug(sprintf('%s:- Looking for existing system [%s] with address [%s]',self::LOGKEY,$system,$address));
					// When searching for a system, we prioritise in this order:
					// - the mailer address is correct
					// - the system name is correct
					$so = System::select('systems.*')
						->join('mailer_system',['mailer_system.system_id'=>'systems.id'])
						->when($address,
							fn($query)=>$query->where(
								fn($query)=>$query
									->where('systems.address',$address)
									->orWhere('mailer_system.address',$address)
							),
							fn($query)=>$query->where('systems.name',$system))
						->single();

					if (! $so) {
						Log::debug(sprintf('%s:- Didnt match on system address, looking for System [%s] AND Sysop [%s]',self::LOGKEY,$system,$sysop));

						$so = System::where('name',$system)
							->where('sysop',$sysop)
							->firstOrNew();

					} else {
						Log::debug(sprintf('%s:- Matched on system [%d] address',self::LOGKEY,$so->id));
					}

					if ($so->exists)
						Log::info(sprintf('%s:= Linking address [%d:%d/%d] to [%s:%s]',self::LOGKEY,$zo->zone_id,$ao->host_id,$ao->node_id,$so->id,$so->name));
					else
						Log::info(sprintf('%s:= New System [%s] with FTN [%d:%d/%d]',self::LOGKEY,$system,$zo->zone_id,$ao->host_id,$ao->node_id));

					$so->name = $system;
					$so->sysop = $sysop;
					$so->active = TRUE;

					if (! $so->exists)
						$so->notes = sprintf('Created by Nodelist Import: %d',$no->id);
				}

				$so->phone = $fields[5] != '-Unpublished-' ? $fields[5] : NULL;
				$so->location = trim(str_replace('_',' ',$fields[3]));

				// Save the system record
				try {
					$so->save();

				} catch (\Exception $e) {
					Log::error(sprintf('%s:! Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]);

					DB::rollBack();
					throw new \Exception($e->getMessage());
				}

				$ao->system_id = $so->id;
			}

			if ($methods->count() && (! $ao->is_private)) {
				$methods->transform(function($item) {
					$item['active'] = Arr::get($item,'active',TRUE);
					return $item;
				});

				$so->mailers()->sync($methods);
			}

			// If our zone didnt exist, we'll create it with this system
			if (! $zo->exists) {
				$zo->system_id = $ao->system_id;
				$zo->save();
			}

			$ao->zone_id = $zo->id;
			$ao->role = $role;

			if ($ao->getDirty())
				$p++;

			try {
				$so->addresses()->save($ao);

				// If we were the hub
				if ($ishub) {
					$ho = $ao;
					$ishub = FALSE;
				}

				$no->addresses()->attach($ao,['role'=>($ao->role_id & $role)]);

			} catch (\Exception $e) {
				Log::error(sprintf('%s:! Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]);

				DB::rollBack();
				throw new \Exception($e->getMessage());
			}

			if (! ($c % 100)) {
				Log::notice(sprintf('%s:= Processed [%s] records',self::LOGKEY,$c),['memory'=>memory_get_usage(TRUE)]);
			}
		}

		// Remove addresses not recorded;
		$no->load('addresses');

		$remove = $zo
			->addresses->pluck('id')
			->diff($no->addresses->pluck('id'))
			->diff(our_address($do)->pluck('id'))
			->diff(our_nodes($do)->pluck('id'));

		$remove = Address::whereIn('id',$remove)->get();
		Log::alert(sprintf('%s:%% Deleting [%d] addresses [%s]',self::LOGKEY,$remove->count(),$remove->pluck('ftn2d')->join(',')));

		Address::whereIN('id',$remove->pluck('id'))->update(['active'=>FALSE]);
		Address::whereIN('id',$remove->pluck('id'))->delete();

		Log::info(sprintf('%s:= Updated %d AKA records from %d Systems',self::LOGKEY,$p,$c));
		$crc = crc16(substr($crc_check,0,-3));

		if ((! $this->testmode) && ($this->ignore_crc || ($crc === $file_crc))) {
			Log::info(sprintf('%s:= Committing nodelist',self::LOGKEY));
			DB::commit();

			if ($this->delete_file and $c)
				unlink($file);

		} else {
			Log::error(sprintf('%s:! Rolling back nodelist, CRC doesnt match [%s != %s] or TEST mode',self::LOGKEY,$crc,$file_crc));
			DB::rollBack();
		}

		fclose($fh);
	}
}