file = $file; $this->domain = $domain; $this->delete_file = $delete_file; $this->delete_recs = $delete_recs; $this->testmode = $test; $this->ignore_crc = $ignore_crc; $this->onQueue(self::QUEUE); } public function __get($key): mixed { switch ($key) { case 'jobname': return ($this->domain) ? sprintf('%s-%s',$this->domain?->name,(is_object($this->file) ? $this->file->name : $this->file)) : (is_object($this->file) ? $this->file->name : $this->file); default: return NULL; } } /** * Execute the job. * * @return void * @throws \Exception */ public function handle() { // Get the file from the host $file = $this->getFileFromHost(self::importkey,$this->file); Log::debug(sprintf('%s:+ Loading file [%s].',static::LOGKEY,$file)); $z = $this->openFile($file); $c = 0; $fh = NULL; while ($c < $z->count()) { // Nodelist files have an extension of numbers, between 1-365 if (preg_match('/^.+\.[0-3][0-9][0-9]$/',$z->getNameIndex($c))) { $fh = $z->getStreamIndex($c); break; } $c++; } if (is_null($fh)) throw new \Exception('Couldnt find nodelist in file'); $lines = $this->getFileLines($fh); Log::debug(sprintf('%s:- Processing [%d] lines.',static::LOGKEY,$lines)); // Rewind $fh = $z->getStreamIndex($c); // Line 1 tells us the nodelist and the CRC $line = stream_get_line($fh,0,"\r\n"); $matches = []; if ((! preg_match('/^;A\ /',$line)) || (! preg_match('/^;A\ (.*)\ Nodelist for ([MTWFS][a-z]+,\ [JFMASOND][a-z]+\ [0-9]{1,2},\ [0-9]{4})\ --\ Day\ number\ ([0-9]+)\ :\ ([0-9a-f]+)$/',$line,$matches))) { Log::error(sprintf('%s:! Nodelist file [%d] is not valid?',self::LOGKEY,$this->file->id),['m'=>$matches,'l'=>$line]); throw new \Exception('Invalid nodelist for file: '.$this->file->id); } $file_crc = (int)$matches[4]; $do = Domain::where('name',strtolower($matches[1] ?: $this->domain))->single(); if (! $do) { Log::error(sprintf('%s:! Domain not found [%s].',static::LOGKEY,strtolower($matches[1] ?: $this->domain))); throw new \Exception('Nodelist Domain not found: '.$this->file); } $date = Carbon::createFromFormat('D, M d, Y H:i',$matches[2].'0:00'); if ($date->dayOfYear !== (int)$matches[3]) { Log::error(sprintf('%s:! Nodelist date doesnt match [%d] (%d:%s).',static::LOGKEY,$matches[3],$date->dayOfYear,$date->format('Y-m-d'))); throw new \Exception('Nodelist date doesnt match for file: '.$this->file->id); } Log::info(sprintf('%s:- Importing nodelist for [%s] dated [%s].',static::LOGKEY,$do->name,$date->format('Y-m-d'))); // We'll only commit this if there were no errors DB::beginTransaction(); $no = Nodelist::firstOrCreate(['date'=>$date,'domain_id'=>$do->id]); if ($this->delete_recs) $no->addresses()->detach(); elseif ($no->addresses->count()) { Log::error(sprintf('%s:! Nodelist [%s] for [%s] has existing records [%d]',self::LOGKEY,$date,$do->name,$no->addresses->count())); return; } $mailer_binkp = Mailer::where('name','BINKP')->sole(); $mailer_emsi = Mailer::where('name','EMSI')->sole(); $p = $c = 0; $region = NULL; $host = NULL; $ishub = FALSE; $zo = NULL; $ho = NULL; $crc_check = ''; while (! feof($fh)) { $line = stream_get_line($fh,0,"\r\n"); $crc_check .= $line."\r\n"; // Lines beginning with a semicolon(;) are comments if ((! $line) OR preg_match('/^;/',$line) OR ($line === chr(0x1a))) continue; // Remove any embedded CR and UTF-8 BOM $line = str_replace("\r",'',$line); $line = preg_replace('/^\x{feff}/u','',$line); $c++; Log::debug(sprintf('%s:| %s',self::LOGKEY,$line)); $fields = str_getcsv(trim($line)); // First field is either zone,region,host,hub,down,pvt (or blank) if ($fields[0] AND ! in_array($fields[0],Nodelist::definitions)) { Log::error(sprintf('%s:! Invalid field zero [%s] - IGNORING record (%s)',self::LOGKEY,$fields[0],$line)); continue; } $node = 0; $role = NULL; switch ($fields[0]) { case 'Zone': Zone::unguard(); $zo = Zone::firstOrNew([ 'zone_id'=>(int)$fields[1], 'domain_id'=>$do->id, 'active'=>TRUE, ]); Zone::reguard(); $region = 0; $host = 0; $ishub = FALSE; $ho = NULL; break; case 'Region': $region = (int)$fields[1]; $host = (int)$fields[1]; $ishub = FALSE; $ho = NULL; break; case 'Host': $host = (int)$fields[1]; $ishub = FALSE; $ho = NULL; break; case 'Hub': $node = (int)$fields[1]; $ishub = TRUE; $ho = NULL; break; case 'Pvt': $node = (int)$fields[1]; $role = Address::NODE_PVT; break; case 'Hold': $node = (int)$fields[1]; $role = Address::NODE_HOLD; break; case 'Down': $node = (int)$fields[1]; $role = Address::NODE_DOWN; break; // Normal Node case '': $node = $fields[1]; break; default: Log::error(sprintf('%s:! Unhandled first field [%s]',self::LOGKEY,$fields[0])); continue 2; } if (! $zo) { Log::error(sprintf('%s:! Zone NOT set, ignoring record...',self::LOGKEY)); continue; } // Find or load an existing entry Address::unguard(); $ao = Address::firstOrNew([ 'zone_id' => $zo?->id, 'host_id' => $host, 'node_id' => $node, 'point_id' => 0, 'active' => TRUE, ]); Address::reguard(); // If the address doesnt exist, we'll need to add in the region if (! $ao->exists) $ao->region_id = $region; // Address moved regions if ($ao->region_id && ($ao->region_id !== $region)) { Log::alert(sprintf('%s:%% Address [%s] changing regions [%d->%d]',self::LOGKEY,$ao->ftn,$ao->region_id,$region)); $ao->region_id = $region; } // Hub details changed if ($ao->hub_id && ($ao->hub_id !== $ho?->id)) { Log::alert(sprintf('%s:%% Address [%s] changing hubs [%d->%s]',self::LOGKEY,$ao->ftn,$ao->hub_id,$ho?->id)); $ao->hub_id = $ho?->id; } $sysop = trim(str_replace('_',' ',$fields[4])); $system = trim(str_replace('_',' ',$fields[2])); $protect = FALSE; if ($ao->exists) { Log::info(sprintf('%s:- Processing existing address [%s] (%d)',self::LOGKEY,$ao->ftn,$ao->id)); // If the address is linked to a user's system, or our system, we'll not process it any further if (our_address()->contains($ao->id)) { Log::info(sprintf('%s:! Limiting update to an address belonging to me',self::LOGKEY)); $protect = TRUE; } elseif ($ao->is_hosted) { Log::info(sprintf('%s:! Limiting update to a system managed by this site',self::LOGKEY)); $protect = TRUE; } elseif ($ao->is_owned) { Log::info(sprintf('%s:! Limiting update to a system managed by a user',self::LOGKEY)); $protect = TRUE; } $so = $ao->system; } // Flags $methods = collect(); $address = ''; for ($i=7;$isplit('/:/'); switch ($x->first()) { // Address case 'INA': $address = $x->get(1); break; // BINKP case 'IBN': case 'ITN': if ($x->first() === 'IBN') { $dport = 24554; $method = $mailer_binkp->id; } else { $dport = 60179; $method = $mailer_emsi->id; } switch ($x->count()) { case 1: $methods->put($method,['port'=>$dport]); break; case 2: $mp = is_numeric($x->get(1)) ? (int)$x->get(1) : $dport; $ma = is_numeric($xx=$x->get(1)) ? NULL : $xx; if ($ma && ($ma !== $address)) $methods->put($method,['address'=>$ma,'port'=>$mp]); else $methods->put($method,['port'=>$mp]); break; case 3: $mp = (int)$x->get(2); $ma = $x->get(1); if ($ma && ($ma !== $address)) $methods->put($method,['address'=>$ma,'port'=>$mp]); else $methods->put($method,['port'=>$mp]); break; } break; // Ignore case 'ZEC': case 'REC': case 'MO': case 'CM': break; default: Log::debug(sprintf('%s:! Not configured to handle flag [%s]',self::LOGKEY,$x->first())); continue 2; } } // If we are a zone/region/host record, then the system servicing that address may change switch ($ao->role_id) { case Address::NODE_ZC: case Address::NODE_RC: case Address::NODE_NC: // For new address, we'll need to add/link to a system if ($ao->exists) { if (($ao->system->address !== $address) || ($ao->system->name !== $system) || ($ao->system->sysop !== $sysop)) { Log::alert(sprintf('%s:! System has changed for [%s], no longer [%s]',self::LOGKEY,$ao->ftn,$ao->system->name)); $ao->active = FALSE; $ao->save(); $ao = $ao->replicate(); $ao->active = TRUE; $ao->system_id = NULL; } } if (! $ao->system_id) { // When searching for a system, we prioritise in this order: // - the mailer address is correct // - the system name is correct $so = System::select('systems.*') ->join('mailer_system',['mailer_system.system_id'=>'systems.id']) ->when($address, fn($query)=>$query->where( fn($query)=>$query ->where('systems.address',$address) ->orWhere('mailer_system.address',$address) ), fn($query)=>$query->where('systems.name',$system)) ->single(); // If no system, we'll need to create one if (! $so) { Log::info(sprintf('%s:= New System for ZC/RC/NC [%s] - System [%s] Sysop [%s]',self::LOGKEY,$ao->ftn,$system,$sysop)); $so = new System; $so->sysop = $sysop; $so->name = $system; $so->address = $address; $so->location = 'TBA'; $so->notes = sprintf('Created by Nodelist Import: %d',$no->id); $so->active = TRUE; $so->save(); } $ao->system_id = $so->id; } $ao->save(); $no->addresses()->attach($ao,['role'=>$ao->role_id]); continue 2; } if (! $protect) { // Normal Node if ($ao->system_id && ( (($ao->system->sysop === $sysop) || ($ao->system->name === $system)) && (($ao->system->address === $address) || $methods->pluck('address')->contains($address)))) { Log::info(sprintf('%s:= Matched [%s] to existing system [%s] with address [%s]',self::LOGKEY,$ao->ftn,$ao->system->name,$ao->system->address)); $so = $ao->system; // If the sysop name is different if ($so->sysop !== $sysop) { Log::alert(sprintf('%s:! Sysop Name changed for BBS [%s:%s] from [%s] to [%s]',self::LOGKEY,$so->id,$so->name,$so->sysop,$sysop)); $so->sysop = $sysop; // We have the same name has changed (except for ZC/RC addresses) } elseif ($so->name !== $system) { Log::alert(sprintf('%s:! System Name changed for BBS [%s:%s] to [%s]',self::LOGKEY,$so->id,$so->name,$system)); $so->name = $system; } // We'll search and see if we already have that system } else { Log::debug(sprintf('%s:- Looking for existing system [%s] with address [%s]',self::LOGKEY,$system,$address)); // When searching for a system, we prioritise in this order: // - the mailer address is correct // - the system name is correct $so = System::select('systems.*') ->join('mailer_system',['mailer_system.system_id'=>'systems.id']) ->when($address, fn($query)=>$query->where( fn($query)=>$query ->where('systems.address',$address) ->orWhere('mailer_system.address',$address) ), fn($query)=>$query->where('systems.name',$system)) ->single(); if (! $so) { Log::debug(sprintf('%s:- Didnt match on system address, looking for System [%s] AND Sysop [%s]',self::LOGKEY,$system,$sysop)); $so = System::where('name',$system) ->where('sysop',$sysop) ->firstOrNew(); } else { Log::debug(sprintf('%s:- Matched on system [%d] address',self::LOGKEY,$so->id)); } if ($so->exists) Log::info(sprintf('%s:= Linking address [%d:%d/%d] to [%s:%s]',self::LOGKEY,$zo->zone_id,$ao->host_id,$ao->node_id,$so->id,$so->name)); else Log::info(sprintf('%s:= New System [%s] with FTN [%d:%d/%d]',self::LOGKEY,$system,$zo->zone_id,$ao->host_id,$ao->node_id)); $so->name = $system; $so->sysop = $sysop; $so->active = TRUE; if (! $so->exists) $so->notes = sprintf('Created by Nodelist Import: %d',$no->id); } $so->phone = $fields[5] != '-Unpublished-' ? $fields[5] : NULL; $so->location = trim(str_replace('_',' ',$fields[3])); // Save the system record try { $so->save(); } catch (\Exception $e) { Log::error(sprintf('%s:! Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]); DB::rollBack(); throw new \Exception($e->getMessage()); } $ao->system_id = $so->id; } if ($methods->count() && (! $ao->is_private)) { $methods->transform(function($item) { $item['active'] = Arr::get($item,'active',TRUE); return $item; }); $so->mailers()->sync($methods); } // If our zone didnt exist, we'll create it with this system if (! $zo->exists) { $zo->system_id = $ao->system_id; $zo->save(); } $ao->zone_id = $zo->id; $ao->role = $role; if ($ao->getDirty()) $p++; try { $so->addresses()->save($ao); // If we were the hub if ($ishub) { $ho = $ao; $ishub = FALSE; } $no->addresses()->attach($ao,['role'=>($ao->role_id & $role)]); } catch (\Exception $e) { Log::error(sprintf('%s:! Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]); DB::rollBack(); throw new \Exception($e->getMessage()); } if (! ($c % 100)) { Log::notice(sprintf('%s:= Processed [%s] records',self::LOGKEY,$c),['memory'=>memory_get_usage(TRUE)]); } } // Remove addresses not recorded; $no->load('addresses'); $remove = $zo ->addresses ->filter(fn($item)=>(! $item->point_id)) ->pluck('id') ->diff($no->addresses->pluck('id')) ->diff(our_address($do)->pluck('id')) ->diff(our_nodes($do)->pluck('id')); $remove = Address::whereIn('id',$remove)->get(); Log::alert(sprintf('%s:%% Deleting [%d] addresses [%s]',self::LOGKEY,$remove->count(),$remove->pluck('ftn2d')->join(','))); Address::whereIN('id',$remove->pluck('id'))->update(['active'=>FALSE]); Address::whereIN('id',$remove->pluck('id'))->delete(); Log::info(sprintf('%s:= Updated %d AKA records from %d Systems',self::LOGKEY,$p,$c)); $crc = crc16(substr($crc_check,0,-3)); if ((! $this->testmode) && ($this->ignore_crc || ($crc === $file_crc))) { Log::info(sprintf('%s:= Committing nodelist',self::LOGKEY)); DB::commit(); if ($this->delete_file and $c) unlink($file); } else { Log::error(sprintf('%s:! Rolling back nodelist, CRC doesnt match [%s != %s] or TEST mode',self::LOGKEY,$crc,$file_crc)); DB::rollBack(); } fclose($fh); } }