Change the way we figure out zones in packets, some packet testing, fix Echomail import

app/Classes/FTN.php

@@ -2,11 +2,11 @@
 
 namespace App\Classes;
 
-use App\Models\{Address,Domain};
+use App\Models\{Address,Zone};
 
 abstract class FTN
 {
-	protected ?Domain $domain;				// Domain the packet is from
+	protected ?Zone $zone;				// Zone the packet is from
 
 	public function __get($key)
 	{
@@ -17,7 +17,7 @@ abstract class FTN
 					$this->fn,
 					$this->ff,
 					$this->fp,
-				).($this->domain ? sprintf('@%s',$this->domain->name) : '');
+				).($this->zone ? sprintf('@%s',$this->zone->domain->name) : '');
 
 			case 'tftn':
 				return sprintf('%d:%d/%d.%d',
@@ -25,7 +25,7 @@ abstract class FTN
 					$this->tn,
 					$this->tf,
 					$this->tp,
-				).($this->domain ? sprintf('@%s',$this->domain->name) : '');
+				).($this->zone ? sprintf('@%s',$this->zone->domain->name) : '');
 
 			case 'fftn_o':
 				return Address::findFTN($this->fftn);

app/Classes/FTN/Message.php

@@ -10,7 +10,7 @@ use Illuminate\Support\Facades\Validator;
 use Illuminate\Validation\Validator as ValidatorResult;
 
 use App\Classes\FTN as FTNBase;
-use App\Models\{Address,Domain};
+use App\Models\{Address,Zone};
 use App\Rules\TwoByteInteger;
 use App\Traits\EncodeUTF8;
 
@@ -163,9 +163,9 @@ class Message extends FTNBase
 		0xfc => 0x207f, 0xfd => 0x00b2, 0xfe => 0x25a0, 0xff => 0x00a0,
 	];
 
-	public function __construct(Domain $domain=NULL)
+	public function __construct(Zone $zone=NULL)
 	{
-		$this->domain = $domain;
+		$this->zone = $zone;
 
 		$this->header = [];
 		$this->kludge = collect();
@@ -204,9 +204,9 @@ class Message extends FTNBase
 	 * @return Message
 	 * @throws InvalidPacketException
 	 */
-	public static function parseMessage(string $msg,Domain $domain=NULL): self
+	public static function parseMessage(string $msg,Zone $zone=NULL): self
 	{
-		$o = new self($domain);
+		$o = new self($zone);
 
 		try {
 			$o->header = unpack(self::unpackheader(self::header),substr($msg,0,self::HEADER_LEN));
@@ -246,7 +246,7 @@ class Message extends FTNBase
 
 		$o->unpackMessage(substr($msg,self::HEADER_LEN+$ptr));
 
-		if (($x=$o->validate($domain))->fails()) {
+		if (($x=$o->validate())->fails()) {
 			Log::debug('Message fails validation',['result'=>$x->errors()]);
 			//throw new \Exception('Message validation fails:'.join(' ',$x->errors()->all()));
 		}
@@ -651,7 +651,7 @@ class Message extends FTNBase
 	 * Extract information out of the message text.
 	 *
 	 * @param string $message
-	 * @throws InvalidPacketException
+	 * @throws \Exception
 	 */
 	public function unpackMessage(string $message): void
 	{
@@ -690,11 +690,13 @@ class Message extends FTNBase
 					preg_match('/^.*\((.*)\)$/',$this->origin,$matches);
 
 					// Double check we have an address in the origin line
-					if (! Arr::get($matches,1))
+					if (! Arr::get($matches,1)) {
-						throw new InvalidPacketException('No address in Origin?');
+						Log::error(sprintf('%s:! Origin line doesnt have an address',self::LOGKEY));
 
+					} else {
 						// Double check, our src and origin match
 						$this->src = Address::parseFTN($matches[1]);
+					}
 
 					// We'll double check our FTN
 					if ($this->isNetmail() && (($this->src['n'] !== $this->fn) || ($this->src['f'] !== $this->ff))) {
@@ -758,6 +760,9 @@ class Message extends FTNBase
 			elseif ($t = $this->kludge('PATH: ',$v))
 				$this->path->push($t);
 
+			elseif ($t = $this->kludge('SEEN-BY: ',$v))
+				$this->seenby->push($t);
+
 			// To Point: <SOH>TOPT <point number><CR>
 			elseif ($t = $this->kludge('TOPT ',$v))
 				$this->point['dst'] = $t;
@@ -777,6 +782,14 @@ class Message extends FTNBase
 		$m = [];
 		if ($this->msgid && preg_match('#([0-9]+:[0-9]+/[0-9]+)?\.?([0-9]+)?@?([A-Za-z-_~]+)?\ +#',$this->msgid,$m)) {
 			$this->src = Address::parseFTN($m[1].((isset($m[2]) && $m[2] != '') ? '.'.$m[2] : '').(isset($m[3]) ? '@'.$m[3] : ''));
+
+		// Without a MSGID, get our domain from the origin
+		} elseif ($this->origin && preg_match('#\(([0-9]+:[0-9]+/[0-9]+)?\.?([0-9]+)?@?([A-Za-z-_~]+)?\)$#',$this->origin,$m)) {
+			$this->src = Address::parseFTN($m[1].((isset($m[2]) && $m[2] != '') ? '.'.$m[2] : '').(isset($m[3]) ? '@'.$m[3] : ''));
+
+		// Otherwise get it from our zone object and packet header
+		} elseif ($this->zone) {
+			$this->src = Address::parseFTN(sprintf('%d:%d/%d.%d',$this->zone->zone_id,$this->fn,$this->ff,$this->fp));
 		}
 
 		// Parse SEEN-BY
@@ -793,7 +806,7 @@ class Message extends FTNBase
 	 *
 	 * @return \Illuminate\Contracts\Validation\Validator
 	 */
-	public function validate(Domain $domain=NULL): ValidatorResult
+	public function validate(): ValidatorResult
 	{
 		// Check lengths
 		$validator = Validator::make([
@@ -820,18 +833,16 @@ class Message extends FTNBase
 			'flags' => 'required|numeric',
 			'cost' => 'required|numeric',
 			'echoarea' => 'nullable|max:'.self::AREATAG_LEN,
-			'ozone' => ['required',$this->domain ? 'in:'.$x=$this->domain->zones->pluck('zone_id')->join(','): ''],
+			'ozone' => ['required'],
-			'dzone' => ['required',$this->domain ? 'in:'.$x : '']
+			'dzone' => ['required']
 		]);
 
-		if ($domain) {
 		$validator->after(function($validator) {
 			if (! $this->fboss_o)
 				$validator->errors()->add('from',sprintf('Undefined Node [%s] sent message.',$this->fboss));
 			if (! $this->tboss_o)
 				$validator->errors()->add('to',sprintf('Undefined Node [%s] for destination.',$this->tboss));
 		});
-		}
 
 		if ($validator->fails())
 			$this->errors = $validator;

app/Classes/FTN/Packet.php

@@ -10,7 +10,7 @@ use Illuminate\Support\Facades\Redis;
 use Symfony\Component\HttpFoundation\File\File;
 
 use App\Classes\FTN as FTNBase;
-use App\Models\{Address,Domain,Setup,Software};
+use App\Models\{Address,Setup,Software,Zone};
 
 class Packet extends FTNBase implements \Iterator, \Countable
 {
@@ -110,11 +110,12 @@ class Packet extends FTNBase implements \Iterator, \Countable
 	 * Open a packet file
 	 *
 	 * @param File $file
-	 * @param Domain|null $domain
+	 * @param Zone|null $zone
+	 * @param bool $use_redis
 	 * @return Packet
 	 * @throws InvalidPacketException
 	 */
-	public static function open(File $file,Domain $domain=NULL): self
+	public static function open(File $file,Zone $zone=NULL,bool $use_redis=TRUE): self
 	{
 		Log::debug(sprintf('%s:+ Opening Packet [%s]',self::LOGKEY,$file));
 
@@ -134,6 +135,8 @@ class Packet extends FTNBase implements \Iterator, \Countable
 			throw new InvalidPacketException('Not a type 2 packet: '.$version);
 
 		$o = new self;
+		$o->zone = $zone;
+		$o->use_redis = $use_redis;
 		$o->name = (string)$file;
 		$o->header = unpack(self::unpackheader(self::v2header),$header);
 
@@ -151,6 +154,10 @@ class Packet extends FTNBase implements \Iterator, \Countable
 		else if (! strlen($x))
 			throw new InvalidPacketException('No message in packet: '.bin2hex($x));
 
+		// If zone is null, we'll take the zone from the packet
+		if (! $o->zone)
+			$o->zone = Zone::where('zone_id',$o->fz)->where('default',TRUE)->single();
+
 		$buf_ptr = 0;
 		$message = '';
 		$readbuf = '';
@@ -174,7 +181,7 @@ class Packet extends FTNBase implements \Iterator, \Countable
 				$last .= substr($readbuf,0,2);
 
 				if (($end=strpos($last,"\x00\x02\x00",$buf_ptr)) !== FALSE) {
-					$o->parseMessage(substr($message,0,$end-2),$domain);
+					$o->parseMessage(substr($message,0,$end-2));
 					$last = '';
 					$message = '';
 					$buf_ptr = 1+$end;
@@ -217,13 +224,13 @@ class Packet extends FTNBase implements \Iterator, \Countable
 			}
 
 			// Look for the next message
-			$o->parseMessage($message,$domain);
+			$o->parseMessage($message);
 			$message = '';
 		}
 
 		// If our message is still set, then we have an unprocessed message
 		if ($message)
-			$o->parseMessage($message,$domain);
+			$o->parseMessage($message);
 
 		return $o;
 	}
@@ -413,12 +420,11 @@ class Packet extends FTNBase implements \Iterator, \Countable
 	 * Parse a message in a mail packet
 	 *
 	 * @param string $message
-	 * @param Domain|null $domain
 	 * @throws InvalidPacketException
 	 */
-	private function parseMessage(string $message,Domain $domain=NULL): void
+	private function parseMessage(string $message): void
 	{
-		$msg = Message::parseMessage($message,$domain);
+		$msg = Message::parseMessage($message,$this->zone);
 
 		// If the message is invalid, we'll ignore it
 		if ($msg->errors && (
@@ -432,8 +438,9 @@ class Packet extends FTNBase implements \Iterator, \Countable
 
 		} else {
 			if ($this->use_redis) {
-				Redis::set($msg->msgid ?: sprintf('%s %s',$msg->fftn,Carbon::now()->timestamp),serialize($msg));
+				$key = $msg->msgid ?: sprintf('%s %s',$msg->fftn,Carbon::now()->timestamp);
-				$this->messages->push($msg->msgid);
+				Redis::set($key,serialize($msg));
+				$this->messages->push($key);
 
 			} else {
 				$this->messages->push($msg);

app/Console/Commands/EchomailDump.php

@@ -0,0 +1,35 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+
+use App\Models\Echomail;
+
+class EchomailDump extends Command
+{
+
+	/**
+	 * The name and signature of the console command.
+	 *
+	 * @var string
+	 */
+	protected $signature = 'echomail:dump {id}';
+
+	/**
+	 * The console command description.
+	 *
+	 * @var string
+	 */
+	protected $description = 'Echomail Dump';
+
+	/**
+	 * Execute the console command.
+	 *
+	 * @return mixed
+	 */
+	public function handle()
+	{
+		dump(Echomail::findOrFail($this->argument('id')));
+	}
+}

app/Console/Commands/PacketInfo.php

@@ -6,8 +6,7 @@ use Illuminate\Console\Command;
 use Symfony\Component\HttpFoundation\File\File;
 
 use App\Classes\FTN\Packet;
-use App\Jobs\ProcessPacket as Job;
+use App\Models\Zone;
-use App\Models\Domain;
 
 class PacketInfo extends Command
 {
@@ -16,7 +15,7 @@ class PacketInfo extends Command
 	 *
 	 * @var string
 	 */
-	protected $signature = 'packet:info {pkt : Packet to process} {domain : Domain the packet is from}';
+	protected $signature = 'packet:info {pkt : Packet to process} {zone? : Zone the packet is from}';
 
 	/**
 	 * The console command description.
@@ -34,9 +33,9 @@ class PacketInfo extends Command
 	public function handle()
 	{
 		$f = new File($this->argument('pkt'));
-		$d = Domain::where('name',$this->argument('domain'))->singleOrFail();
+		$z = $this->argument('zone') ? Zone::where('zone_id',$this->argument('zone'))->singleOrFail() : NULL;
 
-		$pkt = Packet::open($f,$d);
+		$pkt = Packet::open($f,$z);
 
 		$this->info(sprintf('Packet Type: %s',$pkt->type));
 		$this->info(sprintf('From: %s to %s',$pkt->fftn,$pkt->tftn));

app/Console/Commands/ProcessPacket.php

@@ -16,7 +16,7 @@ class ProcessPacket extends Command
 	 *
 	 * @var string
 	 */
-	protected $signature = 'packet:process {pkt : Packet to process} {domain : Domain the packet is from}';
+	protected $signature = 'packet:process {pkt : Packet to process} {domain? : Domain the packet is from}';
 
 	/**
 	 * The console command description.
@@ -34,14 +34,14 @@ class ProcessPacket extends Command
 	public function handle()
 	{
 		$f = new File($this->argument('pkt'));
-		$d = Domain::where('name',$this->argument('domain'))->singleOrFail();
+		$d = $this->argument('domain') ? Domain::where('name',$this->argument('domain'))->singleOrFail() : NULL;
 
 		foreach (Packet::open($f,$d) as $msg) {
 			// @todo Quick check that the packet should be processed by us.
 			// @todo validate that the packet's zone is in the domain.
 
 			// Dispatch job.
-			Job::dispatch($msg);
+			Job::dispatchSync($msg);
 		}
 	}
 }

app/Jobs/ProcessPacket.php

@@ -200,10 +200,10 @@ class ProcessPacket implements ShouldQueue
 			$o->msgid = $this->msg->msgid;
 
 			$o->msg = $this->msg->message_src;
-			$o->path = $this->msg->pathaddress->pluck('id')->jsonSerialize();
+			$o->path = $this->msg->pathaddress->jsonSerialize();
 			$o->rogue_path = $this->msg->rogue_path->jsonSerialize();
-			$o->seenby = $this->msg->seenaddress->pluck('id')->jsonSerialize();
+			$o->seenby = $this->msg->seenaddress->jsonSerialize();
-			$o->rogue_seen = $this->msg->rogue_path->jsonSerialize();
+			$o->rogue_seen = $this->msg->rogue_seen->jsonSerialize();
 			$o->toexport = TRUE;
 
 			$o->save();

tests/Feature/PacketTest.php

@@ -0,0 +1,118 @@
+<?php
+
+namespace Tests\Feature;
+
+use Symfony\Component\HttpFoundation\File\File;
+use Tests\TestCase;
+
+use App\Classes\FTN\Packet;
+use App\Models\{Address,Domain,System,Zone};
+
+class PacketTest extends TestCase
+{
+	private System $so;
+	private Domain $do;
+
+	private function init()
+	{
+		System::unguard();
+		Domain::unguard();
+		$this->so = System::firstOrCreate(['name'=>'test','sysop'=>'sysop','location'=>'location','active'=>TRUE]);
+		$this->do = Domain::firstOrCreate(['name'=>'test','active'=>TRUE]);
+	}
+
+	public function test_nomsgid_origin()
+	{
+		$this->init();
+
+		Zone::unguard();
+		Address::unguard();
+		$zo = Zone::firstOrCreate(['zone_id'=>21,'default'=>TRUE,'active'=>TRUE,'domain_id'=>$this->do->id,'system_id'=>$this->so->id]);
+		$src = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>3,'node_id'=>151,'point_id'=>0,'active'=>TRUE,'system_id'=>$this->so->id]);
+		$hub = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>1,'node_id'=>1,'point_id'=>0,'active'=>TRUE,'system_id'=>$this->so->id]);
+		$ao = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>1,'node_id'=>4,'point_id'=>0,'active'=>TRUE,'system_id'=>$this->so->id]);
+
+		// This packet has an incorrect zone in the Origin
+		$f = new File(__DIR__.'/data/test_nomsgid_origin.pkt');
+		$pkt = Packet::open($f);
+
+		$this->assertEquals(1,$pkt->count());
+
+		$messages = FALSE;
+		foreach ($pkt as $msg) {
+			$messages = TRUE;
+			$this->assertNotTrue($msg->isNetmail());
+
+			$this->assertNotFalse($msg->pathaddress->search($hub->id));
+
+			$this->assertCount(1,$msg->rogue_path);
+			$this->assertNotFalse($msg->rogue_path->search('21:999/1'));
+
+			$this->assertCount(3,$msg->rogue_seen);
+			$this->assertNotFalse($msg->rogue_seen->search('21:999/1'));
+			$this->assertNotFalse($msg->rogue_seen->search('21:999/999'));
+		}
+
+		$this->assertTrue($messages);
+	}
+
+	public function test_nomsgid_noorigin()
+	{
+		$this->init();
+
+		Zone::unguard();
+		Address::unguard();
+		$zo = Zone::firstOrCreate(['zone_id'=>10,'default'=>TRUE,'active'=>TRUE,'domain_id'=>$this->do->id,'system_id'=>$this->so->id]);
+		$src = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>999,'node_id'=>1,'point_id'=>0,'active'=>TRUE,'system_id'=>$this->so->id]);
+
+		// This packet has an incorrect zone in the Origin
+		$f = new File(__DIR__.'/data/test_nomsgid_noorigin.pkt');
+		$pkt = Packet::open($f,$zo,TRUE);
+
+		$this->assertEquals(1,$pkt->count());
+
+		$messages = FALSE;
+		foreach ($pkt as $msg) {
+			$messages = TRUE;
+			$this->assertNotTrue($msg->isNetmail());
+
+			$this->assertCount(1,$msg->rogue_path);
+			$this->assertNotFalse($msg->rogue_path->search('10:1/1'));
+
+			$this->assertCount(0,$msg->rogue_seen);
+		}
+
+		$this->assertTrue($messages);
+	}
+
+	public function test_msgid_origin()
+	{
+		$this->init();
+
+		Zone::unguard();
+		Address::unguard();
+		$zo = Zone::firstOrCreate(['zone_id'=>10,'default'=>TRUE,'active'=>TRUE,'domain_id'=>$this->do->id,'system_id'=>$this->so->id]);
+		$src = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>999,'node_id'=>1,'point_id'=>0,'active'=>TRUE,'system_id'=>$this->so->id]);
+
+		// This packet has an incorrect zone in the Origin
+		$f = new File(__DIR__.'/data/test_msgid_origin.pkt');
+		$pkt = Packet::open($f,$zo,TRUE);
+
+		$this->assertEquals(1,$pkt->count());
+
+		$messages = FALSE;
+		foreach ($pkt as $msg) {
+			$messages = TRUE;
+			$this->assertNotTrue($msg->isNetmail());
+			$this->assertCount(0,$msg->rogue_path);
+
+			$this->assertCount(2,$msg->rogue_seen);
+			$this->assertNotFalse($msg->rogue_seen->search('10:1/1'));
+			$this->assertNotFalse($msg->rogue_seen->search('10:999/999'));
+
+			$this->assertNotFalse($msg->seenaddress->search($src->id));
+		}
+
+		$this->assertTrue($messages);
+	}
+}