From 86a15872b849c193d1b46d9a80d79032bf49c20a Mon Sep 17 00:00:00 2001 From: Deon George Date: Tue, 22 Oct 2024 17:08:48 +1100 Subject: [PATCH] Better catching bad TZUTC in messages, continue parsing mail bundles in an archive if a packet has an error --- app/Classes/FTN/Message.php | 290 ++++++++++++++++++++---------------- app/Classes/FTN/Packet.php | 1 + app/Classes/File.php | 5 + app/Jobs/PacketProcess.php | 156 ++++++++++--------- 4 files changed, 246 insertions(+), 206 deletions(-) diff --git a/app/Classes/FTN/Message.php b/app/Classes/FTN/Message.php index 3d2d87d..6d96125 100644 --- a/app/Classes/FTN/Message.php +++ b/app/Classes/FTN/Message.php @@ -89,7 +89,7 @@ class Message extends FTNBase public const AREATAG_LEN = 35; // private array $header; // Message Header - private int $tzutc = 0; // TZUTC that needs to be converted to be used by Carbon @see self::kludges + private Collection $kludges; // TZUTC that needs to be converted to be used by Carbon @see self::kludges private Echomail|Netmail $mo; // The object storing this packet message private Address $us; // Our address for this message @@ -294,6 +294,7 @@ class Message extends FTNBase public function __construct(Zone $zone) { $this->zone = $zone; + $this->kludges = collect(); } public function __get($key) @@ -476,11 +477,25 @@ class Message extends FTNBase break; + case 'tzutc': + return $this->kludges->get($key); + default: throw new \Exception('Unknown key: '.$key); } } + public function __set(string $key,mixed $value): void + { + switch ($key) { + case 'tzutc': + if (! is_numeric($value)) + throw new InvalidPacketException('TZUTC is not numeric '.$value); + + $this->kludges->put($key,$value); + } + } + /** * Export an FTN message, ready for sending. * @@ -654,147 +669,158 @@ class Message extends FTNBase // First find our kludge lines $ptr_start = 0; + $ptr_end = 0; - while (substr($message,$ptr_start,1) === "\x01") { - $ptr_end = strpos($message,"\r",$ptr_start); + try { + while (substr($message,$ptr_start,1) === "\x01") { + $ptr_end = strpos($message,"\r",$ptr_start); - $m = []; - $kludge = ($x=substr($message,$ptr_start+1,$ptr_end-$ptr_start-1)); - preg_match('/^([^\s]+:?)+\s+(.*)$/',$kludge,$m); + $m = []; + $kludge = ($x=substr($message,$ptr_start+1,$ptr_end-$ptr_start-1)); + preg_match('/^([^\s]+:?)+\s+(.*)$/',$kludge,$m); + + $ptr_start = $ptr_end+1; + + if (! $m) { + Log::alert(sprintf('%s:! Invalid Kluge Line [%s]',self::LOGKEY,$x)); + continue; + } + + // Catch any kludges we need to process here + if (array_key_exists($m[1],self::kludges)) { + // Some earlier mystic message had a blank value for TZUTC + if ((($m[1]) === 'TZUTC:') && (! $m[2])) + $m[2] = '0000'; + + $this->{self::kludges[$m[1]]} = $m[2]; + + } else + $o->kludges = [$m[1],$m[2]]; + } + + // Next our message content ends with '\r * Origin: ... \r' or ... + // FTS-0004.001 + if ($ptr_end=strrpos($message,"\r * Origin: ",$ptr_start)) { + // Find the + $ptr_end = strpos($message,"\r",$ptr_end+1); + + // If there is no ptr_end, then this is not an origin + if (! $ptr_end) + throw new InvalidPacketException('Couldnt find the end of the origin'); + + } elseif (! $ptr_end=strpos($message,"\r\x01",$ptr_start)) { + $ptr_end = strlen($message); + } + + $remaining = substr($message,$ptr_end+1); + + // At this point, the remaining part of the message should start with \x01, PATH or SEEN-BY + if ((substr($remaining,0,9) !== 'SEEN-BY: ') && (substr($remaining,0,5) !== 'PATH:') && ($x=strpos($remaining,"\x01")) !== 0) { + if ($x) + $ptr_end += $x; + else + $ptr_end += strlen($remaining); + } + + // Process the message content + if ($content=substr($message,$ptr_start,$ptr_end-$ptr_start)) { + $o->msg_src = $content; + $o->msg_crc = md5($content); + $ptr_content_start = 0; + + // See if we have a tagline + if ($ptr_content_end=strrpos($content,"\r... ",$ptr_content_start)) { + $o->msg = substr($content,$ptr_content_start,$ptr_content_end+1); + + $ptr_content_start = $ptr_content_end+5; + $ptr_content_end = strpos($content,"\r",$ptr_content_start); + + // If there is no terminating "\r", then that's it + if (! $ptr_content_end) { + $o->set_tagline = substr($content,$ptr_content_start); + $ptr_content_start = strlen($content); + + } else { + $o->set_tagline = substr($content,$ptr_content_start,$ptr_content_end-$ptr_content_start); + $ptr_content_start = $ptr_content_end; + } + } + + // See if we have a tearline + if ($ptr_content_end=strrpos($content,"\r--- ",$ptr_content_start)) { + if (! $ptr_content_start) + $o->msg = substr($content,$ptr_content_start,$ptr_content_end+1); + + $ptr_content_start = $ptr_content_end+5; + $ptr_content_end = strpos($content,"\r",$ptr_content_start); + + // If there is no terminating "\r", then that's it + if (! $ptr_content_end) { + $o->set_tearline = substr($content,$ptr_content_start); + $ptr_content_start = strlen($content); + + } else { + $o->set_tearline = substr($content,$ptr_content_start,$ptr_content_end-$ptr_content_start); + $ptr_content_start = $ptr_content_end; + } + } + + // See if we have an origin + if ($ptr_content_end=strrpos($content,"\r * Origin: ",$ptr_content_start)) { + if (! $ptr_content_start) + $o->msg = substr($content,$ptr_content_start,$ptr_content_end); + + $ptr_content_start = $ptr_content_end+12; + + $ptr_content_end = strpos($content,"\r",$ptr_content_start); + + // If there is no terminating "\r", then that's it + if (! $ptr_content_end) { + $o->set_origin = substr($content,$ptr_content_start); + $ptr_content_start = strlen($content); + + } else { + $o->set_origin = substr($content,$ptr_content_start,$ptr_content_end-$ptr_content_start); + $ptr_content_start = $ptr_content_end+1; + } + } + + // If there wasnt any tagline/tearline/origin, then the whole content is the message + if (! $ptr_content_start) { + $o->msg = $content; + $ptr_content_start = $ptr_end-$ptr_start; + } + + // Trim any right \r from the message + $o->msg = rtrim($o->msg,"\r"); + + // Quick validation that we are done + if ($ptr_content_start !== strlen($content)) { + Log::alert(sprintf('%s:! We failed parsing the message.',self::LOGKEY)); + $o->msg = substr($message,0,$ptr_end); + } + } $ptr_start = $ptr_end+1; - if (! $m) { - Log::alert(sprintf('%s:! Invalid Kluge Line [%s]',self::LOGKEY,$x)); - continue; - } + // Finally work out control kludges + foreach (collect(explode("\r",substr($message,$ptr_start)))->filter() as $line) { + // If the line starts with ignore it + if (substr($line,0,1) === "\x01") + $line = ltrim($line,"\x01"); - // Catch any kludges we need to process here - if (array_key_exists($m[1],self::kludges)) { - // Some earlier mystic message had a blank value for TZUTC - if ((($m[1]) === 'TZUTC:') && (! $m[2])) - $m[2] = '0000'; - - $this->{self::kludges[$m[1]]} = $m[2]; - - } else + $m = []; + preg_match('/^([^\s]+:?)+\s+(.*)$/',$line,$m); $o->kludges = [$m[1],$m[2]]; - } - - // Next our message content ends with '\r * Origin: ... \r' or ... - // FTS-0004.001 - if ($ptr_end=strrpos($message,"\r * Origin: ",$ptr_start)) { - // Find the - $ptr_end = strpos($message,"\r",$ptr_end+1); - - // If there is no ptr_end, then this is not an origin - if (! $ptr_end) - throw new InvalidPacketException('Couldnt find the end of the origin'); - - } elseif (! $ptr_end=strpos($message,"\r\x01",$ptr_start)) { - $ptr_end = strlen($message); - } - - $remaining = substr($message,$ptr_end+1); - - // At this point, the remaining part of the message should start with \x01, PATH or SEEN-BY - if ((substr($remaining,0,9) !== 'SEEN-BY: ') && (substr($remaining,0,5) !== 'PATH:') && ($x=strpos($remaining,"\x01")) !== 0) { - if ($x) - $ptr_end += $x; - else - $ptr_end += strlen($remaining); - } - - // Process the message content - if ($content=substr($message,$ptr_start,$ptr_end-$ptr_start)) { - $o->msg_src = $content; - $o->msg_crc = md5($content); - $ptr_content_start = 0; - - // See if we have a tagline - if ($ptr_content_end=strrpos($content,"\r... ",$ptr_content_start)) { - $o->msg = substr($content,$ptr_content_start,$ptr_content_end+1); - - $ptr_content_start = $ptr_content_end+5; - $ptr_content_end = strpos($content,"\r",$ptr_content_start); - - // If there is no terminating "\r", then that's it - if (! $ptr_content_end) { - $o->set_tagline = substr($content,$ptr_content_start); - $ptr_content_start = strlen($content); - - } else { - $o->set_tagline = substr($content,$ptr_content_start,$ptr_content_end-$ptr_content_start); - $ptr_content_start = $ptr_content_end; - } } - // See if we have a tearline - if ($ptr_content_end=strrpos($content,"\r--- ",$ptr_content_start)) { - if (! $ptr_content_start) - $o->msg = substr($content,$ptr_content_start,$ptr_content_end+1); + } catch (\Exception $e) { + Log::error(sprintf('%s:! Error parsing message, now at offset [0x%02x] (%s)', + self::LOGKEY, + $ptr_start, + $e->getMessage()),['dump'=>hex_dump($message)]); - $ptr_content_start = $ptr_content_end+5; - $ptr_content_end = strpos($content,"\r",$ptr_content_start); - - // If there is no terminating "\r", then that's it - if (! $ptr_content_end) { - $o->set_tearline = substr($content,$ptr_content_start); - $ptr_content_start = strlen($content); - - } else { - $o->set_tearline = substr($content,$ptr_content_start,$ptr_content_end-$ptr_content_start); - $ptr_content_start = $ptr_content_end; - } - } - - // See if we have an origin - if ($ptr_content_end=strrpos($content,"\r * Origin: ",$ptr_content_start)) { - if (! $ptr_content_start) - $o->msg = substr($content,$ptr_content_start,$ptr_content_end); - - $ptr_content_start = $ptr_content_end+12; - - $ptr_content_end = strpos($content,"\r",$ptr_content_start); - - // If there is no terminating "\r", then that's it - if (! $ptr_content_end) { - $o->set_origin = substr($content,$ptr_content_start); - $ptr_content_start = strlen($content); - - } else { - $o->set_origin = substr($content,$ptr_content_start,$ptr_content_end-$ptr_content_start); - $ptr_content_start = $ptr_content_end+1; - } - } - - // If there wasnt any tagline/tearline/origin, then the whole content is the message - if (! $ptr_content_start) { - $o->msg = $content; - $ptr_content_start = $ptr_end-$ptr_start; - } - - // Trim any right \r from the message - $o->msg = rtrim($o->msg,"\r"); - - // Quick validation that we are done - if ($ptr_content_start !== strlen($content)) { - Log::alert(sprintf('%s:! We failed parsing the message.',self::LOGKEY)); - $o->msg = substr($message,0,$ptr_end); - } - } - - $ptr_start = $ptr_end+1; - - // Finally work out control kludges - foreach (collect(explode("\r",substr($message,$ptr_start)))->filter() as $line) { - // If the line starts with ignore it - if (substr($line,0,1) === "\x01") - $line = ltrim($line,"\x01"); - - $m = []; - preg_match('/^([^\s]+:?)+\s+(.*)$/',$line,$m); - $o->kludges = [$m[1],$m[2]]; + throw new InvalidPacketException('Error parsing message'); } return $o; diff --git a/app/Classes/FTN/Packet.php b/app/Classes/FTN/Packet.php index 7fc4a79..b7da2ed 100644 --- a/app/Classes/FTN/Packet.php +++ b/app/Classes/FTN/Packet.php @@ -174,6 +174,7 @@ abstract class Packet extends FTNBase implements \Iterator, \Countable || (($end=strpos($msgbuf,"\x00".self::PACKED_END,$leader)) !== FALSE)) { // Parse our message + Log::debug(sprintf('%s:- Message at offset [%d] in [%s]',self::LOGKEY,$read_ptr-strlen($readbuf),$name)); $o->parseMessage(substr($msgbuf,0,$end)); $msgbuf = substr($msgbuf,$end+3); diff --git a/app/Classes/File.php b/app/Classes/File.php index 9c53284..8df49ce 100644 --- a/app/Classes/File.php +++ b/app/Classes/File.php @@ -83,6 +83,11 @@ class File extends FileBase implements \Iterator /* METHODS */ + public function isArchive(): bool + { + return $this->isArchive; + } + /** * Determine if the file is a mail packet * diff --git a/app/Jobs/PacketProcess.php b/app/Jobs/PacketProcess.php index afa8520..fcdae4a 100644 --- a/app/Jobs/PacketProcess.php +++ b/app/Jobs/PacketProcess.php @@ -67,95 +67,103 @@ class PacketProcess implements ShouldQueue $f = new File($fs->path($this->filename)); $processed = FALSE; + $bad_archive = FALSE; foreach ($f as $packet) { - $pkt = Packet::process($packet,Arr::get(stream_get_meta_data($packet),'uri'),$f->itemSize(),$this->do); + try { + $pkt = Packet::process($packet,Arr::get(stream_get_meta_data($packet),'uri'),$f->itemSize(),$this->do); - // Check that the packet is from a system that is defined in the DB - if (! $pkt->fftn) { - Log::error(sprintf('%s:! Packet [%s] is not from a system we know about? [%s]',self::LOGKEY,$this->filename,$pkt->fftn_t)); + // Check that the packet is from a system that is defined in the DB + if (! $pkt->fftn) { + Log::error(sprintf('%s:! Packet [%s] is not from a system we know about? [%s]',self::LOGKEY,$this->filename,$pkt->fftn_t)); - break; - } - - if (! our_nodes($pkt->fftn->zone->domain)->contains($pkt->fftn)) { - Log::error(sprintf('%s:! Packet [%s] is from a system that is not configured with us? [%s] for [%s]',self::LOGKEY,$this->filename,$pkt->fftn_t,$this->do->name)); - - // @todo Notification::route('netmail',$pkt->fftn)->notify(new UnexpectedPacketFromYou($this->filename)); - // @todo Parse the packet for netmails and process them. We'll only accept netmails to us, and ignore all others - break; - } - - // If we dont have the tftn in the DB, then packet cannot be to us - if (! $pkt->tftn) { - Log::error(sprintf('%s:! Packet [%s] is from a system [%s] we dont know about?',self::LOGKEY,$this->filename,$pkt->tftn_t)); - - // @todo Notification::route('netmail',$pkt->fftn)->notify(new UnexpectedPacketToUs($this->filename)); - break; - } - - // Check the packet is to our address, if not we'll reject it. - if (! our_address($pkt->tftn->zone->domain)->contains($pkt->tftn)) { - Log::error(sprintf('%s:! Packet [%s] is not to our address? [%s]',self::LOGKEY,$this->filename,$pkt->tftn->ftn)); - - // @todo Notification::route('netmail',$pkt->fftn)->notify(new UnexpectedPacketToUs($this->filename)); - break; - } - - // Check the packet password - if ($pkt->fftn->pass_packet !== strtoupper($pkt->password)) { - Log::error(sprintf('%s:! Packet from [%s] with password [%s] is invalid.',self::LOGKEY,$pkt->fftn->ftn,$pkt->password)); - - Notification::route('netmail',$pkt->fftn)->notify(new PacketPasswordInvalid($pkt->password,$f->pktName())); - break; - } - - Log::info(sprintf('%s:- Packet has [%d] messages',self::LOGKEY,$pkt->count())); - - // Queue messages if there are too many in the packet. - if ($queue = ($pkt->count() > config('fido.queue_msgs'))) - Log::info(sprintf('%s:- Messages will be sent to the queue for processing',self::LOGKEY)); - - $count = 0; - foreach ($pkt as $msg) { - if ($msg instanceof Netmail) - Log::info(sprintf('%s:- Netmail from [%s] to [%s]',self::LOGKEY,$msg->fftn->ftn,$msg->tftn->ftn)); - elseif ($msg instanceof Echomail) - Log::info(sprintf('%s:- Echomail from [%s]',self::LOGKEY,$msg->fftn->ftn)); - - if ($msg->errors->count()) { - Log::error(sprintf('%s:! Message [%s] has [%d] errors, unable to process',self::LOGKEY,$msg->msgid,$msg->errors->count())); - - continue; + break; } - $msg->set_sender = $pkt->fftn->withoutRelations(); - // Record receiving packet and sender - $msg->set_pkt = $f->pktName(); - $msg->set_recvtime = $this->rcvd_time; + if (! our_nodes($pkt->fftn->zone->domain)->contains($pkt->fftn)) { + Log::error(sprintf('%s:! Packet [%s] is from a system that is not configured with us? [%s] for [%s]',self::LOGKEY,$this->filename,$pkt->fftn_t,$this->do->name)); - if ($queue || (! $this->interactive)) - Log::info(sprintf('%s:! Message [%s] will be sent to the queue to process',self::LOGKEY,$msg->msgid)); + // @todo Notification::route('netmail',$pkt->fftn)->notify(new UnexpectedPacketFromYou($this->filename)); + // @todo Parse the packet for netmails and process them. We'll only accept netmails to us, and ignore all others + break; + } + + // If we dont have the tftn in the DB, then packet cannot be to us + if (! $pkt->tftn) { + Log::error(sprintf('%s:! Packet [%s] is from a system [%s] we dont know about?',self::LOGKEY,$this->filename,$pkt->tftn_t)); + + // @todo Notification::route('netmail',$pkt->fftn)->notify(new UnexpectedPacketToUs($this->filename)); + break; + } + + // Check the packet is to our address, if not we'll reject it. + if (! our_address($pkt->tftn->zone->domain)->contains($pkt->tftn)) { + Log::error(sprintf('%s:! Packet [%s] is not to our address? [%s]',self::LOGKEY,$this->filename,$pkt->tftn->ftn)); + + // @todo Notification::route('netmail',$pkt->fftn)->notify(new UnexpectedPacketToUs($this->filename)); + break; + } + + // Check the packet password + if ($pkt->fftn->pass_packet !== strtoupper($pkt->password)) { + Log::error(sprintf('%s:! Packet from [%s] with password [%s] is invalid.',self::LOGKEY,$pkt->fftn->ftn,$pkt->password)); + + Notification::route('netmail',$pkt->fftn)->notify(new PacketPasswordInvalid($pkt->password,$f->pktName())); + break; + } + + Log::info(sprintf('%s:- Packet has [%d] messages',self::LOGKEY,$pkt->count())); + + // Queue messages if there are too many in the packet. + if ($queue = ($pkt->count() > config('fido.queue_msgs'))) + Log::info(sprintf('%s:- Messages will be sent to the queue for processing',self::LOGKEY)); + + $count = 0; + foreach ($pkt as $msg) { + if ($msg instanceof Netmail) + Log::info(sprintf('%s:- Netmail from [%s] to [%s]',self::LOGKEY,$msg->fftn->ftn,$msg->tftn->ftn)); + elseif ($msg instanceof Echomail) + Log::info(sprintf('%s:- Echomail from [%s]',self::LOGKEY,$msg->fftn->ftn)); + + if ($msg->errors->count()) { + Log::error(sprintf('%s:! Message [%s] has [%d] errors, unable to process',self::LOGKEY,$msg->msgid,$msg->errors->count())); + + continue; + } + + $msg->set_sender = $pkt->fftn->withoutRelations(); + // Record receiving packet and sender + $msg->set_pkt = $f->pktName(); + $msg->set_recvtime = $this->rcvd_time; - try { - // Dispatch job. if ($queue || (! $this->interactive)) - MessageProcess::dispatch($msg->withoutRelations(),$this->nobot); - else - MessageProcess::dispatchSync($msg->withoutRelations(),$this->nobot); + Log::info(sprintf('%s:! Message [%s] will be sent to the queue to process',self::LOGKEY,$msg->msgid)); - $count++; + try { + // Dispatch job. + if ($queue || (! $this->interactive)) + MessageProcess::dispatch($msg->withoutRelations(),$this->nobot); + else + MessageProcess::dispatchSync($msg->withoutRelations(),$this->nobot); - } catch (\Exception $e) { - Log::error(sprintf('%s:! Got error [%s] dispatching message [%s] (%d:%s-%s).',self::LOGKEY,get_class($e),$msg->msgid,$e->getLine(),$e->getFile(),$e->getMessage())); + $count++; + + } catch (\Exception $e) { + Log::error(sprintf('%s:! Got error [%s] dispatching message [%s] (%d:%s-%s).',self::LOGKEY,get_class($e),$msg->msgid,$e->getLine(),$e->getFile(),$e->getMessage())); + } } - } - if ($count === $pkt->count()) - $processed = TRUE; + if ($count === $pkt->count()) + $processed = TRUE; + + } catch (\Exception $e) { + Log::error(sprintf('%s:! Got an exception [%s] processing packet',self::LOGKEY,$e->getMessage())); + + $bad_archive = TRUE; + } } - if (! $processed) { + if ((! $processed) || $bad_archive) { Log::alert(sprintf('%s:- Not deleting packet [%s], it doesnt seem to be processed?',self::LOGKEY,$this->filename)); } else {