bbs/clrghouz
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Better catching bad TZUTC in messages, continue parsing mail bundles in an archive if a packet has an error
All checks were successful
Create Docker Image / Build Docker Image (x86_64) (push) Successful in 36s
Details
Create Docker Image / Build Docker Image (arm64) (push) Successful in 1m48s
Details
Create Docker Image / Final Docker Image Manifest (push) Successful in 10s
Details

Browse Source
This commit is contained in:
Deon George 2024-10-22 17:08:48 +11:00
parent eb61c5ea6e
commit 86a15872b8
4 changed files with 246 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

290
app/Classes/FTN/Message.php
View File

@ -89,7 +89,7 @@ class Message extends FTNBase
public const AREATAG_LEN = 35; // public const AREATAG_LEN = 35; //
private array $header; // Message Header private array $header; // Message Header
private int $tzutc = 0; // TZUTC that needs to be converted to be used by Carbon @see self::kludges private Collection $kludges; // TZUTC that needs to be converted to be used by Carbon @see self::kludges
private Echomail|Netmail $mo; // The object storing this packet message private Echomail|Netmail $mo; // The object storing this packet message
private Address $us; // Our address for this message private Address $us; // Our address for this message
@ -294,6 +294,7 @@ class Message extends FTNBase
public function __construct(Zone $zone) public function __construct(Zone $zone)
{ {
$this->zone = $zone; $this->zone = $zone;
$this->kludges = collect();
} }
public function __get($key) public function __get($key)
@ -476,11 +477,25 @@ class Message extends FTNBase
break; break;
case 'tzutc':
return $this->kludges->get($key);
default: default:
throw new \Exception('Unknown key: '.$key); throw new \Exception('Unknown key: '.$key);
} }
} }
public function __set(string $key,mixed $value): void
{
switch ($key) {
case 'tzutc':
if (! is_numeric($value))
throw new InvalidPacketException('TZUTC is not numeric '.$value);
$this->kludges->put($key,$value);
}
}
/** /**
* Export an FTN message, ready for sending. * Export an FTN message, ready for sending.
* *
@ -654,147 +669,158 @@ class Message extends FTNBase
// First find our kludge lines // First find our kludge lines
$ptr_start = 0; $ptr_start = 0;
$ptr_end = 0;
while (substr($message,$ptr_start,1) === "\x01") { try {
$ptr_end = strpos($message,"\r",$ptr_start); while (substr($message,$ptr_start,1) === "\x01") {
$ptr_end = strpos($message,"\r",$ptr_start);
$m = []; $m = [];
$kludge = ($x=substr($message,$ptr_start+1,$ptr_end-$ptr_start-1)); $kludge = ($x=substr($message,$ptr_start+1,$ptr_end-$ptr_start-1));
preg_match('/^([^\s]+:?)+\s+(.*)$/',$kludge,$m); preg_match('/^([^\s]+:?)+\s+(.*)$/',$kludge,$m);
$ptr_start = $ptr_end+1;
if (! $m) {
Log::alert(sprintf('%s:! Invalid Kluge Line [%s]',self::LOGKEY,$x));
continue;
}
// Catch any kludges we need to process here
if (array_key_exists($m[1],self::kludges)) {
// Some earlier mystic message had a blank value for TZUTC
if ((($m[1]) === 'TZUTC:') && (! $m[2]))
$m[2] = '0000';
$this->{self::kludges[$m[1]]} = $m[2];
} else
$o->kludges = [$m[1],$m[2]];
}
// Next our message content ends with '\r * Origin: ... \r' or <soh>...
// FTS-0004.001
if ($ptr_end=strrpos($message,"\r * Origin: ",$ptr_start)) {
// Find the <cr>
$ptr_end = strpos($message,"\r",$ptr_end+1);
// If there is no ptr_end, then this is not an origin
if (! $ptr_end)
throw new InvalidPacketException('Couldnt find the end of the origin');
} elseif (! $ptr_end=strpos($message,"\r\x01",$ptr_start)) {
$ptr_end = strlen($message);
}
$remaining = substr($message,$ptr_end+1);
// At this point, the remaining part of the message should start with \x01, PATH or SEEN-BY
if ((substr($remaining,0,9) !== 'SEEN-BY: ') && (substr($remaining,0,5) !== 'PATH:') && ($x=strpos($remaining,"\x01")) !== 0) {
if ($x)
$ptr_end += $x;
else
$ptr_end += strlen($remaining);
}
// Process the message content
if ($content=substr($message,$ptr_start,$ptr_end-$ptr_start)) {
$o->msg_src = $content;
$o->msg_crc = md5($content);
$ptr_content_start = 0;
// See if we have a tagline
if ($ptr_content_end=strrpos($content,"\r... ",$ptr_content_start)) {
$o->msg = substr($content,$ptr_content_start,$ptr_content_end+1);
$ptr_content_start = $ptr_content_end+5;
$ptr_content_end = strpos($content,"\r",$ptr_content_start);
// If there is no terminating "\r", then that's it
if (! $ptr_content_end) {
$o->set_tagline = substr($content,$ptr_content_start);
$ptr_content_start = strlen($content);
} else {
$o->set_tagline = substr($content,$ptr_content_start,$ptr_content_end-$ptr_content_start);
$ptr_content_start = $ptr_content_end;
}
}
// See if we have a tearline
if ($ptr_content_end=strrpos($content,"\r--- ",$ptr_content_start)) {
if (! $ptr_content_start)
$o->msg = substr($content,$ptr_content_start,$ptr_content_end+1);
$ptr_content_start = $ptr_content_end+5;
$ptr_content_end = strpos($content,"\r",$ptr_content_start);
// If there is no terminating "\r", then that's it
if (! $ptr_content_end) {
$o->set_tearline = substr($content,$ptr_content_start);
$ptr_content_start = strlen($content);
} else {
$o->set_tearline = substr($content,$ptr_content_start,$ptr_content_end-$ptr_content_start);
$ptr_content_start = $ptr_content_end;
}
}
// See if we have an origin
if ($ptr_content_end=strrpos($content,"\r * Origin: ",$ptr_content_start)) {
if (! $ptr_content_start)
$o->msg = substr($content,$ptr_content_start,$ptr_content_end);
$ptr_content_start = $ptr_content_end+12;
$ptr_content_end = strpos($content,"\r",$ptr_content_start);
// If there is no terminating "\r", then that's it
if (! $ptr_content_end) {
$o->set_origin = substr($content,$ptr_content_start);
$ptr_content_start = strlen($content);
} else {
$o->set_origin = substr($content,$ptr_content_start,$ptr_content_end-$ptr_content_start);
$ptr_content_start = $ptr_content_end+1;
}
}
// If there wasnt any tagline/tearline/origin, then the whole content is the message
if (! $ptr_content_start) {
$o->msg = $content;
$ptr_content_start = $ptr_end-$ptr_start;
}
// Trim any right \r from the message
$o->msg = rtrim($o->msg,"\r");
// Quick validation that we are done
if ($ptr_content_start !== strlen($content)) {
Log::alert(sprintf('%s:! We failed parsing the message.',self::LOGKEY));
$o->msg = substr($message,0,$ptr_end);
}
}
$ptr_start = $ptr_end+1; $ptr_start = $ptr_end+1;
if (! $m) { // Finally work out control kludges
Log::alert(sprintf('%s:! Invalid Kluge Line [%s]',self::LOGKEY,$x)); foreach (collect(explode("\r",substr($message,$ptr_start)))->filter() as $line) {
continue; // If the line starts with <soh> ignore it
} if (substr($line,0,1) === "\x01")
$line = ltrim($line,"\x01");
// Catch any kludges we need to process here $m = [];
if (array_key_exists($m[1],self::kludges)) { preg_match('/^([^\s]+:?)+\s+(.*)$/',$line,$m);
// Some earlier mystic message had a blank value for TZUTC
if ((($m[1]) === 'TZUTC:') && (! $m[2]))
$m[2] = '0000';
$this->{self::kludges[$m[1]]} = $m[2];
} else
$o->kludges = [$m[1],$m[2]]; $o->kludges = [$m[1],$m[2]];
}
// Next our message content ends with '\r * Origin: ... \r' or <soh>...
// FTS-0004.001
if ($ptr_end=strrpos($message,"\r * Origin: ",$ptr_start)) {
// Find the <cr>
$ptr_end = strpos($message,"\r",$ptr_end+1);
// If there is no ptr_end, then this is not an origin
if (! $ptr_end)
throw new InvalidPacketException('Couldnt find the end of the origin');
} elseif (! $ptr_end=strpos($message,"\r\x01",$ptr_start)) {
$ptr_end = strlen($message);
}
$remaining = substr($message,$ptr_end+1);
// At this point, the remaining part of the message should start with \x01, PATH or SEEN-BY
if ((substr($remaining,0,9) !== 'SEEN-BY: ') && (substr($remaining,0,5) !== 'PATH:') && ($x=strpos($remaining,"\x01")) !== 0) {
if ($x)
$ptr_end += $x;
else
$ptr_end += strlen($remaining);
}
// Process the message content
if ($content=substr($message,$ptr_start,$ptr_end-$ptr_start)) {
$o->msg_src = $content;
$o->msg_crc = md5($content);
$ptr_content_start = 0;
// See if we have a tagline
if ($ptr_content_end=strrpos($content,"\r... ",$ptr_content_start)) {
$o->msg = substr($content,$ptr_content_start,$ptr_content_end+1);
$ptr_content_start = $ptr_content_end+5;
$ptr_content_end = strpos($content,"\r",$ptr_content_start);
// If there is no terminating "\r", then that's it
if (! $ptr_content_end) {
$o->set_tagline = substr($content,$ptr_content_start);
$ptr_content_start = strlen($content);
} else {
$o->set_tagline = substr($content,$ptr_content_start,$ptr_content_end-$ptr_content_start);
$ptr_content_start = $ptr_content_end;
}
} }
// See if we have a tearline } catch (\Exception $e) {
if ($ptr_content_end=strrpos($content,"\r--- ",$ptr_content_start)) { Log::error(sprintf('%s:! Error parsing message, now at offset [0x%02x] (%s)',
if (! $ptr_content_start) self::LOGKEY,
$o->msg = substr($content,$ptr_content_start,$ptr_content_end+1); $ptr_start,
$e->getMessage()),['dump'=>hex_dump($message)]);
$ptr_content_start = $ptr_content_end+5; throw new InvalidPacketException('Error parsing message');
$ptr_content_end = strpos($content,"\r",$ptr_content_start);
// If there is no terminating "\r", then that's it
if (! $ptr_content_end) {
$o->set_tearline = substr($content,$ptr_content_start);
$ptr_content_start = strlen($content);
} else {
$o->set_tearline = substr($content,$ptr_content_start,$ptr_content_end-$ptr_content_start);
$ptr_content_start = $ptr_content_end;
}
}
// See if we have an origin
if ($ptr_content_end=strrpos($content,"\r * Origin: ",$ptr_content_start)) {
if (! $ptr_content_start)
$o->msg = substr($content,$ptr_content_start,$ptr_content_end);
$ptr_content_start = $ptr_content_end+12;
$ptr_content_end = strpos($content,"\r",$ptr_content_start);
// If there is no terminating "\r", then that's it
if (! $ptr_content_end) {
$o->set_origin = substr($content,$ptr_content_start);
$ptr_content_start = strlen($content);
} else {
$o->set_origin = substr($content,$ptr_content_start,$ptr_content_end-$ptr_content_start);
$ptr_content_start = $ptr_content_end+1;
}
}
// If there wasnt any tagline/tearline/origin, then the whole content is the message
if (! $ptr_content_start) {
$o->msg = $content;
$ptr_content_start = $ptr_end-$ptr_start;
}
// Trim any right \r from the message
$o->msg = rtrim($o->msg,"\r");
// Quick validation that we are done
if ($ptr_content_start !== strlen($content)) {
Log::alert(sprintf('%s:! We failed parsing the message.',self::LOGKEY));
$o->msg = substr($message,0,$ptr_end);
}
}
$ptr_start = $ptr_end+1;
// Finally work out control kludges
foreach (collect(explode("\r",substr($message,$ptr_start)))->filter() as $line) {
// If the line starts with <soh> ignore it
if (substr($line,0,1) === "\x01")
$line = ltrim($line,"\x01");
$m = [];
preg_match('/^([^\s]+:?)+\s+(.*)$/',$line,$m);
$o->kludges = [$m[1],$m[2]];
} }
return $o; return $o;

1
app/Classes/FTN/Packet.php
View File

@ -174,6 +174,7 @@ abstract class Packet extends FTNBase implements \Iterator, \Countable
|| (($end=strpos($msgbuf,"\x00".self::PACKED_END,$leader)) !== FALSE)) || (($end=strpos($msgbuf,"\x00".self::PACKED_END,$leader)) !== FALSE))
{ {
// Parse our message // Parse our message
Log::debug(sprintf('%s:- Message at offset [%d] in [%s]',self::LOGKEY,$read_ptr-strlen($readbuf),$name));
$o->parseMessage(substr($msgbuf,0,$end)); $o->parseMessage(substr($msgbuf,0,$end));
$msgbuf = substr($msgbuf,$end+3); $msgbuf = substr($msgbuf,$end+3);

5
app/Classes/File.php
View File

@ -83,6 +83,11 @@ class File extends FileBase implements \Iterator
/* METHODS */ /* METHODS */
public function isArchive(): bool
{
return $this->isArchive;
}
/** /**
* Determine if the file is a mail packet * Determine if the file is a mail packet
* *

156
app/Jobs/PacketProcess.php
View File

@ -67,95 +67,103 @@ class PacketProcess implements ShouldQueue
$f = new File($fs->path($this->filename)); $f = new File($fs->path($this->filename));
$processed = FALSE; $processed = FALSE;
$bad_archive = FALSE;
foreach ($f as $packet) { foreach ($f as $packet) {
$pkt = Packet::process($packet,Arr::get(stream_get_meta_data($packet),'uri'),$f->itemSize(),$this->do); try {
$pkt = Packet::process($packet,Arr::get(stream_get_meta_data($packet),'uri'),$f->itemSize(),$this->do);
// Check that the packet is from a system that is defined in the DB // Check that the packet is from a system that is defined in the DB
if (! $pkt->fftn) { if (! $pkt->fftn) {
Log::error(sprintf('%s:! Packet [%s] is not from a system we know about? [%s]',self::LOGKEY,$this->filename,$pkt->fftn_t)); Log::error(sprintf('%s:! Packet [%s] is not from a system we know about? [%s]',self::LOGKEY,$this->filename,$pkt->fftn_t));
break; break;
}
if (! our_nodes($pkt->fftn->zone->domain)->contains($pkt->fftn)) {
Log::error(sprintf('%s:! Packet [%s] is from a system that is not configured with us? [%s] for [%s]',self::LOGKEY,$this->filename,$pkt->fftn_t,$this->do->name));
// @todo Notification::route('netmail',$pkt->fftn)->notify(new UnexpectedPacketFromYou($this->filename));
// @todo Parse the packet for netmails and process them. We'll only accept netmails to us, and ignore all others
break;
}
// If we dont have the tftn in the DB, then packet cannot be to us
if (! $pkt->tftn) {
Log::error(sprintf('%s:! Packet [%s] is from a system [%s] we dont know about?',self::LOGKEY,$this->filename,$pkt->tftn_t));
// @todo Notification::route('netmail',$pkt->fftn)->notify(new UnexpectedPacketToUs($this->filename));
break;
}
// Check the packet is to our address, if not we'll reject it.
if (! our_address($pkt->tftn->zone->domain)->contains($pkt->tftn)) {
Log::error(sprintf('%s:! Packet [%s] is not to our address? [%s]',self::LOGKEY,$this->filename,$pkt->tftn->ftn));
// @todo Notification::route('netmail',$pkt->fftn)->notify(new UnexpectedPacketToUs($this->filename));
break;
}
// Check the packet password
if ($pkt->fftn->pass_packet !== strtoupper($pkt->password)) {
Log::error(sprintf('%s:! Packet from [%s] with password [%s] is invalid.',self::LOGKEY,$pkt->fftn->ftn,$pkt->password));
Notification::route('netmail',$pkt->fftn)->notify(new PacketPasswordInvalid($pkt->password,$f->pktName()));
break;
}
Log::info(sprintf('%s:- Packet has [%d] messages',self::LOGKEY,$pkt->count()));
// Queue messages if there are too many in the packet.
if ($queue = ($pkt->count() > config('fido.queue_msgs')))
Log::info(sprintf('%s:- Messages will be sent to the queue for processing',self::LOGKEY));
$count = 0;
foreach ($pkt as $msg) {
if ($msg instanceof Netmail)
Log::info(sprintf('%s:- Netmail from [%s] to [%s]',self::LOGKEY,$msg->fftn->ftn,$msg->tftn->ftn));
elseif ($msg instanceof Echomail)
Log::info(sprintf('%s:- Echomail from [%s]',self::LOGKEY,$msg->fftn->ftn));
if ($msg->errors->count()) {
Log::error(sprintf('%s:! Message [%s] has [%d] errors, unable to process',self::LOGKEY,$msg->msgid,$msg->errors->count()));
continue;
} }
$msg->set_sender = $pkt->fftn->withoutRelations(); if (! our_nodes($pkt->fftn->zone->domain)->contains($pkt->fftn)) {
// Record receiving packet and sender Log::error(sprintf('%s:! Packet [%s] is from a system that is not configured with us? [%s] for [%s]',self::LOGKEY,$this->filename,$pkt->fftn_t,$this->do->name));
$msg->set_pkt = $f->pktName();
$msg->set_recvtime = $this->rcvd_time;
if ($queue || (! $this->interactive)) // @todo Notification::route('netmail',$pkt->fftn)->notify(new UnexpectedPacketFromYou($this->filename));
Log::info(sprintf('%s:! Message [%s] will be sent to the queue to process',self::LOGKEY,$msg->msgid)); // @todo Parse the packet for netmails and process them. We'll only accept netmails to us, and ignore all others
break;
}
// If we dont have the tftn in the DB, then packet cannot be to us
if (! $pkt->tftn) {
Log::error(sprintf('%s:! Packet [%s] is from a system [%s] we dont know about?',self::LOGKEY,$this->filename,$pkt->tftn_t));
// @todo Notification::route('netmail',$pkt->fftn)->notify(new UnexpectedPacketToUs($this->filename));
break;
}
// Check the packet is to our address, if not we'll reject it.
if (! our_address($pkt->tftn->zone->domain)->contains($pkt->tftn)) {
Log::error(sprintf('%s:! Packet [%s] is not to our address? [%s]',self::LOGKEY,$this->filename,$pkt->tftn->ftn));
// @todo Notification::route('netmail',$pkt->fftn)->notify(new UnexpectedPacketToUs($this->filename));
break;
}
// Check the packet password
if ($pkt->fftn->pass_packet !== strtoupper($pkt->password)) {
Log::error(sprintf('%s:! Packet from [%s] with password [%s] is invalid.',self::LOGKEY,$pkt->fftn->ftn,$pkt->password));
Notification::route('netmail',$pkt->fftn)->notify(new PacketPasswordInvalid($pkt->password,$f->pktName()));
break;
}
Log::info(sprintf('%s:- Packet has [%d] messages',self::LOGKEY,$pkt->count()));
// Queue messages if there are too many in the packet.
if ($queue = ($pkt->count() > config('fido.queue_msgs')))
Log::info(sprintf('%s:- Messages will be sent to the queue for processing',self::LOGKEY));
$count = 0;
foreach ($pkt as $msg) {
if ($msg instanceof Netmail)
Log::info(sprintf('%s:- Netmail from [%s] to [%s]',self::LOGKEY,$msg->fftn->ftn,$msg->tftn->ftn));
elseif ($msg instanceof Echomail)
Log::info(sprintf('%s:- Echomail from [%s]',self::LOGKEY,$msg->fftn->ftn));
if ($msg->errors->count()) {
Log::error(sprintf('%s:! Message [%s] has [%d] errors, unable to process',self::LOGKEY,$msg->msgid,$msg->errors->count()));
continue;
}
$msg->set_sender = $pkt->fftn->withoutRelations();
// Record receiving packet and sender
$msg->set_pkt = $f->pktName();
$msg->set_recvtime = $this->rcvd_time;
try {
// Dispatch job.
if ($queue || (! $this->interactive)) if ($queue || (! $this->interactive))
MessageProcess::dispatch($msg->withoutRelations(),$this->nobot); Log::info(sprintf('%s:! Message [%s] will be sent to the queue to process',self::LOGKEY,$msg->msgid));
else
MessageProcess::dispatchSync($msg->withoutRelations(),$this->nobot);
$count++; try {
// Dispatch job.
if ($queue || (! $this->interactive))
MessageProcess::dispatch($msg->withoutRelations(),$this->nobot);
else
MessageProcess::dispatchSync($msg->withoutRelations(),$this->nobot);
} catch (\Exception $e) { $count++;
Log::error(sprintf('%s:! Got error [%s] dispatching message [%s] (%d:%s-%s).',self::LOGKEY,get_class($e),$msg->msgid,$e->getLine(),$e->getFile(),$e->getMessage()));
} catch (\Exception $e) {
Log::error(sprintf('%s:! Got error [%s] dispatching message [%s] (%d:%s-%s).',self::LOGKEY,get_class($e),$msg->msgid,$e->getLine(),$e->getFile(),$e->getMessage()));
}
} }
}
if ($count === $pkt->count()) if ($count === $pkt->count())
$processed = TRUE; $processed = TRUE;
} catch (\Exception $e) {
Log::error(sprintf('%s:! Got an exception [%s] processing packet',self::LOGKEY,$e->getMessage()));
$bad_archive = TRUE;
}
} }
if (! $processed) { if ((! $processed) || $bad_archive) {
Log::alert(sprintf('%s:- Not deleting packet [%s], it doesnt seem to be processed?',self::LOGKEY,$this->filename)); Log::alert(sprintf('%s:- Not deleting packet [%s], it doesnt seem to be processed?',self::LOGKEY,$this->filename));
} else { } else {