From c1ec4eff36657e20c7831c72a9496040635f1475 Mon Sep 17 00:00:00 2001 From: Deon George Date: Mon, 17 Jul 2023 16:36:53 +1000 Subject: [PATCH] Optimised our sending and receiving of items --- app/Classes/FTN/Tic.php | 29 ++-- app/Classes/File/Base.php | 107 ++++++++++++ app/Classes/File/File.php | 101 ++++++++++++ app/Classes/File/Item.php | 205 +++++------------------ app/Classes/File/Mail.php | 72 ++++++--- app/Classes/File/Receive.php | 180 ++++++++++----------- app/Classes/File/Send.php | 259 ++++++++++-------------------- app/Classes/File/Tic.php | 82 +++++++--- app/Classes/Protocol/Binkp.php | 84 ++++++---- app/Classes/Protocol/EMSI.php | 4 +- app/Classes/Protocol/Zmodem.php | 20 +-- app/Classes/Sock/SocketClient.php | 13 +- app/Jobs/MessageProcess.php | 2 +- app/Models/Address.php | 48 ++---- 14 files changed, 634 insertions(+), 572 deletions(-) create mode 100644 app/Classes/File/Base.php create mode 100644 app/Classes/File/File.php diff --git a/app/Classes/FTN/Tic.php b/app/Classes/FTN/Tic.php index cefc58f..78ce544 100644 --- a/app/Classes/FTN/Tic.php +++ b/app/Classes/FTN/Tic.php @@ -73,25 +73,12 @@ class Tic extends FTNBase $this->values = collect(); } - /** - * Does this TIC file bring us a nodelist - * - * @return bool - */ - public function isNodelist(): bool - { - return (($this->fo->nodelist_filearea_id === $this->fo->filearea->domain->filearea_id) - && (preg_match(str_replace(['.','?'],['\.','.'],'#^'.$this->fo->filearea->domain->nodelist_filename.'$#i'),$this->fo->name))); - } - /** * Generate a TIC file for an address * - * @param Address $ao - * @param File $fo * @return string */ - public function generate(Address $ao,File $fo): string + public static function generate(Address $ao,File $fo): string { $sysaddress = Setup::findOrFail(config('app.id'))->system->match($ao->zone)->first(); @@ -107,7 +94,8 @@ class Tic extends FTNBase $result->put('DESC',$fo->description); $result->put('AREA',$fo->filearea->name); $result->put('AREADESC',$fo->filearea->description); - $result->put('PW',$ao->session('ticpass')); + if ($x=$ao->session('ticpass')) + $result->put('PW',$x); $result->put('CRC',sprintf("%X",$fo->crc)); $out = ''; @@ -123,6 +111,17 @@ class Tic extends FTNBase return $out; } + /** + * Does this TIC file bring us a nodelist + * + * @return bool + */ + public function isNodelist(): bool + { + return (($this->fo->nodelist_filearea_id === $this->fo->filearea->domain->filearea_id) + && (preg_match(str_replace(['.','?'],['\.','.'],'#^'.$this->fo->filearea->domain->nodelist_filename.'$#i'),$this->fo->name))); + } + /** * Load a TIC file from an existing filename * diff --git a/app/Classes/File/Base.php b/app/Classes/File/Base.php new file mode 100644 index 0000000..0fadf63 --- /dev/null +++ b/app/Classes/File/Base.php @@ -0,0 +1,107 @@ +ftype&0xff) & $type; + } + + protected function whatType(): int + { + static $ext = ['su','mo','tu','we','th','fr','sa','req']; + + $x = strrchr($this->full_name,'.'); + + if (! $x || (strlen(substr($x,1)) != 3)) + return self::IS_FILE; + + if (strcasecmp(substr($x,1),'pkt') === 0) + return self::IS_PKT; + + if (strcasecmp(substr($x,1),'req') === 0) + return self::IS_REQ; + + if (strcasecmp(substr($x,1),'tic') === 0) + return self::IS_TIC; + + for ($i=0;$if = $file; + $this->ftype = ((($type&0xff)<<8)|self::IS_FILE); + } + + public function __get($key) { + switch ($key) { + case 'dbids': + return collect([$this->f->id]); + + case 'full_name': + case 'nameas': + return $this->f->name; + + case 'mtime': + return $this->f->datetime->timestamp; + + case 'name': + case 'size': + return $this->f->{$key}; + + case 'type': + return ($this->ftype&0xff00)>>8; + + default: + return parent::__get($key); + } + } + + public function close(bool $successful): void + { + if ($successful) + $this->complete = TRUE; + + fclose($this->fd); + } + + public function feof(): bool + { + return feof($this->fd); + } + + /** + * Open a file for sending + * + * @param string $compress The compression method that will be used (not implemented) + * @return bool + */ + public function open(string $compress=''): bool + { + // If sending file is a File::class, then our file is s3 + if ($this->nameas && $this->f instanceof FileModel) { + $this->fd = Storage::readStream($this->f->full_storage_path); + + } else { + $this->fd = fopen($this->full_name,'rb'); + + if (! $this->fd) { + Log::error(sprintf('%s:! Unable to open file [%s] for reading',self::LOGKEY,$this->full_name)); + + return FALSE; + } + + Log::info(sprintf('%s:= File [%s] opened with size [%d]',self::LOGKEY,$this->full_name,$this->size)); + } + + return TRUE; + } + + public function read(int $length): string + { + return fread($this->fd,$length); + } + + public function seek(int $pos): bool + { + return (fseek($this->f,$pos,SEEK_SET) === 0); + } +} \ No newline at end of file diff --git a/app/Classes/File/Item.php b/app/Classes/File/Item.php index ccbc3a9..6f2c8be 100644 --- a/app/Classes/File/Item.php +++ b/app/Classes/File/Item.php @@ -2,187 +2,68 @@ namespace App\Classes\File; -use Illuminate\Contracts\Filesystem\FileNotFoundException; use Illuminate\Support\Facades\Storage; -use League\Flysystem\UnreadableFileEncountered; -use App\Models\File; +use App\Models\Address; -/** - * A file we are sending or receiving - * - * @property string $name - * @property string $recvas - * @property int $size - */ -class Item +final class Item extends Receive { - private const LOGKEY = 'I--'; + private const LOCATION = 'local'; - // For deep debugging - protected bool $DEBUG = FALSE; - - /** @var int max size of file to use compression */ - // @todo MAX_COMPSIZE hasnt been implemented in RECEIVE OR SEND - protected const MAX_COMPSIZE = 0x1fff; - - protected const IS_PKT = (1<<1); - protected const IS_ARC = (1<<2); - protected const IS_FILE = (1<<3); - protected const IS_FLO = (1<<4); - protected const IS_REQ = (1<<5); - protected const IS_TIC = (1<<6); - - protected const I_RECV = (1<<0); - protected const I_SEND = (1<<1); - - protected string $file_name; - protected int $file_size; - protected int $file_mtime; - /** Current read/write pointer */ - protected int $file_pos = 0; - /** File descriptor */ - protected mixed $f = NULL; - protected int $type; - protected int $action; - protected File $filemodel; - - public bool $sent = FALSE; - public bool $received = FALSE; - public bool $incomplete = FALSE; - /** Time we started sending/receiving */ - protected int $start; + /** @var Address The address that sent us this item */ + private Address $ao; + private string $recvas; + private int $recvmtime; + private int $recvsize; /** - * @throws FileNotFoundException - * @throws UnreadableFileEncountered * @throws \Exception */ - public function __construct($file,int $action) + public function __construct(Address $ao,string $recvas,int $mtime,int $size) { - switch ($action) { - case self::I_SEND: - if ($file instanceof File) { - $this->filemodel = $file; - // @todo We should catch any exceptions if the default storage is s3 (it is) and we cannot find the file, or the s3 call fails - $this->file_size = Storage::size($file->full_storage_path); - $this->file_mtime = Storage::lastModified($file->full_storage_path); + parent::__construct(); - } else { - if (! is_string($file)) - throw new \Exception('Invalid object creation - file should be a string'); + $this->ao = $ao; + $this->recvas = $recvas; + $this->recvmtime = $mtime; + $this->recvsize = $size; - if (! file_exists($file)) - throw new FileNotFoundException('Item doesnt exist: '.$file); - - if (! is_readable($file)) - throw new UnreadableFileEncountered('Item cannot be read: '.$file); - - $this->file_name = $file; - $x = stat($file); - $this->file_size = $x['size']; - $this->file_mtime = $x['mtime']; - } - - break; - - case self::I_RECV; - $keys = ['name','mtime','size']; - - if (! is_array($file) || array_diff(array_keys($file),$keys)) - throw new \Exception('Invalid object creation - file is not a valid array :'.serialize(array_diff(array_keys($file),$keys))); - - $this->file_name = $file['name']; - $this->file_size = $file['size']; - $this->file_mtime = $file['mtime']; - - break; - - default: - throw new \Exception('Unknown action: '.$action); - } - - $this->action = $action; - $this->type = $this->whatType(); + $this->ftype = self::IS_FILE; } - /** - * @throws \Exception - */ - public function __get($key) - { - switch($key) { + public function __get($key) { + switch ($key) { + case 'exists': + return Storage::disk(self::LOCATION)->exists($this->rel_name); + + case 'rel_name': + return sprintf('%s/%04X-%s',config('app.fido'),$this->ao->id,$this->recvas); + case 'full_name': + return Storage::disk(self::LOCATION)->path($this->rel_name); + + case 'match_mtime': + return $this->mtime === $this->recvmtime; + case 'match_size': + return $this->size === $this->recvsize; + + case 'nameas': + return $this->recvas; + case 'recvmtime': + return $this->recvmtime; + case 'recvsize': + return $this->recvsize; + + case 'name_size_time': + return sprintf('%s %lu %lu',$this->recvas,$this->recvsize,$this->recvmtime); + case 'mtime': - if ($this instanceof Mail) - $this->youngest()->timestamp; - - if ($this->action & self::I_RECV|self::I_SEND) - return $this->{'file_'.$key}; - - throw new \Exception('Invalid request for key: '.$key); - - case 'name': - if ($this instanceof Mail) - return sprintf('%08x',timew($this->youngest())); - - if ($this->action & self::I_RECV|self::I_SEND) - return $this->{'file_'.$key}; - - throw new \Exception('Invalid request for key: '.$key); + return Storage::disk(self::LOCATION)->lastModified($this->rel_name); case 'size': - if ($this instanceof Mail) - return strlen($this->file); - - if ($this->action & self::I_RECV|self::I_SEND) - return $this->{'file_'.$key}; - - throw new \Exception('Invalid request for key: '.$key); - - case 'recvas': - return $this->file_name; - - case 'sendas': - if ($this instanceof Mail) - return sprintf('%s.pkt',$this->name); - - return $this->file_name ? basename($this->file_name) : $this->filemodel->name; + return Storage::disk(self::LOCATION)->size($this->rel_name); default: - throw new \Exception('Unknown key: '.$key); + return parent::__get($key); } } - - protected function isType(int $type): bool - { - return $this->type & $type; - } - - private function whatType(): int - { - static $ext = ['su','mo','tu','we','th','fr','sa','req']; - - $x = strrchr($this->file_name,'.'); - - if (! $x || (strlen(substr($x,1)) != 3)) - return self::IS_FILE; - - if (strcasecmp(substr($x,2),'lo') === 0) - return self::IS_FLO; - - if (strcasecmp(substr($x,1),'pkt') === 0) - return self::IS_PKT; - - if (strcasecmp(substr($x,1),'req') === 0) - return self::IS_REQ; - - if (strcasecmp(substr($x,1),'tic') === 0) - return self::IS_TIC; - - for ($i=0;$ifile = $mail; + parent::__construct(); - break; - - default: - throw new \Exception('Unknown action: '.$action); - } - - $this->action = $action; + $this->f = $mail; + $this->ftype = ((($type&0xff)<<8)|self::IS_PKT); + $this->readpos = 0; } public function __get($key) { switch ($key) { - case 'file': return $this->file; - case 'messages': return $this->file->messages; + case 'dbids': + return $this->f->messages->pluck('dbid'); + + case 'name': + return sprintf('%08x',timew($this->youngest())); + + case 'nameas': + return sprintf('%s.pkt',$this->name); + + case 'mtime': + return $this->youngest()->timestamp; + + case 'size': + return strlen($this->f); + + case 'type': + return ($this->ftype&0xff00)>>8; + default: return parent::__get($key); } } - public function read(int $start,int $length): string + public function close(bool $successful): void { - return substr((string)$this->file,$start,$length); + if ($successful) + $this->complete = TRUE; + } + + public function feof(): bool + { + return ($this->readpos === $this->size); + } + + public function open(string $compress=''): bool + { + return TRUE; + } + + public function read(int $length): string + { + $result = substr((string)$this->f,$this->readpos,$length); + $this->readpos += strlen($result); + + return $result; + } + + public function seek(int $pos): bool + { + $this->readpos = ($pos < $this->size) ? $pos : $this->size; + return TRUE; } public function youngest(): Carbon { - return $this->file->messages->pluck('date')->sort()->last(); + return $this->f->messages->pluck('date')->sort()->last(); } } \ No newline at end of file diff --git a/app/Classes/File/Receive.php b/app/Classes/File/Receive.php index 87a5928..05a8139 100644 --- a/app/Classes/File/Receive.php +++ b/app/Classes/File/Receive.php @@ -21,7 +21,7 @@ use App\Models\Address; * @property-read int total_recv * @property-read int total_recv_bytes */ -final class Receive extends Item +class Receive extends Base { private const LOGKEY = 'IR-'; @@ -31,10 +31,7 @@ final class Receive extends Item ]; private Address $ao; - private Collection $list; - private ?Item $receiving; - private string $file; // Local filename for file received /** @var ?string The compression used by the incoming file */ private ?string $comp; /** @var string|null The compressed data received */ @@ -43,41 +40,47 @@ final class Receive extends Item public function __construct() { // Initialise our variables - $this->list = collect(); - $this->receiving = NULL; + if (get_class($this) === self::class) { + $this->list = collect(); + $this->f = NULL; + } } public function __get($key) { switch ($key) { + case 'completed': + return $this->list + ->filter(function($item) { return $item->complete === TRUE; }); + case 'fd': return is_resource($this->f); - case 'filepos': - return $this->file_pos; - case 'mtime': - case 'name': + case 'nameas': case 'size': - return $this->receiving?->{'file_'.$key}; - case 'name_size_time': - return sprintf('%s %lu %lu',$this->name,$this->size,$this->mtime); + return $this->receiving->{$key}; - case 'to_get': + case 'pos': + return $this->pos; + + case 'receiving': + return $this->list->get($this->index); + + case 'ready': + return (! is_null($this->index)); + + case 'togo_count': return $this->list - ->filter(function($item) { return ($item->action & self::I_RECV) && $item->received === FALSE; }) + ->filter(function($item) { return $item->complete === FALSE; }) ->count(); case 'total_recv': - return $this->list - ->filter(function($item) { return ($item->action & self::I_RECV) && $item->received === TRUE; }) - ->count(); + return $this->completed->count(); case 'total_recv_bytes': - return $this->list - ->filter(function($item) { return ($item->action & self::I_RECV) && $item->received === TRUE; }) - ->sum(function($item) { return $item->size; }); + return $this->completed->sum(function($item) { return $item->size; }); default: throw new \Exception('Unknown key: '.$key); @@ -95,36 +98,31 @@ final class Receive extends Item throw new \Exception('No file to close'); if ($this->f) { - if ($this->file_pos !== $this->receiving->size) { - Log::warning(sprintf('%s:- Closing [%s], but missing [%d] bytes',self::LOGKEY,$this->receiving->name,$this->receiving->size-$this->file_pos)); - $this->receiving->incomplete = TRUE; - } - - $this->receiving->received = TRUE; + if ($this->pos !== $this->receiving->recvsize) + Log::warning(sprintf('%s:- Closing [%s], but missing [%d] bytes',self::LOGKEY,$this->receiving->nameas,$this->receiving->recvsize-$this->pos)); + else + $this->receiving->complete = TRUE; $end = time()-$this->start; - Log::debug(sprintf('%s:- Closing [%s], received in [%d]',self::LOGKEY,$this->receiving->name,$end)); + Log::debug(sprintf('%s:- Closing [%s], received in [%d]',self::LOGKEY,$this->receiving->nameas,$end)); if ($this->comp) - Log::info(sprintf('%s:= Compressed file using [%s] was [%d] bytes (%d). Compression rate [%3.2f%%]',self::LOGKEY,$this->comp,$x=strlen($this->comp_data),$this->receiving->size,$x/$this->receiving->size*100)); + Log::info(sprintf('%s:= Compressed file using [%s] was [%d] bytes (%d). Compression rate [%3.2f%%]',self::LOGKEY,$this->comp,$x=strlen($this->comp_data),$this->receiving->recvsize,$x/$this->receiving->recvsize*100)); fclose($this->f); // Set our mtime - touch($this->file,$this->mtime); - $this->file_pos = 0; + touch($this->receiving->full_name,$this->receiving->mtime); $this->f = NULL; - // If the packet has been received but not the right size, dont process it any more. - - // If we received a packet, we'll dispatch a job to process it - if (! $this->receiving->incomplete) - switch ($this->receiving->type) { + // If we received a packet, we'll dispatch a job to process it, if we got it all + if ($this->receiving->complete) + switch ($x=$this->receiving->whatType()) { case self::IS_ARC: case self::IS_PKT: - Log::info(sprintf('%s:- Processing mail %s [%s]',self::LOGKEY,$this->receiving->type === self::IS_PKT ? 'PACKET' : 'ARCHIVE',$this->file)); + Log::info(sprintf('%s:- Processing mail %s [%s]',self::LOGKEY,$x === self::IS_PKT ? 'PACKET' : 'ARCHIVE',$this->receiving->nameas)); try { - $f = new File($this->file); + $f = new File($this->receiving->full_name); $processed = FALSE; foreach ($f as $packet) { @@ -177,54 +175,72 @@ final class Receive extends Item } if (! $processed) { - Log::alert(sprintf('%s:- Not deleting packet [%s], it doesnt seem to be processed?',self::LOGKEY,$this->file)); + Log::alert(sprintf('%s:- Not deleting packet [%s], it doesnt seem to be processed?',self::LOGKEY,$this->receiving->nameas)); // If we want to keep the packet, we could do that logic here } elseif (! config('app.packet_keep')) { - Log::debug(sprintf('%s:- Deleting processed packet [%s]',self::LOGKEY,$this->file)); - unlink($this->file); + Log::debug(sprintf('%s:- Deleting processed packet [%s]',self::LOGKEY,$this->receiving->full_name)); + unlink($this->receiving->full_name); } } catch (InvalidPacketException $e) { - Log::error(sprintf('%s:- Not deleting packet [%s], as it generated an InvalidPacketException',self::LOGKEY,$this->file),['e'=>$e->getMessage()]); + Log::error(sprintf('%s:- Not deleting packet [%s], as it generated an InvalidPacketException',self::LOGKEY,$this->receiving->nameas),['e'=>$e->getMessage()]); } catch (\Exception $e) { - Log::error(sprintf('%s:- Not deleting packet [%s], as it generated an uncaught exception',self::LOGKEY,$this->file),['e'=>$e->getMessage()]); + Log::error(sprintf('%s:- Not deleting packet [%s], as it generated an uncaught exception',self::LOGKEY,$this->receiving->nameas),['e'=>$e->getMessage()]); } break; case self::IS_TIC: - Log::info(sprintf('%s:- Processing TIC file [%s]',self::LOGKEY,$this->file)); + Log::info(sprintf('%s:- Processing TIC file [%s]',self::LOGKEY,$this->receiving->nameas)); // Queue the tic to be processed later, in case the referenced file hasnt been received yet - TicProcess::dispatch($this->file); + TicProcess::dispatch($this->receiving->nameas); break; default: - Log::debug(sprintf('%s:- Leaving file [%s] in the inbound dir',self::LOGKEY,$this->file)); + Log::debug(sprintf('%s:- Leaving file [%s] in the inbound dir',self::LOGKEY,$this->receiving->nameas)); } } - $this->receiving = NULL; + $this->index = NULL; + } + + /** + * Add a new file to receive + * + * @param array $file + * @param Address $ao + * @throws \Exception + */ + public function new(array $file,Address $ao): void + { + Log::debug(sprintf('%s:+ Receiving new file [%s]',self::LOGKEY,join('|',$file))); + + if ($this->index) + throw new \Exception('Can only have 1 file receiving at a time'); + + $this->ao = $ao; + + $this->list->push(new Item($ao,Arr::get($file,'name'),(int)Arr::get($file,'mtime'),(int)Arr::get($file,'size'))); + $this->index = $this->list->count()-1; } /** * Open the file descriptor to receive a file * - * @param Address $ao * @param bool $check * @param string|null $comp If the incoming file will be compressed * @return int * @throws \Exception * @todo $comp should be parsed, in case it contains other items */ - public function open(Address $ao,bool $check=FALSE,string $comp=NULL): int + public function open(bool $check=FALSE,string $comp=NULL): int { // @todo implement return 4 - SUSPEND(?) file - // @todo Change to use Storage::class() - if (! $this->receiving) + if (is_null($this->index)) throw new \Exception('No files currently receiving'); $this->comp_data = ''; @@ -244,65 +260,37 @@ final class Receive extends Item elseif ($this->comp) Log::debug(sprintf('%s:- Receiving file with [%s] compression',self::LOGKEY,$this->comp)); - $this->ao = $ao; - $this->file_pos = 0; + $this->pos = 0; $this->start = time(); - $this->file = sprintf('storage/app/%s',$this->local_path($ao)); - if (file_exists($this->file) - && (Storage::disk('local')->lastModified($this->local_path($ao)) === $this->mtime) - && (Storage::disk('local')->size($this->local_path($ao)) === $this->size)) - { - Log::alert(sprintf('%s:- File already exists - skipping [%s]', self::LOGKEY, $this->file)); + if ($this->receiving->exists && $this->receiving->match_mtime && $this->receiving->match_size) { + Log::alert(sprintf('%s:- File already exists - skipping [%s]', self::LOGKEY,$this->receiving->nameas)); return Protocol::FOP_GOT; - } elseif (file_exists($this->file) && (Storage::disk('local')->size($this->local_path($ao)) > 0)) { - Log::alert(sprintf('%s:- File exists with different details - skipping [%s]',self::LOGKEY,$this->file)); + } elseif ($this->receiving->exists && $this->receiving->size > 0) { + Log::alert(sprintf('%s:- File exists with different details - skipping [%s] (size: %d, mtime: %d)',self::LOGKEY,$this->receiving->nameas,$this->receiving->size,$this->receiving->mtime)); return Protocol::FOP_SKIP; } else { // @todo I dont think we are enabling resumable sessions - need to check - Log::debug(sprintf('%s:- Opening [%s]',self::LOGKEY,$this->file)); + Log::debug(sprintf('%s:- Opening [%s]',self::LOGKEY,$this->receiving->nameas)); } // If we are only checking, we'll return (NR mode) if ($check) return Protocol::FOP_OK; - $this->f = fopen($this->file,'wb'); + $this->f = fopen($this->receiving->full_name,'wb'); + if (! $this->f) { - Log::error(sprintf('%s:! Unable to open file [%s] for writing',self::LOGKEY,$this->receiving->name)); + Log::error(sprintf('%s:! Unable to open file [%s] for writing',self::LOGKEY,$this->receiving->nameas)); return Protocol::FOP_ERROR; } - Log::info(sprintf('%s:= open - File [%s] opened for writing',self::LOGKEY,$this->receiving->name)); + Log::info(sprintf('%s:= File [%s] opened for writing',self::LOGKEY,$this->receiving->nameas)); return Protocol::FOP_OK; } - /** - * Add a new file to receive - * - * @param array $file - * @throws \Exception - */ - public function new(array $file): void - { - Log::debug(sprintf('%s:+ new [%s]',self::LOGKEY,join('|',$file))); - - if ($this->receiving) - throw new \Exception('Can only have 1 file receiving at a time'); - - $o = new Item($file,self::I_RECV); - $this->list->push($o); - - $this->receiving = $o; - } - - private function local_path(Address $ao): string - { - return sprintf('%s/%04X-%s',config('app.fido'),$ao->id,$this->receiving->recvas); - } - /** * Write data to the file we are receiving * @@ -312,7 +300,7 @@ final class Receive extends Item */ public function write(string $buf): bool { - if (! $this->f) + if (! $this->fd) throw new \Exception('No file open for write'); $data = ''; @@ -354,19 +342,19 @@ final class Receive extends Item $data = $buf; } - if ($this->file_pos+strlen($data) > $this->receiving->size) - throw new \Exception(sprintf('Too many bytes received [%d] (%d)?',$this->file_pos+strlen($buf),$this->receiving->size)); + if (($x=$this->pos+strlen($data)) > $this->receiving->recvsize) + throw new \Exception(sprintf('Too many bytes received [%d] (%d)?',$x,$this->receiving->recvsize)); $rc = fwrite($this->f,$data); if ($rc === FALSE) throw new FileException('Error while writing to file'); - $this->file_pos += $rc; - Log::debug(sprintf('%s:- Write [%d] bytes, file pos now [%d] of [%d] (%d)',self::LOGKEY,$rc,$this->file_pos,$this->receiving->size,strlen($buf))); + $this->pos += $rc; + Log::debug(sprintf('%s:- Write [%d] bytes, file pos now [%d] of [%d] (%d)',self::LOGKEY,$rc,$this->pos,$this->receiving->recvsize,strlen($buf))); - if (strlen($this->comp_data) > $this->receiving->size) - Log::alert(sprintf('%s:- Compression grew the file during transfer (%d->%d)',self::LOGKEY,$this->receiving->size,strlen($this->comp_data))); + if (strlen($this->comp_data) > $this->receiving->recvsize) + Log::alert(sprintf('%s:- Compression grew the file during transfer (%d->%d)',self::LOGKEY,$this->receiving->recvsize,strlen($this->comp_data))); return TRUE; } diff --git a/app/Classes/File/Send.php b/app/Classes/File/Send.php index e290793..d7f0fff 100644 --- a/app/Classes/File/Send.php +++ b/app/Classes/File/Send.php @@ -3,10 +3,7 @@ namespace App\Classes\File; use Exception; -use Illuminate\Contracts\Filesystem\FileNotFoundException; -use Illuminate\Support\Collection; use Illuminate\Support\Facades\Log; -use Illuminate\Support\Facades\Storage; use League\Flysystem\UnreadableFileEncountered; use App\Models\Address; @@ -15,37 +12,45 @@ use App\Models\Address; * Object representing the files we are sending * * @property-read resource fd - * @property-read int mail_size - * @property-read int total_count + * @property-read int files_size The size of the files waiting to be sent + * @property-read int mail_size The size of the mail waiting to be sent + * @property-read int togo_count The total number of items that havent been sent yet + * @property-read int total_size The size of the items waiting to be sent * @property-read int total_sent * @property-read int total_sent_bytes */ -final class Send extends Item +class Send extends Base { private const LOGKEY = 'IS-'; - private Collection $list; - private ?Item $sending; - private Collection $packets; + public const T_NONE = 0; + /** @var int This file contains a file from the DB */ + public const T_FILE = (1<<0); + /** @var int This file contains a bundle of Netmail */ + public const T_NETMAIL = (1<<1); + /** @var int This file contains a bundle of Echomail */ + public const T_ECHOMAIL = (1<<2); private string $comp_data; public function __construct() { // Initialise our variables - $this->list = collect(); - $this->packets = collect(); - $this->sending = NULL; + if (get_class($this) === self::class) { + $this->list = collect(); + $this->f = NULL; + } } public function __get($key) { switch ($key) { - case 'dbids': - return $this->sending->messages->pluck('echoarea','dbid'); + case 'completed': + return $this->list + ->filter(function($item) { return $item->complete === TRUE; }); case 'fd': - return is_resource($this->f) ?: $this->f; + return ! is_null($this->index); case 'files_count': return $this->list @@ -57,63 +62,46 @@ final class Send extends Item ->filter(function($item) { return $item->isType(self::IS_FILE|self::IS_TIC); }) ->sum(function($item) { return $item->size; }); - case 'filepos': - return $this->file_pos; - case 'mail_count': return $this->list ->filter(function($item) { return $item->isType(self::IS_ARC|self::IS_PKT); }) - ->count() - + $this->packets->count(); + ->count(); case 'mail_size': return $this->list ->filter(function($item) { return $item->isType(self::IS_ARC|self::IS_PKT); }) - ->sum(function($item) { return $item->size; }) - + $this->packets->sum(function($item) { return $item->size; }); + ->sum(function($item) { return $item->size; }); - case 'sendas': - return $this->sending?->{$key}; - - // The mtime is the time of the youngest message in the packet for the sending packet - case 'mtime': - return $this->sending?->youngest()->timestamp; - - // The name is derived from the youngest message in the packet + case 'dbids': case 'name': - return sprintf('%08x',timew($this->sending?->youngest())); - + case 'nameas': + case 'mtime': case 'size': - return strlen($this->sending?->file); + case 'type': + return $this->sending->{$key}; + + case 'pos': + return $this->{$key}; + + case 'sending': + return $this->list->get($this->index); + + case 'togo_count': + return $this->list + ->filter(function($item) { return $item->complete === FALSE; }) + ->count(); case 'total_sent': - return $this->list - ->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === TRUE; }) - ->count() - + $this->packets - ->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === TRUE; }) - ->count(); + return $this->completed + ->count(); case 'total_sent_bytes': - return $this->list - ->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === TRUE; }) - ->sum(function($item) { return $item->size; }) - + $this->packets - ->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === TRUE; }) - ->sum(function($item) { return $item->size; }); - - case 'total_count': - return $this->list - ->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === FALSE; }) - ->count() - + $this->packets - ->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === FALSE; }) - ->count(); + return $this->completed + ->sum(function($item) { return $item->size; }); case 'total_size': return $this->list - ->sum(function($item) { return $item->size; }) - + $this->packets->sum(function($item) { return $item->size; }); + ->sum(function($item) { return $item->size; }); default: throw new Exception('Unknown key: '.$key); @@ -121,31 +109,23 @@ final class Send extends Item } /** - * Add a file to the list of files to send + * Close the file descriptor of the file we are sending * - * @param string $file + * @param bool $successful * @throws Exception - * @todo Catch if we add the same file twice */ - public function add(string $file): void + public function close(bool $successful): void { - Log::debug(sprintf('%s:+ add [%s]',self::LOGKEY,$file)); + if (! $this->fd) + throw new Exception('No file to close'); - try { - $this->list->push(new Item($file,self::I_SEND)); - - } catch (FileNotFoundException) { - Log::error(sprintf('%s:! Item [%s] doesnt exist',self::LOGKEY,$file)); - return; - - } catch (UnreadableFileEncountered) { - Log::error(sprintf('%s:! Item [%s] cannot be read',self::LOGKEY,$file)); - return; - - // Uncaught, rethrow the error - } catch (Exception $e) { - throw new Exception($e->getMessage()); + if ($successful) { + $end = time()-$this->start; + Log::debug(sprintf('%s: - Closing [%s], sent in [%d]',self::LOGKEY,$this->sending->nameas,$end)); } + + $this->sending->close($successful); + $this->index = NULL; } /* @@ -163,32 +143,6 @@ final class Send extends Item } */ - /** - * Close the file descriptor of the file we are sending - * - * @param bool $successful - * @throws Exception - */ - public function close(bool $successful): void - { - if (! $this->f) - throw new Exception('No file to close'); - - if ($successful) { - $this->sending->sent = TRUE; - $end = time()-$this->start; - Log::debug(sprintf('%s: - Closing [%s], sent in [%d]',self::LOGKEY,$this->sending->name,$end)); - } - - // @todo This should be done better isType == file? - if ((! $this->sending instanceof Mail) && (! $this->sending->isType(self::IS_TIC))) - fclose($this->f); - - $this->sending = NULL; - $this->file_pos = 0; - $this->f = NULL; - } - /** * Check if we are at the end of the file * @@ -196,9 +150,7 @@ final class Send extends Item */ public function feof(): bool { - return (($this->sending instanceof Mail) || ($this->sending->isType(self::IS_TIC))) - ? ($this->file_pos === $this->size) - : feof($this->f); + return $this->sending->feof(); } /** @@ -207,7 +159,6 @@ final class Send extends Item * @param Address $ao * @return bool * @throws Exception - * @todo We need to make this into a transaction, incase the transfer fails. */ public function files(Address $ao): bool { @@ -221,13 +172,13 @@ final class Send extends Item } // Files - if (($x=$ao->getFiles())->count()) { + if (($x=$ao->filesWaiting())->count()) { Log::debug(sprintf('%s:- [%d] Files(s) added for sending to [%s]',self::LOGKEY,$x->count(),$ao->ftn)); // Add Files foreach ($x as $fo) { - $this->list->push(new Item($fo,self::I_SEND)); - $this->list->push(new Tic($ao,$fo,self::I_SEND)); + $this->list->push(new File($fo,self::T_FILE)); + $this->list->push(new Tic($fo,$ao,self::T_NONE)); } $file = TRUE; @@ -247,16 +198,13 @@ final class Send extends Item { Log::debug(sprintf('%s:+ Opening file to send',self::LOGKEY)); - // If we have mail, we'll send that first - if ($this->sending = $this->packets - ->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === FALSE; }) - ->first()) + if ((($this->index=$this->list->search(function($item) { return $item->complete === FALSE; })) !== FALSE) + && $this->sending->open()) { - Log::debug(sprintf('%s:- Sending [%s]',self::LOGKEY,$this->sending->name)); + Log::debug(sprintf('%s:- Sending item [%d] (%s)',self::LOGKEY,$this->index,$this->sending->nameas)); - $this->file_pos = 0; + $this->pos = 0; $this->start = time(); - $this->f = TRUE; /* if ($compress) @@ -264,39 +212,9 @@ final class Send extends Item */ return TRUE; - } - - $this->sending = $this->list - ->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === FALSE; }) - ->first(); - - if (! $this->sending) - throw new Exception('No files to open'); - - $this->file_pos = 0; - $this->start = time(); - - // If sending->file is a string, then we dont need to actually open anything - if ($this->sending->isType(self::IS_TIC)) { - $this->f = TRUE; - return TRUE; - } - - // If sending file is a File::class, then our file is s3 - if (! $this->sending->name && $this->sending->filemodel) { - $this->f = Storage::readStream($this->sending->filemodel->full_storage_path); - return TRUE; } else { - $this->f = fopen($this->sending->name,'rb'); - - if (! $this->f) { - Log::error(sprintf('%s:! Unable to open file [%s] for reading',self::LOGKEY,$this->sending->name)); - return FALSE; - } - - Log::info(sprintf('%s:= open - File [%s] opened with size [%d]',self::LOGKEY,$this->sending->name,$this->sending->size)); - return TRUE; + throw new Exception('No files to open'); } } @@ -304,9 +222,9 @@ final class Send extends Item * Add our mail to the send queue * * @param Address $ao + * @param bool $update * @return bool * @throws Exception - * @todo We need to make this into a transaction, incase the transfer fails. */ public function mail(Address $ao,bool $update=TRUE): bool { @@ -323,7 +241,7 @@ final class Send extends Item if ($x=$ao->getNetmail($update)) { Log::debug(sprintf('%s: - Netmail(s) added for sending to [%s]',self::LOGKEY,$ao->ftn)); - $this->packets->push(new Mail($x,self::I_SEND)); + $this->list->push(new Mail($x,self::T_NETMAIL)); $mail = TRUE; } @@ -331,7 +249,7 @@ final class Send extends Item if ($x=$ao->getEchomail($update)) { Log::debug(sprintf('%s: - Echomail(s) added for sending to [%s]',self::LOGKEY,$ao->ftn)); - $this->packets->push(new Mail($x,self::I_SEND)); + $this->list->push(new Mail($x,self::T_ECHOMAIL)); $mail = TRUE; } @@ -348,29 +266,18 @@ final class Send extends Item */ public function read(int $length): ?string { - if (! $this->f) + if (! $this->fd) throw new Exception('No file open for read'); - // We are sending mail - if ($this->sending instanceof Mail) { - $data = $this->sending->read($this->file_pos,$length); - - // We are sending a tic file - } else if ($this->sending->isType(self::IS_TIC)) { - $data = $this->sending->read($this->file_pos,$length); - - } else { - $data = fread($this->f,$length); - } - - $this->file_pos += strlen($data); - - if ($this->DEBUG) - Log::debug(sprintf('%s: - Read [%d] bytes, file pos now [%d]',self::LOGKEY,strlen($data),$this->file_pos)); + $data = $this->sending->read($length); if ($data === FALSE) throw new UnreadableFileEncountered('Error reading file: '.$this->sending->name); + $this->pos += strlen($data); + + Log::debug(sprintf('%s:- Read [%d] bytes, file pos now [%d]',self::LOGKEY,strlen($data),$this->pos)); + return $data; } @@ -383,22 +290,20 @@ final class Send extends Item */ public function seek(int $pos): bool { - if (! $this->f) + if (! $this->fd) throw new Exception('No file open for seek'); - if (($this->sending instanceof Mail) || $this->sending->isType(self::IS_TIC)) { - $pos = ($pos < $this->size) ? $pos : $this->size; - $rc = TRUE; + if ($this->sending->seek($pos)) { + $this->pos = $pos; + + Log::debug(sprintf('%s:= Seeked to [%d]',self::LOGKEY,$this->pos)); + + return TRUE; } else { - $rc = (fseek($this->f,$pos,SEEK_SET) === 0); + Log::error(sprintf('%s:! Failed to seek to [%d]',self::LOGKEY,$pos)); + + return FALSE; } - - if ($rc) - $this->file_pos = $pos; - - Log::debug(sprintf('%s:= Seeked to [%d]',self::LOGKEY,$this->file_pos)); - - return $rc; } } \ No newline at end of file diff --git a/app/Classes/File/Tic.php b/app/Classes/File/Tic.php index 3cdff24..e6752fa 100644 --- a/app/Classes/File/Tic.php +++ b/app/Classes/File/Tic.php @@ -2,38 +2,84 @@ namespace App\Classes\File; +use App\Models\Address; +use App\Models\File; use App\Classes\FTN\Tic as FTNTic; -use App\Models\{Address,File}; -class Tic extends Item +final class Tic extends Send { - private string $file; + /** @var int Our internal position counter */ + private int $readpos; + private Address $ao; + private string $tic; /** * @throws \Exception */ - public function __construct(Address $ao,File $fo,int $action) + public function __construct(File $file,Address $ao,int $type) { - switch ($action) { - case self::I_SEND: - $tic = new FTNTic; - $this->file = $tic->generate($ao,$fo); - $this->file_name = sprintf('%s.tic',sprintf('%08x',$fo->id)); - $this->file_size = strlen($this->file); - $this->file_mtime = $fo->created_at->timestamp; + parent::__construct(); - break; + $this->f = $file; + $this->ao = $ao; + $this->ftype = ((($type&0xff)<<8)|self::IS_TIC); + $this->readpos = 0; + + $this->tic = FTNTic::generate($ao,$file); + } + + public function __get($key) { + switch ($key) { + case 'dbids': + return collect([$this->f->id]); + + case 'name': + return sprintf('%08x',timew($this->f->created_at)); + + case 'nameas': + return sprintf('%s.tic',$this->name); + + case 'mtime': + return $this->f->datetime->timestamp; + + case 'size': + return strlen($this->tic); + + case 'type': + return ($this->ftype&0xff00)>>8; default: - throw new \Exception('Unknown action: '.$action); + return parent::__get($key); } - - $this->action = $action; - $this->type = self::IS_TIC; } - public function read(int $start,int $length): string + public function close(bool $successful): void { - return substr($this->file,$start,$length); + if ($successful) + $this->complete = TRUE; + } + + public function feof(): bool + { + return ($this->readpos === $this->size); + } + + public function open(string $compress=''): bool + { + return TRUE; + } + + public function read(int $length): string + { + $result = substr($this->tic,$this->readpos,$length); + $this->readpos += strlen($result); + + return $result; + } + + public function seek(int $pos): bool + { + $this->readpos = ($pos < $this->size) ? $pos : $this->size; + return TRUE; } } \ No newline at end of file diff --git a/app/Classes/Protocol/Binkp.php b/app/Classes/Protocol/Binkp.php index dd828f5..d256c26 100644 --- a/app/Classes/Protocol/Binkp.php +++ b/app/Classes/Protocol/Binkp.php @@ -11,6 +11,7 @@ use League\Flysystem\UnreadableFileEncountered; use App\Classes\Crypt; use App\Classes\Protocol as BaseProtocol; +use App\Classes\File\Send; use App\Classes\FTN\Message; use App\Classes\Sock\SocketClient; use App\Classes\Sock\SocketException; @@ -429,7 +430,7 @@ final class Binkp extends BaseProtocol $msg = ord(substr($this->rx_buf,0,1)); if ($msg > self::BPM_MAX) { - Log::error(sprintf('%s:! Unknown message received [%d] (%d-%s)',self::LOGKEY,$rc,strlen($this->rx_buf),$this->rx_buf)); + Log::error(sprintf('%s:! Unknown message received [%d] (%d-%s)',self::LOGKEY,$msg,strlen($this->rx_buf),$this->rx_buf)); $rc = TRUE; } else { @@ -487,7 +488,7 @@ final class Binkp extends BaseProtocol break; default: - Log::error(sprintf('%s:! BINKP command not implemented [%d]',self::LOGKEY,$rc)); + Log::error(sprintf('%s:! BINKP command not implemented [%d]',self::LOGKEY,$msg)); $rc = TRUE; } } @@ -500,7 +501,7 @@ final class Binkp extends BaseProtocol } catch (FileGrewException $e) { // Retry the file without compression Log::error(sprintf('%s:! %s',self::LOGKEY,$e->getMessage())); - $this->msgs(self::BPM_GET,sprintf('%s %ld NZ',$this->recv->name_size_time,$this->recv->filepos)); + $this->msgs(self::BPM_GET,sprintf('%s %ld NZ',$this->recv->name_size_time,$this->recv->pos)); } catch (\Exception $e) { Log::error(sprintf('%s:! %s',self::LOGKEY,$e->getMessage())); @@ -511,8 +512,8 @@ final class Binkp extends BaseProtocol $rc = TRUE; - if ($this->recv->filepos === $this->recv->size) { - Log::info(sprintf('%s:- Finished receiving file [%s] with size [%d]',self::LOGKEY,$this->recv->name,$this->recv->size)); + if ($this->recv->pos === $this->recv->size) { + Log::info(sprintf('%s:- Finished receiving file [%s] with size [%d]',self::LOGKEY,$this->recv->nameas,$this->recv->size)); $this->msgs(self::BPM_GOTSKIP,$this->recv->name_size_time); $this->recv->close(); @@ -577,6 +578,8 @@ final class Binkp extends BaseProtocol } catch (\Exception $e) { Log::error(sprintf('%s:! BINKP send unexpected ERROR [%s]',self::LOGKEY,$e->getMessage())); + + throw new \Exception($e->getMessage()); } if ($buf) { @@ -596,7 +599,7 @@ final class Binkp extends BaseProtocol } // @todo should this be less than BLOCKSIZE? Since a read could return a blocksize and it could be the end of the file? - if ($this->send->filepos === $this->send->size) { + if ($this->send->pos === $this->send->size) { $this->sessionSet(self::SE_WAITGOT); $this->sessionClear(self::SE_SENDFILE); } @@ -810,7 +813,7 @@ final class Binkp extends BaseProtocol $this->sessionSet(self::SE_RECVEOB); $this->sessionClear(self::SE_DELAYEOB); - if (! $this->send->total_count && $this->sessionGet(self::SE_NOFILES) && $this->capGet(self::F_MULTIBATCH,self::O_YES)) { + if (! $this->send->togo_count && $this->sessionGet(self::SE_NOFILES) && $this->capGet(self::F_MULTIBATCH,self::O_YES)) { // Add our mail to the queue if we have authenticated if ($this->node->aka_authed) foreach ($this->node->aka_remote_authed as $ao) { @@ -819,8 +822,8 @@ final class Binkp extends BaseProtocol $this->send->files($ao); } - Log::debug(sprintf('%s:- We have [%d] items to send to [%s]',self::LOGKEY,$this->send->total_count,$ao->ftn)); - if ($this->send->total_count) + Log::debug(sprintf('%s:- We have [%d] items to send to [%s]',self::LOGKEY,$this->send->togo_count,$ao->ftn)); + if ($this->send->togo_count) $this->sessionClear(self::SE_NOFILES|self::SE_SENTEOB); } @@ -852,10 +855,10 @@ final class Binkp extends BaseProtocol $this->sessionClear(self::SE_RECVEOB); - if ($this->recv->fd) - $this->recv->close(); + //if ($this->recv->fd) + // $this->recv->close(); - if (! $file=$this->file_parse($buf)) { + if (! ($file=$this->file_parse($buf))) { Log::error(sprintf('%s:! UNPARSABLE file info [%s]',self::LOGKEY,$buf)); $this->msgs(self::BPM_ERR,sprintf('M_FILE: unparsable file info: "%s", what are you on?',$buf)); @@ -868,20 +871,22 @@ final class Binkp extends BaseProtocol } // In NR mode, when we got -1 for the file offsite, the reply to our get will confirm our requested offset. - if ($this->recv->name && ! strncasecmp(Arr::get($file,'file.name'),$this->recv->name,self::MAX_PATH) + if ($this->recv->ready + && $this->recv->nameas + && (! strncasecmp(Arr::get($file,'file.name'),$this->recv->nameas,self::MAX_PATH)) && $this->recv->mtime === Arr::get($file,'file.mtime') && $this->recv->size === Arr::get($file,'file.size') - && $this->recv->filepos === $file['offs']) + && $this->recv->pos === $file['offs']) { - $this->recv->open($this->node->address,$file['offs']<0,$file['flags']); + $this->recv->open($file['offs']<0,$file['flags']); return TRUE; } - $this->recv->new($file['file']); + $this->recv->new($file['file'],$this->node->address); try { - switch ($this->recv->open($this->node->address,$file['offs']<0,$file['flags'])) { + switch ($this->recv->open($file['offs']<0,$file['flags'])) { case self::FOP_ERROR: Log::error(sprintf('%s:! File ERROR',self::LOGKEY)); @@ -943,8 +948,8 @@ final class Binkp extends BaseProtocol if ($file=$this->file_parse($buf)) { if ($this->sessionGet(self::SE_SENDFILE) - && $this->send->sendas - && ! strncasecmp(Arr::get($file,'file.name'),$this->send->sendas,self::MAX_PATH) + && $this->send->nameas + && ! strncasecmp(Arr::get($file,'file.name'),$this->send->nameas,self::MAX_PATH) && $this->send->mtime === Arr::get($file,'file.mtime') && $this->send->size === Arr::get($file,'file.size')) { @@ -957,8 +962,8 @@ final class Binkp extends BaseProtocol } else { $this->sessionClear(self::SE_WAITGET); - Log::debug(sprintf('%s:Sending file [%s] as [%s]',self::LOGKEY,$this->send->name,$this->send->sendas)); - $this->msgs(self::BPM_FILE,sprintf('%s %lu %ld %lu %s',$this->send->sendas,$this->send->size,$this->send->mtime,$file['offs'],$file['flags'])); + Log::debug(sprintf('%s:Sending file [%s] as [%s]',self::LOGKEY,$this->send->name,$this->send->nameas)); + $this->msgs(self::BPM_FILE,sprintf('%s %lu %ld %lu %s',$this->send->nameas,$this->send->size,$this->send->mtime,$file['offs'],$file['flags'])); } } else { @@ -984,8 +989,8 @@ final class Binkp extends BaseProtocol Log::debug(sprintf('%s:+ Remote confirms receipt for file [%s]',self::LOGKEY,$buf)); if ($file = $this->file_parse($buf)) { - if ($this->send->sendas - && ! strncasecmp(Arr::get($file,'file.name'),$this->send->sendas,self::MAX_PATH) + if ($this->send->nameas + && ! strncasecmp(Arr::get($file,'file.name'),$this->send->nameas,self::MAX_PATH) && $this->send->mtime === Arr::get($file,'file.mtime') && $this->send->size === Arr::get($file,'file.size')) { @@ -993,11 +998,12 @@ final class Binkp extends BaseProtocol Log::error(sprintf('%s:! M_got[skip] for unknown file [%s]',self::LOGKEY,$buf)); } else { - Log::info(sprintf('%s:= Packet/File [%s] sent with [%s].',self::LOGKEY,$this->send->name,$this->send->dbids->join(','))); + Log::info(sprintf('%s:= Packet/File [%s], type [%d] sent with [%d] items.',self::LOGKEY,$this->send->nameas,$this->send->type,$this->send->dbids->count())); $this->sessionClear(self::SE_WAITGOT|self::SE_SENDFILE); // Update netmail table - if ($x=$this->send->dbids->filter(function($item) { return (! $item); })->keys()->filter()) + if (($this->send->type === Send::T_NETMAIL) + && ($x=$this->send->dbids)->count()) DB::table('netmails') ->whereIn('id',$x) ->update([ @@ -1008,15 +1014,28 @@ final class Binkp extends BaseProtocol ]); // Update echomails table - if ($x=$this->send->dbids->filter(function($item) { return $item; })->keys()->filter()) + elseif (($this->send->type === Send::T_ECHOMAIL) + && ($x=$this->send->dbids)->count() + && $this->node->aka_remote_authed->count()) DB::table('echomail_seenby') ->whereIn('echomail_id',$x) - ->where('address_id',$this->node->address->id) + ->whereIn('address_id',$this->node->aka_remote_authed->pluck('id')) ->update([ 'sent_at'=>Carbon::now(), 'sent_pkt'=>$this->send->name, ]); + // Update the file seenby + elseif (($this->send->type === Send::T_FILE) + && ($x=$this->send->dbids)->count() + && $this->node->aka_remote_authed->count()) + DB::table('file_seenby') + ->whereIn('file_id',$x) + ->whereIn('address_id',$this->node->aka_remote_authed->pluck('id')) + ->update([ + 'sent_at'=>Carbon::now(), + ]); + $this->send->close(TRUE); } } @@ -1299,7 +1318,7 @@ final class Binkp extends BaseProtocol $this->send->files($ao); } - $this->msgs(self::BPM_NUL,sprintf('TRF %lu %lu',$this->send->mail_size,$this->send->size)); + $this->msgs(self::BPM_NUL,sprintf('TRF %lu %lu',$this->send->mail_size,$this->send->files_size)); $this->msgs(self::BPM_OK,sprintf('%ssecure',$have_pwd ? '' : 'non-')); return $this->binkp_hsdone(); @@ -1332,7 +1351,7 @@ final class Binkp extends BaseProtocol && (! $this->send->fd)) { // Open our next file to send - if ($this->send->total_count && ! $this->send->fd) { + if ($this->send->togo_count && ! $this->send->fd) { Log::info(sprintf('%s:- Opening next file to send',self::LOGKEY)); $this->send->open(); } @@ -1350,7 +1369,7 @@ final class Binkp extends BaseProtocol $this->msgs(self::BPM_FILE, sprintf('%s %lu %lu %ld %s', - $this->send->sendas, + $this->send->nameas, $this->send->size, $this->send->mtime, $this->sessionGet(self::SE_WAITGET) ? -1 : 0, @@ -1360,7 +1379,7 @@ final class Binkp extends BaseProtocol // We dont have anything to send } else { - Log::info(sprintf('%s:- Nothing else to send',self::LOGKEY)); + Log::info(sprintf('%s:- Nothing left to send in this batch',self::LOGKEY)); $this->sessionSet(self::SE_NOFILES); } } @@ -1389,6 +1408,7 @@ final class Binkp extends BaseProtocol $this->mib = 0; $this->sessionClear(self::SE_RECVEOB|self::SE_SENTEOB); + $this->sessionSet(self::SE_DELAYEOB); } $wd = ($this->mqueue->count() || $this->tx_left || ($this->sessionGet(self::SE_SENDFILE) && $this->send->fd && ! $this->sessionGet(self::SE_WAITGET))); @@ -1424,7 +1444,7 @@ final class Binkp extends BaseProtocol break; } - if (($this->mqueue->count() || $wd) && ! $this->binkp_send() && (! $this->send->total_count)) { + if (($this->mqueue->count() || $wd) && ! $this->binkp_send() && (! $this->send->togo_count)) { Log::info(sprintf('%s:- BINKP finished sending',self::LOGKEY)); break; diff --git a/app/Classes/Protocol/EMSI.php b/app/Classes/Protocol/EMSI.php index 92a68c2..089f52d 100644 --- a/app/Classes/Protocol/EMSI.php +++ b/app/Classes/Protocol/EMSI.php @@ -1208,7 +1208,7 @@ final class EMSI extends BaseProtocol implements CRCInterface,ZmodemInterface while ($this->send->mail($ao)) { $z = new Zmodem; - if (! $z->zmodem_sendinit($this->client,$zap) && $this->send->total_count) + if (! $z->zmodem_sendinit($this->client,$zap) && $this->send->togo_count) $z->zmodem_sendfile($this->send); } @@ -1216,7 +1216,7 @@ final class EMSI extends BaseProtocol implements CRCInterface,ZmodemInterface while ($this->send->files($ao)) { $z = new Zmodem; - if (! $z->zmodem_sendinit($this->client,$zap) && $this->send->total_count) + if (! $z->zmodem_sendinit($this->client,$zap) && $this->send->togo_count) $z->zmodem_sendfile($this->send); } } diff --git a/app/Classes/Protocol/Zmodem.php b/app/Classes/Protocol/Zmodem.php index 9c067e5..9f44701 100644 --- a/app/Classes/Protocol/Zmodem.php +++ b/app/Classes/Protocol/Zmodem.php @@ -272,7 +272,7 @@ final class Zmodem extends Protocol implements CRCInterface,ZmodemInterface $proto = $this->originate ? $this->node->optionGet(self::P_MASK) : $this->optionGet(self::P_MASK); if ($this->originate) { - if (! $z->zmodem_sendinit($this->client,$proto) && $this->send->total_count) + if (! $z->zmodem_sendinit($this->client,$proto) && $this->send->togo_count) $this->zmodem_sendfile($this->send); $rc = $this->zmodem_senddone(); @@ -304,7 +304,7 @@ final class Zmodem extends Protocol implements CRCInterface,ZmodemInterface if ($canzap&0x0100) $opts |= self::LSZ_OPTFIRSTBATCH; - switch ($rc=$this->ls_zrecvfinfo(self::ZRINIT,($this->ls_Protocol&self::LSZ_OPTFIRSTBATCH) ? 1 : 0)) { + switch ($rc=$this->ls_zrecvfinfo(self::ZRINIT,($this->ls_Protocol&self::LSZ_OPTFIRSTBATCH) ? 1 : 0,$ao)) { case self::ZFIN: Log::debug(sprintf('%s:= zmodem_receive ZFIN after INIT, empty batch',self::LOGKEY)); $this->ls_zdonereceiver(); @@ -333,13 +333,13 @@ final class Zmodem extends Protocol implements CRCInterface,ZmodemInterface return self::OK; case self::ZFILE: - if (! $this->recv->to_get) { + if (! $this->recv->togo_count) { Log::error(sprintf('%s: ! zmodem_receive No files to get?',self::LOGKEY)); $frame = self::ZSKIP; } else { - switch ($this->recv->open($ao)) { + switch ($this->recv->open()) { case self::FOP_SKIP: Log::info(sprintf('%s: = zmodem_receive Skip this file [%s]',self::LOGKEY,$this->recv->name)); $frame = self::ZSKIP; @@ -408,7 +408,7 @@ final class Zmodem extends Protocol implements CRCInterface,ZmodemInterface return self::ERROR; } - $rc = $this->ls_zrecvfinfo($frame,1); + $rc = $this->ls_zrecvfinfo($frame,1,$ao); } return self::OK; @@ -504,9 +504,9 @@ final class Zmodem extends Protocol implements CRCInterface,ZmodemInterface { Log::debug(sprintf('%s:+ zmodem_sendfile',self::LOGKEY)); - while ($send->total_count && $send->open()) { + while ($send->togo_count && $send->open()) { try { - $rc = $this->ls_zsendfile($send,$this->ls_SerialNum++,$send->total_count,$send->total_size); + $rc = $this->ls_zsendfile($send,$this->ls_SerialNum++,$send->togo_count,$send->total_size); switch ($rc) { case self::OK: @@ -1363,7 +1363,7 @@ final class Zmodem extends Protocol implements CRCInterface,ZmodemInterface * @return int * @throws \Exception */ - private function ls_zrecvfinfo(int $frame,int $first): int + private function ls_zrecvfinfo(int $frame,int $first,Address $ao): int { Log::debug(sprintf('%s:+ ls_zrecvfinfo - Frame [%d], First [%d]',self::LOGKEY,$frame,$first)); @@ -1478,7 +1478,7 @@ final class Zmodem extends Protocol implements CRCInterface,ZmodemInterface $filesleft = -1; } else { - $this->recv->new($file); + $this->recv->new($file,$ao); } return self::ZFILE; @@ -2390,7 +2390,7 @@ final class Zmodem extends Protocol implements CRCInterface,ZmodemInterface $buf = ''; $this->client->tx_purge(); - $buf = $send->sendas.chr(0); + $buf = $send->nameas.chr(0); $buf .= sprintf('%ld %lo %o %o %ld %ld', $send->size, diff --git a/app/Classes/Sock/SocketClient.php b/app/Classes/Sock/SocketClient.php index c742c1c..288a5df 100644 --- a/app/Classes/Sock/SocketClient.php +++ b/app/Classes/Sock/SocketClient.php @@ -17,7 +17,7 @@ final class SocketClient { private const LOGKEY = 'SC-'; // For deep debugging - private bool $DEBUG = FALSE; + private bool $DEBUG = TRUE; private \Socket $connection; private string $address_local = ''; @@ -214,7 +214,7 @@ final class SocketClient { $sent = $this->send(substr($this->tx_buf,0,self::TX_SIZE),0); if ($this->DEBUG) - Log::debug(sprintf('%s:- Sent [%d] chars [%s]',self::LOGKEY,$rc,Str::limit($this->tx_buf,15))); + Log::debug(sprintf('%s:- Sent [%d] chars [%s]',self::LOGKEY,$sent,Str::limit($this->tx_buf,15))); $this->tx_buf = substr($this->tx_buf,$sent); @@ -356,7 +356,7 @@ final class SocketClient { $this->rx_buf .= $buf; if ($this->DEBUG) - Log::debug(sprintf('%s:- Added [%d] chars to the RX buffer',self::LOGKEY,strlen($buf))); + Log::debug(sprintf('%s:- Added [%d] chars to the RX buffer',self::LOGKEY,strlen($buf)),['rx_buf'=>hex_dump($this->rx_buf)]); // Loop again and return the data, now that it is in the RX buffer return $this->read($timeout,$len); @@ -404,13 +404,12 @@ final class SocketClient { /** * Send data to the client * - * @param $message + * @param string $message * @param int $timeout - * @param null $length - * @return false|int + * @return int|false * @throws \Exception */ - public function send($message,int $timeout) + public function send(string $message,int $timeout): int|false { if ($timeout AND (! $rc=$this->canSend($timeout))) return $rc; diff --git a/app/Jobs/MessageProcess.php b/app/Jobs/MessageProcess.php index d842c04..e682af0 100644 --- a/app/Jobs/MessageProcess.php +++ b/app/Jobs/MessageProcess.php @@ -103,7 +103,7 @@ class MessageProcess implements ShouldQueue $o->flags |= Message::FLAG_RECD; $o->save(); - Log::info(sprintf('%s:! Netmail [%s] from (%s:%s) - was processed by us [%d]', + Log::info(sprintf('%s:= Netmail [%s] from (%s:%s) - was processed by us internally [%d]', self::LOGKEY, $this->msg->msgid, $this->msg->user_from, diff --git a/app/Models/Address.php b/app/Models/Address.php index ac26f11..9f984db 100644 --- a/app/Models/Address.php +++ b/app/Models/Address.php @@ -9,19 +9,10 @@ use Illuminate\Database\Eloquent\SoftDeletes; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; -use App\Classes\FTN\Packet; +use App\Classes\FTN\{Message,Packet}; use App\Http\Controllers\DomainController; use App\Traits\ScopeActive; -/** - * @todo Need to stop this from happening: - * In this example nn:3/1 can be defined 3 different ways. - * + id | zone_id | region_id | host_id | node_id | point_id | status | role | system_id | hub_id - * + ----+---------+-----------+---------+---------+----------+--------+------+-----------+-------- - * + 533 | 6 | 3 | 0 | 1 | 0 | | 4 | 1 | - * + 534 | 6 | n | 3 | 1 | 0 | | 2 | 1 | - * + 535 | 6 | 0 | 3 | 1 | 0 | | 2 | 1 | - */ class Address extends Model { private const LOGKEY = 'MA-'; @@ -338,7 +329,7 @@ class Address extends Model public function getFTN3DAttribute(): string { - return sprintf('%d:%d/%d',$this->zone->zone_id,$this->host_id ?: $this->region_id,$this->node_id); + return sprintf('%d:%s',$this->zone->zone_id,$this->getFTN2DAttribute()); } public function getFTN4DAttribute(): string @@ -546,8 +537,8 @@ class Address extends Model public function echomailWaiting(): Collection { return $this->echomails() - ->whereNull('echomail_seenby.sent_at') - ->whereNotNull('echomail_seenby.export_at') + ->whereNull('sent_at') + ->whereNotNull('export_at') ->get(); } @@ -559,8 +550,8 @@ class Address extends Model public function filesWaiting(): Collection { return $this->files() - ->whereNull('file_seenby.sent_at') - ->whereNotNull('file_seenby.export_at') + ->whereNull('sent_at') + ->whereNotNull('export_at') ->get(); } @@ -597,8 +588,7 @@ class Address extends Model ->whereIn('echomail_id',$x->pluck('id')) ->where('address_id',$this->id) ->whereNull('sent_at') - ->whereNull('sent_pkt') - ->whereNotNull('echomail_seenby.export_at') + ->whereNotNull('export_at') ->update(['sent_pkt'=>$pkt->name]); } @@ -610,25 +600,11 @@ class Address extends Model * * @param bool $update * @return Collection + * @deprecated use filesWaiting() directly */ public function getFiles(bool $update=TRUE): Collection { - if (($files=$this->filesWaiting()) - ->count()) - { - Log::debug(sprintf('%s:= Got [%d] files for [%s] for sending',self::LOGKEY,$files->count(),$this->ftn)); - - // @todo This should be transactional, incase the transfer fails - if ($files->count() && $update) - DB::table('file_seenby') - ->whereIn('file_id',$files->pluck('id')) - ->where('address_id',$this->id) - ->whereNull('sent_at') - ->whereNotNull('export_at') - ->update(['sent_at'=>Carbon::now()]); - } - - return $files; + return $this->filesWaiting(); } /** @@ -705,7 +681,11 @@ class Address extends Model public function netmailWaiting(): Collection { return Netmail::whereIn('tftn_id',(($x=$this->children) ? $x->pluck('id') : collect())->push($this->id)) - ->where('local',FALSE) + ->where(function($query) { + return $query->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_INTRANSIT)) + ->orWhereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL)); + }) + ->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT)) ->whereNull('sent_at') ->get(); }