bbs/clrghouz
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Optimised our sending and receiving of items

Browse Source
This commit is contained in:
Deon George
2023-07-17 16:36:53 +10:00
parent a8f76aec31
commit c1ec4eff36
14 changed files with 634 additions and 572 deletions
Download Patch File Download Diff File Expand all files Collapse all files

259
app/Classes/File/Send.php
View File

@@ -3,10 +3,7 @@
namespace App\Classes\File;
use Exception;
use Illuminate\Contracts\Filesystem\FileNotFoundException;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use League\Flysystem\UnreadableFileEncountered;
use App\Models\Address;
@@ -15,37 +12,45 @@ use App\Models\Address;
* Object representing the files we are sending
*
* @property-read resource fd
* @property-read int mail_size
* @property-read int total_count
* @property-read int files_size The size of the files waiting to be sent
* @property-read int mail_size The size of the mail waiting to be sent
* @property-read int togo_count The total number of items that havent been sent yet
* @property-read int total_size The size of the items waiting to be sent
* @property-read int total_sent
* @property-read int total_sent_bytes
*/
final class Send extends Item
class Send extends Base
{
private const LOGKEY = 'IS-';
private Collection $list;
private ?Item $sending;
private Collection $packets;
public const T_NONE = 0;
/** @var int This file contains a file from the DB */
public const T_FILE = (1<<0);
/** @var int This file contains a bundle of Netmail */
public const T_NETMAIL = (1<<1);
/** @var int This file contains a bundle of Echomail */
public const T_ECHOMAIL = (1<<2);
private string $comp_data;
public function __construct()
{
// Initialise our variables
$this->list = collect();
$this->packets = collect();
$this->sending = NULL;
if (get_class($this) === self::class) {
$this->list = collect();
$this->f = NULL;
}
}
public function __get($key)
{
switch ($key) {
case 'dbids':
return $this->sending->messages->pluck('echoarea','dbid');
case 'completed':
return $this->list
->filter(function($item) { return $item->complete === TRUE; });
case 'fd':
return is_resource($this->f) ?: $this->f;
return ! is_null($this->index);
case 'files_count':
return $this->list
@@ -57,63 +62,46 @@ final class Send extends Item
->filter(function($item) { return $item->isType(self::IS_FILE|self::IS_TIC); })
->sum(function($item) { return $item->size; });
case 'filepos':
return $this->file_pos;
case 'mail_count':
return $this->list
->filter(function($item) { return $item->isType(self::IS_ARC|self::IS_PKT); })
->count()
+ $this->packets->count();
->count();
case 'mail_size':
return $this->list
->filter(function($item) { return $item->isType(self::IS_ARC|self::IS_PKT); })
->sum(function($item) { return $item->size; })
+ $this->packets->sum(function($item) { return $item->size; });
->sum(function($item) { return $item->size; });
case 'sendas':
return $this->sending?->{$key};
// The mtime is the time of the youngest message in the packet for the sending packet
case 'mtime':
return $this->sending?->youngest()->timestamp;
// The name is derived from the youngest message in the packet
case 'dbids':
case 'name':
return sprintf('%08x',timew($this->sending?->youngest()));
case 'nameas':
case 'mtime':
case 'size':
return strlen($this->sending?->file);
case 'type':
return $this->sending->{$key};
case 'pos':
return $this->{$key};
case 'sending':
return $this->list->get($this->index);
case 'togo_count':
return $this->list
->filter(function($item) { return $item->complete === FALSE; })
->count();
case 'total_sent':
return $this->list
->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === TRUE; })
->count()
+ $this->packets
->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === TRUE; })
->count();
return $this->completed
->count();
case 'total_sent_bytes':
return $this->list
->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === TRUE; })
->sum(function($item) { return $item->size; })
+ $this->packets
->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === TRUE; })
->sum(function($item) { return $item->size; });
case 'total_count':
return $this->list
->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === FALSE; })
->count()
+ $this->packets
->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === FALSE; })
->count();
return $this->completed
->sum(function($item) { return $item->size; });
case 'total_size':
return $this->list
->sum(function($item) { return $item->size; })
+ $this->packets->sum(function($item) { return $item->size; });
->sum(function($item) { return $item->size; });
default:
throw new Exception('Unknown key: '.$key);
@@ -121,31 +109,23 @@ final class Send extends Item
}
/**
* Add a file to the list of files to send
* Close the file descriptor of the file we are sending
*
* @param string $file
* @param bool $successful
* @throws Exception
* @todo Catch if we add the same file twice
*/
public function add(string $file): void
public function close(bool $successful): void
{
Log::debug(sprintf('%s:+ add [%s]',self::LOGKEY,$file));
if (! $this->fd)
throw new Exception('No file to close');
try {
$this->list->push(new Item($file,self::I_SEND));
} catch (FileNotFoundException) {
Log::error(sprintf('%s:! Item [%s] doesnt exist',self::LOGKEY,$file));
return;
} catch (UnreadableFileEncountered) {
Log::error(sprintf('%s:! Item [%s] cannot be read',self::LOGKEY,$file));
return;
// Uncaught, rethrow the error
} catch (Exception $e) {
throw new Exception($e->getMessage());
if ($successful) {
$end = time()-$this->start;
Log::debug(sprintf('%s: - Closing [%s], sent in [%d]',self::LOGKEY,$this->sending->nameas,$end));
}
$this->sending->close($successful);
$this->index = NULL;
}
/*
@@ -163,32 +143,6 @@ final class Send extends Item
}
*/
/**
* Close the file descriptor of the file we are sending
*
* @param bool $successful
* @throws Exception
*/
public function close(bool $successful): void
{
if (! $this->f)
throw new Exception('No file to close');
if ($successful) {
$this->sending->sent = TRUE;
$end = time()-$this->start;
Log::debug(sprintf('%s: - Closing [%s], sent in [%d]',self::LOGKEY,$this->sending->name,$end));
}
// @todo This should be done better isType == file?
if ((! $this->sending instanceof Mail) && (! $this->sending->isType(self::IS_TIC)))
fclose($this->f);
$this->sending = NULL;
$this->file_pos = 0;
$this->f = NULL;
}
/**
* Check if we are at the end of the file
*
@@ -196,9 +150,7 @@ final class Send extends Item
*/
public function feof(): bool
{
return (($this->sending instanceof Mail) || ($this->sending->isType(self::IS_TIC)))
? ($this->file_pos === $this->size)
: feof($this->f);
return $this->sending->feof();
}
/**
@@ -207,7 +159,6 @@ final class Send extends Item
* @param Address $ao
* @return bool
* @throws Exception
* @todo We need to make this into a transaction, incase the transfer fails.
*/
public function files(Address $ao): bool
{
@@ -221,13 +172,13 @@ final class Send extends Item
}
// Files
if (($x=$ao->getFiles())->count()) {
if (($x=$ao->filesWaiting())->count()) {
Log::debug(sprintf('%s:- [%d] Files(s) added for sending to [%s]',self::LOGKEY,$x->count(),$ao->ftn));
// Add Files
foreach ($x as $fo) {
$this->list->push(new Item($fo,self::I_SEND));
$this->list->push(new Tic($ao,$fo,self::I_SEND));
$this->list->push(new File($fo,self::T_FILE));
$this->list->push(new Tic($fo,$ao,self::T_NONE));
}
$file = TRUE;
@@ -247,16 +198,13 @@ final class Send extends Item
{
Log::debug(sprintf('%s:+ Opening file to send',self::LOGKEY));
// If we have mail, we'll send that first
if ($this->sending = $this->packets
->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === FALSE; })
->first())
if ((($this->index=$this->list->search(function($item) { return $item->complete === FALSE; })) !== FALSE)
&& $this->sending->open())
{
Log::debug(sprintf('%s:- Sending [%s]',self::LOGKEY,$this->sending->name));
Log::debug(sprintf('%s:- Sending item [%d] (%s)',self::LOGKEY,$this->index,$this->sending->nameas));
$this->file_pos = 0;
$this->pos = 0;
$this->start = time();
$this->f = TRUE;
/*
if ($compress)
@@ -264,39 +212,9 @@ final class Send extends Item
*/
return TRUE;
}
$this->sending = $this->list
->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === FALSE; })
->first();
if (! $this->sending)
throw new Exception('No files to open');
$this->file_pos = 0;
$this->start = time();
// If sending->file is a string, then we dont need to actually open anything
if ($this->sending->isType(self::IS_TIC)) {
$this->f = TRUE;
return TRUE;
}
// If sending file is a File::class, then our file is s3
if (! $this->sending->name && $this->sending->filemodel) {
$this->f = Storage::readStream($this->sending->filemodel->full_storage_path);
return TRUE;
} else {
$this->f = fopen($this->sending->name,'rb');
if (! $this->f) {
Log::error(sprintf('%s:! Unable to open file [%s] for reading',self::LOGKEY,$this->sending->name));
return FALSE;
}
Log::info(sprintf('%s:= open - File [%s] opened with size [%d]',self::LOGKEY,$this->sending->name,$this->sending->size));
return TRUE;
throw new Exception('No files to open');
}
}
@@ -304,9 +222,9 @@ final class Send extends Item
* Add our mail to the send queue
*
* @param Address $ao
* @param bool $update
* @return bool
* @throws Exception
* @todo We need to make this into a transaction, incase the transfer fails.
*/
public function mail(Address $ao,bool $update=TRUE): bool
{
@@ -323,7 +241,7 @@ final class Send extends Item
if ($x=$ao->getNetmail($update)) {
Log::debug(sprintf('%s: - Netmail(s) added for sending to [%s]',self::LOGKEY,$ao->ftn));
$this->packets->push(new Mail($x,self::I_SEND));
$this->list->push(new Mail($x,self::T_NETMAIL));
$mail = TRUE;
}
@@ -331,7 +249,7 @@ final class Send extends Item
if ($x=$ao->getEchomail($update)) {
Log::debug(sprintf('%s: - Echomail(s) added for sending to [%s]',self::LOGKEY,$ao->ftn));
$this->packets->push(new Mail($x,self::I_SEND));
$this->list->push(new Mail($x,self::T_ECHOMAIL));
$mail = TRUE;
}
@@ -348,29 +266,18 @@ final class Send extends Item
*/
public function read(int $length): ?string
{
if (! $this->f)
if (! $this->fd)
throw new Exception('No file open for read');
// We are sending mail
if ($this->sending instanceof Mail) {
$data = $this->sending->read($this->file_pos,$length);
// We are sending a tic file
} else if ($this->sending->isType(self::IS_TIC)) {
$data = $this->sending->read($this->file_pos,$length);
} else {
$data = fread($this->f,$length);
}
$this->file_pos += strlen($data);
if ($this->DEBUG)
Log::debug(sprintf('%s: - Read [%d] bytes, file pos now [%d]',self::LOGKEY,strlen($data),$this->file_pos));
$data = $this->sending->read($length);
if ($data === FALSE)
throw new UnreadableFileEncountered('Error reading file: '.$this->sending->name);
$this->pos += strlen($data);
Log::debug(sprintf('%s:- Read [%d] bytes, file pos now [%d]',self::LOGKEY,strlen($data),$this->pos));
return $data;
}
@@ -383,22 +290,20 @@ final class Send extends Item
*/
public function seek(int $pos): bool
{
if (! $this->f)
if (! $this->fd)
throw new Exception('No file open for seek');
if (($this->sending instanceof Mail) || $this->sending->isType(self::IS_TIC)) {
$pos = ($pos < $this->size) ? $pos : $this->size;
$rc = TRUE;
if ($this->sending->seek($pos)) {
$this->pos = $pos;
Log::debug(sprintf('%s:= Seeked to [%d]',self::LOGKEY,$this->pos));
return TRUE;
} else {
$rc = (fseek($this->f,$pos,SEEK_SET) === 0);
Log::error(sprintf('%s:! Failed to seek to [%d]',self::LOGKEY,$pos));
return FALSE;
}
if ($rc)
$this->file_pos = $pos;
Log::debug(sprintf('%s:= Seeked to [%d]',self::LOGKEY,$this->file_pos));
return $rc;
}
}