bbs/clrghouz
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Optimised our sending and receiving of items

Browse Source
This commit is contained in:
Deon George
2023-07-17 16:36:53 +10:00
parent a8f76aec31
commit c1ec4eff36
14 changed files with 634 additions and 572 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
app/Classes/File/Base.php Normal file
View File

@@ -0,0 +1,107 @@
<?php
namespace App\Classes\File;
use Illuminate\Support\Collection;
/**
* A file we are sending or receiving
*
* For sending, we have sub types of Mail (Netmail/Echomail) and Files (File & TIC)
* When sending, we can queue up a list of items, and mark one active (the one being sent) at a time.
*
* + Netmail/Echomail/TIC
* + name is dynamically calculated, based on timew() of the youngest item in the mail bundle
* + size is dynamically calculated based on the size of the bundle
* + mtime is dynamically calculated, based on the age of the youngest item
* + sendas (nameas) is name + [.pkt|.tic]
*
* = Need to know the DB ID for the items being sent, so as to mark them sent
*
* + Files
* + name is the full path file name
* + size is the file size
* + mtime is the file mtime
* + sendas (nameas) is filename only
*
* For receiving, we just received files, and based on its name, we determine if it is a mail/file/tic or request
* When receiving, we receive one file at a time, and our list has the list of files we've recevied
*
* + name our name (including path) with the address ID prefixed of the sender
* + size is advised to us by the remote
* + mtime is advised to us by the remote
* + receiveas (nameas) advised to us by the remote
*
* @property string $name The (full path) name of the file we are sending or receiving
* @property string $nameas The name as we communicate with the remote
* @property int $mtime The date/time timestamp of the item we are sending or receiving
* @property int $pos Our counter of where we have read to when sending
* @property int $size The size of the item we are sending or receiving
*/
abstract class Base
{
private const LOGKEY = 'I--';
// @todo MAX_COMPSIZE hasnt been implemented in RECEIVE OR SEND
/** @var int max size of file to use compression */
protected const MAX_COMPSIZE = 0x1fff;
// 4 BITS of type
protected const IS_FILE = (1<<0);
protected const IS_PKT = (1<<1);
protected const IS_ARC = (1<<2);
protected const IS_REQ = (1<<3);
protected const IS_TIC = (1<<4);
/** Current read/write pointer */
protected int $pos;
/** File descriptor - for receiving this is the file resource, for sending, this is a string of the data to send */
protected mixed $f;
/** @var int Type of Item - first 8 bits are the type of item, the last 8 bits are dependent on the send/receive T_* values */
protected int $ftype;
/** @var Collection The list of items we are sending */
protected Collection $list;
/** @var int|null The item in the list that we are sending/receiving */
protected ?int $index = NULL;
/** @var bool Item has been sent or received */
protected bool $complete = FALSE;
/** Time we started sending/receiving */
protected int $start;
protected function isType(int $type): bool
{
return ($this->ftype&0xff) & $type;
}
protected function whatType(): int
{
static $ext = ['su','mo','tu','we','th','fr','sa','req'];
$x = strrchr($this->full_name,'.');
if (! $x || (strlen(substr($x,1)) != 3))
return self::IS_FILE;
if (strcasecmp(substr($x,1),'pkt') === 0)
return self::IS_PKT;
if (strcasecmp(substr($x,1),'req') === 0)
return self::IS_REQ;
if (strcasecmp(substr($x,1),'tic') === 0)
return self::IS_TIC;
for ($i=0;$i<count($ext);$i++)
if (! strncasecmp($x,'.'.$ext[$i],strlen($ext[$i]))
&& (preg_match('/^[0-9a-z]/',strtolower(substr($x,3,1)))))
return self::IS_ARC;
return self::IS_FILE;
}
}

101
app/Classes/File/File.php Normal file
View File

@@ -0,0 +1,101 @@
<?php
namespace App\Classes\File;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use App\Models\File as FileModel;
final class File extends Send
{
private const LOGKEY = 'ISF';
/** @var mixed Our open file descriptor */
private mixed $fd;
/**
* @throws \Exception
*/
public function __construct(FileModel $file,int $type)
{
parent::__construct();
$this->f = $file;
$this->ftype = ((($type&0xff)<<8)|self::IS_FILE);
}
public function __get($key) {
switch ($key) {
case 'dbids':
return collect([$this->f->id]);
case 'full_name':
case 'nameas':
return $this->f->name;
case 'mtime':
return $this->f->datetime->timestamp;
case 'name':
case 'size':
return $this->f->{$key};
case 'type':
return ($this->ftype&0xff00)>>8;
default:
return parent::__get($key);
}
}
public function close(bool $successful): void
{
if ($successful)
$this->complete = TRUE;
fclose($this->fd);
}
public function feof(): bool
{
return feof($this->fd);
}
/**
* Open a file for sending
*
* @param string $compress The compression method that will be used (not implemented)
* @return bool
*/
public function open(string $compress=''): bool
{
// If sending file is a File::class, then our file is s3
if ($this->nameas && $this->f instanceof FileModel) {
$this->fd = Storage::readStream($this->f->full_storage_path);
} else {
$this->fd = fopen($this->full_name,'rb');
if (! $this->fd) {
Log::error(sprintf('%s:! Unable to open file [%s] for reading',self::LOGKEY,$this->full_name));
return FALSE;
}
Log::info(sprintf('%s:= File [%s] opened with size [%d]',self::LOGKEY,$this->full_name,$this->size));
}
return TRUE;
}
public function read(int $length): string
{
return fread($this->fd,$length);
}
public function seek(int $pos): bool
{
return (fseek($this->f,$pos,SEEK_SET) === 0);
}
}

205
app/Classes/File/Item.php
View File

@@ -2,187 +2,68 @@
namespace App\Classes\File;
use Illuminate\Contracts\Filesystem\FileNotFoundException;
use Illuminate\Support\Facades\Storage;
use League\Flysystem\UnreadableFileEncountered;
use App\Models\File;
use App\Models\Address;
/**
* A file we are sending or receiving
*
* @property string $name
* @property string $recvas
* @property int $size
*/
class Item
final class Item extends Receive
{
private const LOGKEY = 'I--';
private const LOCATION = 'local';
// For deep debugging
protected bool $DEBUG = FALSE;
/** @var int max size of file to use compression */
// @todo MAX_COMPSIZE hasnt been implemented in RECEIVE OR SEND
protected const MAX_COMPSIZE = 0x1fff;
protected const IS_PKT = (1<<1);
protected const IS_ARC = (1<<2);
protected const IS_FILE = (1<<3);
protected const IS_FLO = (1<<4);
protected const IS_REQ = (1<<5);
protected const IS_TIC = (1<<6);
protected const I_RECV = (1<<0);
protected const I_SEND = (1<<1);
protected string $file_name;
protected int $file_size;
protected int $file_mtime;
/** Current read/write pointer */
protected int $file_pos = 0;
/** File descriptor */
protected mixed $f = NULL;
protected int $type;
protected int $action;
protected File $filemodel;
public bool $sent = FALSE;
public bool $received = FALSE;
public bool $incomplete = FALSE;
/** Time we started sending/receiving */
protected int $start;
/** @var Address The address that sent us this item */
private Address $ao;
private string $recvas;
private int $recvmtime;
private int $recvsize;
/**
* @throws FileNotFoundException
* @throws UnreadableFileEncountered
* @throws \Exception
*/
public function __construct($file,int $action)
public function __construct(Address $ao,string $recvas,int $mtime,int $size)
{
switch ($action) {
case self::I_SEND:
if ($file instanceof File) {
$this->filemodel = $file;
// @todo We should catch any exceptions if the default storage is s3 (it is) and we cannot find the file, or the s3 call fails
$this->file_size = Storage::size($file->full_storage_path);
$this->file_mtime = Storage::lastModified($file->full_storage_path);
parent::__construct();
} else {
if (! is_string($file))
throw new \Exception('Invalid object creation - file should be a string');
$this->ao = $ao;
$this->recvas = $recvas;
$this->recvmtime = $mtime;
$this->recvsize = $size;
if (! file_exists($file))
throw new FileNotFoundException('Item doesnt exist: '.$file);
if (! is_readable($file))
throw new UnreadableFileEncountered('Item cannot be read: '.$file);
$this->file_name = $file;
$x = stat($file);
$this->file_size = $x['size'];
$this->file_mtime = $x['mtime'];
}
break;
case self::I_RECV;
$keys = ['name','mtime','size'];
if (! is_array($file) || array_diff(array_keys($file),$keys))
throw new \Exception('Invalid object creation - file is not a valid array :'.serialize(array_diff(array_keys($file),$keys)));
$this->file_name = $file['name'];
$this->file_size = $file['size'];
$this->file_mtime = $file['mtime'];
break;
default:
throw new \Exception('Unknown action: '.$action);
}
$this->action = $action;
$this->type = $this->whatType();
$this->ftype = self::IS_FILE;
}
/**
* @throws \Exception
*/
public function __get($key)
{
switch($key) {
public function __get($key) {
switch ($key) {
case 'exists':
return Storage::disk(self::LOCATION)->exists($this->rel_name);
case 'rel_name':
return sprintf('%s/%04X-%s',config('app.fido'),$this->ao->id,$this->recvas);
case 'full_name':
return Storage::disk(self::LOCATION)->path($this->rel_name);
case 'match_mtime':
return $this->mtime === $this->recvmtime;
case 'match_size':
return $this->size === $this->recvsize;
case 'nameas':
return $this->recvas;
case 'recvmtime':
return $this->recvmtime;
case 'recvsize':
return $this->recvsize;
case 'name_size_time':
return sprintf('%s %lu %lu',$this->recvas,$this->recvsize,$this->recvmtime);
case 'mtime':
if ($this instanceof Mail)
$this->youngest()->timestamp;
if ($this->action & self::I_RECV|self::I_SEND)
return $this->{'file_'.$key};
throw new \Exception('Invalid request for key: '.$key);
case 'name':
if ($this instanceof Mail)
return sprintf('%08x',timew($this->youngest()));
if ($this->action & self::I_RECV|self::I_SEND)
return $this->{'file_'.$key};
throw new \Exception('Invalid request for key: '.$key);
return Storage::disk(self::LOCATION)->lastModified($this->rel_name);
case 'size':
if ($this instanceof Mail)
return strlen($this->file);
if ($this->action & self::I_RECV|self::I_SEND)
return $this->{'file_'.$key};
throw new \Exception('Invalid request for key: '.$key);
case 'recvas':
return $this->file_name;
case 'sendas':
if ($this instanceof Mail)
return sprintf('%s.pkt',$this->name);
return $this->file_name ? basename($this->file_name) : $this->filemodel->name;
return Storage::disk(self::LOCATION)->size($this->rel_name);
default:
throw new \Exception('Unknown key: '.$key);
return parent::__get($key);
}
}
protected function isType(int $type): bool
{
return $this->type & $type;
}
private function whatType(): int
{
static $ext = ['su','mo','tu','we','th','fr','sa','req'];
$x = strrchr($this->file_name,'.');
if (! $x || (strlen(substr($x,1)) != 3))
return self::IS_FILE;
if (strcasecmp(substr($x,2),'lo') === 0)
return self::IS_FLO;
if (strcasecmp(substr($x,1),'pkt') === 0)
return self::IS_PKT;
if (strcasecmp(substr($x,1),'req') === 0)
return self::IS_REQ;
if (strcasecmp(substr($x,1),'tic') === 0)
return self::IS_TIC;
for ($i=0;$i<count($ext);$i++)
if (! strncasecmp($x,'.'.$ext[$i],strlen($ext[$i])) && (preg_match('/^[0-9a-z]/',strtolower(substr($x,3,1)))))
return self::IS_ARC;
return self::IS_FILE;
}
}

72
app/Classes/File/Mail.php
View File

@@ -6,44 +6,80 @@ use Carbon\Carbon;
use App\Classes\FTN\Packet;
class Mail extends Item
final class Mail extends Send
{
private Packet $file;
/** @var int Our internal position counter */
private int $readpos;
/**
* @throws \Exception
*/
public function __construct(Packet $mail,int $action)
public function __construct(Packet $mail,int $type)
{
switch ($action) {
case self::I_SEND:
$this->file = $mail;
parent::__construct();
break;
default:
throw new \Exception('Unknown action: '.$action);
}
$this->action = $action;
$this->f = $mail;
$this->ftype = ((($type&0xff)<<8)|self::IS_PKT);
$this->readpos = 0;
}
public function __get($key) {
switch ($key) {
case 'file': return $this->file;
case 'messages': return $this->file->messages;
case 'dbids':
return $this->f->messages->pluck('dbid');
case 'name':
return sprintf('%08x',timew($this->youngest()));
case 'nameas':
return sprintf('%s.pkt',$this->name);
case 'mtime':
return $this->youngest()->timestamp;
case 'size':
return strlen($this->f);
case 'type':
return ($this->ftype&0xff00)>>8;
default:
return parent::__get($key);
}
}
public function read(int $start,int $length): string
public function close(bool $successful): void
{
return substr((string)$this->file,$start,$length);
if ($successful)
$this->complete = TRUE;
}
public function feof(): bool
{
return ($this->readpos === $this->size);
}
public function open(string $compress=''): bool
{
return TRUE;
}
public function read(int $length): string
{
$result = substr((string)$this->f,$this->readpos,$length);
$this->readpos += strlen($result);
return $result;
}
public function seek(int $pos): bool
{
$this->readpos = ($pos < $this->size) ? $pos : $this->size;
return TRUE;
}
public function youngest(): Carbon
{
return $this->file->messages->pluck('date')->sort()->last();
return $this->f->messages->pluck('date')->sort()->last();
}
}

180
app/Classes/File/Receive.php
View File

@@ -21,7 +21,7 @@ use App\Models\Address;
* @property-read int total_recv
* @property-read int total_recv_bytes
*/
final class Receive extends Item
class Receive extends Base
{
private const LOGKEY = 'IR-';
@@ -31,10 +31,7 @@ final class Receive extends Item
];
private Address $ao;
private Collection $list;
private ?Item $receiving;
private string $file; // Local filename for file received
/** @var ?string The compression used by the incoming file */
private ?string $comp;
/** @var string|null The compressed data received */
@@ -43,41 +40,47 @@ final class Receive extends Item
public function __construct()
{
// Initialise our variables
$this->list = collect();
$this->receiving = NULL;
if (get_class($this) === self::class) {
$this->list = collect();
$this->f = NULL;
}
}
public function __get($key)
{
switch ($key) {
case 'completed':
return $this->list
->filter(function($item) { return $item->complete === TRUE; });
case 'fd':
return is_resource($this->f);
case 'filepos':
return $this->file_pos;
case 'mtime':
case 'name':
case 'nameas':
case 'size':
return $this->receiving?->{'file_'.$key};
case 'name_size_time':
return sprintf('%s %lu %lu',$this->name,$this->size,$this->mtime);
return $this->receiving->{$key};
case 'to_get':
case 'pos':
return $this->pos;
case 'receiving':
return $this->list->get($this->index);
case 'ready':
return (! is_null($this->index));
case 'togo_count':
return $this->list
->filter(function($item) { return ($item->action & self::I_RECV) && $item->received === FALSE; })
->filter(function($item) { return $item->complete === FALSE; })
->count();
case 'total_recv':
return $this->list
->filter(function($item) { return ($item->action & self::I_RECV) && $item->received === TRUE; })
->count();
return $this->completed->count();
case 'total_recv_bytes':
return $this->list
->filter(function($item) { return ($item->action & self::I_RECV) && $item->received === TRUE; })
->sum(function($item) { return $item->size; });
return $this->completed->sum(function($item) { return $item->size; });
default:
throw new \Exception('Unknown key: '.$key);
@@ -95,36 +98,31 @@ final class Receive extends Item
throw new \Exception('No file to close');
if ($this->f) {
if ($this->file_pos !== $this->receiving->size) {
Log::warning(sprintf('%s:- Closing [%s], but missing [%d] bytes',self::LOGKEY,$this->receiving->name,$this->receiving->size-$this->file_pos));
$this->receiving->incomplete = TRUE;
}
$this->receiving->received = TRUE;
if ($this->pos !== $this->receiving->recvsize)
Log::warning(sprintf('%s:- Closing [%s], but missing [%d] bytes',self::LOGKEY,$this->receiving->nameas,$this->receiving->recvsize-$this->pos));
else
$this->receiving->complete = TRUE;
$end = time()-$this->start;
Log::debug(sprintf('%s:- Closing [%s], received in [%d]',self::LOGKEY,$this->receiving->name,$end));
Log::debug(sprintf('%s:- Closing [%s], received in [%d]',self::LOGKEY,$this->receiving->nameas,$end));
if ($this->comp)
Log::info(sprintf('%s:= Compressed file using [%s] was [%d] bytes (%d). Compression rate [%3.2f%%]',self::LOGKEY,$this->comp,$x=strlen($this->comp_data),$this->receiving->size,$x/$this->receiving->size*100));
Log::info(sprintf('%s:= Compressed file using [%s] was [%d] bytes (%d). Compression rate [%3.2f%%]',self::LOGKEY,$this->comp,$x=strlen($this->comp_data),$this->receiving->recvsize,$x/$this->receiving->recvsize*100));
fclose($this->f);
// Set our mtime
touch($this->file,$this->mtime);
$this->file_pos = 0;
touch($this->receiving->full_name,$this->receiving->mtime);
$this->f = NULL;
// If the packet has been received but not the right size, dont process it any more.
// If we received a packet, we'll dispatch a job to process it
if (! $this->receiving->incomplete)
switch ($this->receiving->type) {
// If we received a packet, we'll dispatch a job to process it, if we got it all
if ($this->receiving->complete)
switch ($x=$this->receiving->whatType()) {
case self::IS_ARC:
case self::IS_PKT:
Log::info(sprintf('%s:- Processing mail %s [%s]',self::LOGKEY,$this->receiving->type === self::IS_PKT ? 'PACKET' : 'ARCHIVE',$this->file));
Log::info(sprintf('%s:- Processing mail %s [%s]',self::LOGKEY,$x === self::IS_PKT ? 'PACKET' : 'ARCHIVE',$this->receiving->nameas));
try {
$f = new File($this->file);
$f = new File($this->receiving->full_name);
$processed = FALSE;
foreach ($f as $packet) {
@@ -177,54 +175,72 @@ final class Receive extends Item
}
if (! $processed) {
Log::alert(sprintf('%s:- Not deleting packet [%s], it doesnt seem to be processed?',self::LOGKEY,$this->file));
Log::alert(sprintf('%s:- Not deleting packet [%s], it doesnt seem to be processed?',self::LOGKEY,$this->receiving->nameas));
// If we want to keep the packet, we could do that logic here
} elseif (! config('app.packet_keep')) {
Log::debug(sprintf('%s:- Deleting processed packet [%s]',self::LOGKEY,$this->file));
unlink($this->file);
Log::debug(sprintf('%s:- Deleting processed packet [%s]',self::LOGKEY,$this->receiving->full_name));
unlink($this->receiving->full_name);
}
} catch (InvalidPacketException $e) {
Log::error(sprintf('%s:- Not deleting packet [%s], as it generated an InvalidPacketException',self::LOGKEY,$this->file),['e'=>$e->getMessage()]);
Log::error(sprintf('%s:- Not deleting packet [%s], as it generated an InvalidPacketException',self::LOGKEY,$this->receiving->nameas),['e'=>$e->getMessage()]);
} catch (\Exception $e) {
Log::error(sprintf('%s:- Not deleting packet [%s], as it generated an uncaught exception',self::LOGKEY,$this->file),['e'=>$e->getMessage()]);
Log::error(sprintf('%s:- Not deleting packet [%s], as it generated an uncaught exception',self::LOGKEY,$this->receiving->nameas),['e'=>$e->getMessage()]);
}
break;
case self::IS_TIC:
Log::info(sprintf('%s:- Processing TIC file [%s]',self::LOGKEY,$this->file));
Log::info(sprintf('%s:- Processing TIC file [%s]',self::LOGKEY,$this->receiving->nameas));
// Queue the tic to be processed later, in case the referenced file hasnt been received yet
TicProcess::dispatch($this->file);
TicProcess::dispatch($this->receiving->nameas);
break;
default:
Log::debug(sprintf('%s:- Leaving file [%s] in the inbound dir',self::LOGKEY,$this->file));
Log::debug(sprintf('%s:- Leaving file [%s] in the inbound dir',self::LOGKEY,$this->receiving->nameas));
}
}
$this->receiving = NULL;
$this->index = NULL;
}
/**
* Add a new file to receive
*
* @param array $file
* @param Address $ao
* @throws \Exception
*/
public function new(array $file,Address $ao): void
{
Log::debug(sprintf('%s:+ Receiving new file [%s]',self::LOGKEY,join('|',$file)));
if ($this->index)
throw new \Exception('Can only have 1 file receiving at a time');
$this->ao = $ao;
$this->list->push(new Item($ao,Arr::get($file,'name'),(int)Arr::get($file,'mtime'),(int)Arr::get($file,'size')));
$this->index = $this->list->count()-1;
}
/**
* Open the file descriptor to receive a file
*
* @param Address $ao
* @param bool $check
* @param string|null $comp If the incoming file will be compressed
* @return int
* @throws \Exception
* @todo $comp should be parsed, in case it contains other items
*/
public function open(Address $ao,bool $check=FALSE,string $comp=NULL): int
public function open(bool $check=FALSE,string $comp=NULL): int
{
// @todo implement return 4 - SUSPEND(?) file
// @todo Change to use Storage::class()
if (! $this->receiving)
if (is_null($this->index))
throw new \Exception('No files currently receiving');
$this->comp_data = '';
@@ -244,65 +260,37 @@ final class Receive extends Item
elseif ($this->comp)
Log::debug(sprintf('%s:- Receiving file with [%s] compression',self::LOGKEY,$this->comp));
$this->ao = $ao;
$this->file_pos = 0;
$this->pos = 0;
$this->start = time();
$this->file = sprintf('storage/app/%s',$this->local_path($ao));
if (file_exists($this->file)
&& (Storage::disk('local')->lastModified($this->local_path($ao)) === $this->mtime)
&& (Storage::disk('local')->size($this->local_path($ao)) === $this->size))
{
Log::alert(sprintf('%s:- File already exists - skipping [%s]', self::LOGKEY, $this->file));
if ($this->receiving->exists && $this->receiving->match_mtime && $this->receiving->match_size) {
Log::alert(sprintf('%s:- File already exists - skipping [%s]', self::LOGKEY,$this->receiving->nameas));
return Protocol::FOP_GOT;
} elseif (file_exists($this->file) && (Storage::disk('local')->size($this->local_path($ao)) > 0)) {
Log::alert(sprintf('%s:- File exists with different details - skipping [%s]',self::LOGKEY,$this->file));
} elseif ($this->receiving->exists && $this->receiving->size > 0) {
Log::alert(sprintf('%s:- File exists with different details - skipping [%s] (size: %d, mtime: %d)',self::LOGKEY,$this->receiving->nameas,$this->receiving->size,$this->receiving->mtime));
return Protocol::FOP_SKIP;
} else {
// @todo I dont think we are enabling resumable sessions - need to check
Log::debug(sprintf('%s:- Opening [%s]',self::LOGKEY,$this->file));
Log::debug(sprintf('%s:- Opening [%s]',self::LOGKEY,$this->receiving->nameas));
}
// If we are only checking, we'll return (NR mode)
if ($check)
return Protocol::FOP_OK;
$this->f = fopen($this->file,'wb');
$this->f = fopen($this->receiving->full_name,'wb');
if (! $this->f) {
Log::error(sprintf('%s:! Unable to open file [%s] for writing',self::LOGKEY,$this->receiving->name));
Log::error(sprintf('%s:! Unable to open file [%s] for writing',self::LOGKEY,$this->receiving->nameas));
return Protocol::FOP_ERROR;
}
Log::info(sprintf('%s:= open - File [%s] opened for writing',self::LOGKEY,$this->receiving->name));
Log::info(sprintf('%s:= File [%s] opened for writing',self::LOGKEY,$this->receiving->nameas));
return Protocol::FOP_OK;
}
/**
* Add a new file to receive
*
* @param array $file
* @throws \Exception
*/
public function new(array $file): void
{
Log::debug(sprintf('%s:+ new [%s]',self::LOGKEY,join('|',$file)));
if ($this->receiving)
throw new \Exception('Can only have 1 file receiving at a time');
$o = new Item($file,self::I_RECV);
$this->list->push($o);
$this->receiving = $o;
}
private function local_path(Address $ao): string
{
return sprintf('%s/%04X-%s',config('app.fido'),$ao->id,$this->receiving->recvas);
}
/**
* Write data to the file we are receiving
*
@@ -312,7 +300,7 @@ final class Receive extends Item
*/
public function write(string $buf): bool
{
if (! $this->f)
if (! $this->fd)
throw new \Exception('No file open for write');
$data = '';
@@ -354,19 +342,19 @@ final class Receive extends Item
$data = $buf;
}
if ($this->file_pos+strlen($data) > $this->receiving->size)
throw new \Exception(sprintf('Too many bytes received [%d] (%d)?',$this->file_pos+strlen($buf),$this->receiving->size));
if (($x=$this->pos+strlen($data)) > $this->receiving->recvsize)
throw new \Exception(sprintf('Too many bytes received [%d] (%d)?',$x,$this->receiving->recvsize));
$rc = fwrite($this->f,$data);
if ($rc === FALSE)
throw new FileException('Error while writing to file');
$this->file_pos += $rc;
Log::debug(sprintf('%s:- Write [%d] bytes, file pos now [%d] of [%d] (%d)',self::LOGKEY,$rc,$this->file_pos,$this->receiving->size,strlen($buf)));
$this->pos += $rc;
Log::debug(sprintf('%s:- Write [%d] bytes, file pos now [%d] of [%d] (%d)',self::LOGKEY,$rc,$this->pos,$this->receiving->recvsize,strlen($buf)));
if (strlen($this->comp_data) > $this->receiving->size)
Log::alert(sprintf('%s:- Compression grew the file during transfer (%d->%d)',self::LOGKEY,$this->receiving->size,strlen($this->comp_data)));
if (strlen($this->comp_data) > $this->receiving->recvsize)
Log::alert(sprintf('%s:- Compression grew the file during transfer (%d->%d)',self::LOGKEY,$this->receiving->recvsize,strlen($this->comp_data)));
return TRUE;
}

259
app/Classes/File/Send.php
View File

@@ -3,10 +3,7 @@
namespace App\Classes\File;
use Exception;
use Illuminate\Contracts\Filesystem\FileNotFoundException;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use League\Flysystem\UnreadableFileEncountered;
use App\Models\Address;
@@ -15,37 +12,45 @@ use App\Models\Address;
* Object representing the files we are sending
*
* @property-read resource fd
* @property-read int mail_size
* @property-read int total_count
* @property-read int files_size The size of the files waiting to be sent
* @property-read int mail_size The size of the mail waiting to be sent
* @property-read int togo_count The total number of items that havent been sent yet
* @property-read int total_size The size of the items waiting to be sent
* @property-read int total_sent
* @property-read int total_sent_bytes
*/
final class Send extends Item
class Send extends Base
{
private const LOGKEY = 'IS-';
private Collection $list;
private ?Item $sending;
private Collection $packets;
public const T_NONE = 0;
/** @var int This file contains a file from the DB */
public const T_FILE = (1<<0);
/** @var int This file contains a bundle of Netmail */
public const T_NETMAIL = (1<<1);
/** @var int This file contains a bundle of Echomail */
public const T_ECHOMAIL = (1<<2);
private string $comp_data;
public function __construct()
{
// Initialise our variables
$this->list = collect();
$this->packets = collect();
$this->sending = NULL;
if (get_class($this) === self::class) {
$this->list = collect();
$this->f = NULL;
}
}
public function __get($key)
{
switch ($key) {
case 'dbids':
return $this->sending->messages->pluck('echoarea','dbid');
case 'completed':
return $this->list
->filter(function($item) { return $item->complete === TRUE; });
case 'fd':
return is_resource($this->f) ?: $this->f;
return ! is_null($this->index);
case 'files_count':
return $this->list
@@ -57,63 +62,46 @@ final class Send extends Item
->filter(function($item) { return $item->isType(self::IS_FILE|self::IS_TIC); })
->sum(function($item) { return $item->size; });
case 'filepos':
return $this->file_pos;
case 'mail_count':
return $this->list
->filter(function($item) { return $item->isType(self::IS_ARC|self::IS_PKT); })
->count()
+ $this->packets->count();
->count();
case 'mail_size':
return $this->list
->filter(function($item) { return $item->isType(self::IS_ARC|self::IS_PKT); })
->sum(function($item) { return $item->size; })
+ $this->packets->sum(function($item) { return $item->size; });
->sum(function($item) { return $item->size; });
case 'sendas':
return $this->sending?->{$key};
// The mtime is the time of the youngest message in the packet for the sending packet
case 'mtime':
return $this->sending?->youngest()->timestamp;
// The name is derived from the youngest message in the packet
case 'dbids':
case 'name':
return sprintf('%08x',timew($this->sending?->youngest()));
case 'nameas':
case 'mtime':
case 'size':
return strlen($this->sending?->file);
case 'type':
return $this->sending->{$key};
case 'pos':
return $this->{$key};
case 'sending':
return $this->list->get($this->index);
case 'togo_count':
return $this->list
->filter(function($item) { return $item->complete === FALSE; })
->count();
case 'total_sent':
return $this->list
->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === TRUE; })
->count()
+ $this->packets
->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === TRUE; })
->count();
return $this->completed
->count();
case 'total_sent_bytes':
return $this->list
->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === TRUE; })
->sum(function($item) { return $item->size; })
+ $this->packets
->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === TRUE; })
->sum(function($item) { return $item->size; });
case 'total_count':
return $this->list
->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === FALSE; })
->count()
+ $this->packets
->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === FALSE; })
->count();
return $this->completed
->sum(function($item) { return $item->size; });
case 'total_size':
return $this->list
->sum(function($item) { return $item->size; })
+ $this->packets->sum(function($item) { return $item->size; });
->sum(function($item) { return $item->size; });
default:
throw new Exception('Unknown key: '.$key);
@@ -121,31 +109,23 @@ final class Send extends Item
}
/**
* Add a file to the list of files to send
* Close the file descriptor of the file we are sending
*
* @param string $file
* @param bool $successful
* @throws Exception
* @todo Catch if we add the same file twice
*/
public function add(string $file): void
public function close(bool $successful): void
{
Log::debug(sprintf('%s:+ add [%s]',self::LOGKEY,$file));
if (! $this->fd)
throw new Exception('No file to close');
try {
$this->list->push(new Item($file,self::I_SEND));
} catch (FileNotFoundException) {
Log::error(sprintf('%s:! Item [%s] doesnt exist',self::LOGKEY,$file));
return;
} catch (UnreadableFileEncountered) {
Log::error(sprintf('%s:! Item [%s] cannot be read',self::LOGKEY,$file));
return;
// Uncaught, rethrow the error
} catch (Exception $e) {
throw new Exception($e->getMessage());
if ($successful) {
$end = time()-$this->start;
Log::debug(sprintf('%s: - Closing [%s], sent in [%d]',self::LOGKEY,$this->sending->nameas,$end));
}
$this->sending->close($successful);
$this->index = NULL;
}
/*
@@ -163,32 +143,6 @@ final class Send extends Item
}
*/
/**
* Close the file descriptor of the file we are sending
*
* @param bool $successful
* @throws Exception
*/
public function close(bool $successful): void
{
if (! $this->f)
throw new Exception('No file to close');
if ($successful) {
$this->sending->sent = TRUE;
$end = time()-$this->start;
Log::debug(sprintf('%s: - Closing [%s], sent in [%d]',self::LOGKEY,$this->sending->name,$end));
}
// @todo This should be done better isType == file?
if ((! $this->sending instanceof Mail) && (! $this->sending->isType(self::IS_TIC)))
fclose($this->f);
$this->sending = NULL;
$this->file_pos = 0;
$this->f = NULL;
}
/**
* Check if we are at the end of the file
*
@@ -196,9 +150,7 @@ final class Send extends Item
*/
public function feof(): bool
{
return (($this->sending instanceof Mail) || ($this->sending->isType(self::IS_TIC)))
? ($this->file_pos === $this->size)
: feof($this->f);
return $this->sending->feof();
}
/**
@@ -207,7 +159,6 @@ final class Send extends Item
* @param Address $ao
* @return bool
* @throws Exception
* @todo We need to make this into a transaction, incase the transfer fails.
*/
public function files(Address $ao): bool
{
@@ -221,13 +172,13 @@ final class Send extends Item
}
// Files
if (($x=$ao->getFiles())->count()) {
if (($x=$ao->filesWaiting())->count()) {
Log::debug(sprintf('%s:- [%d] Files(s) added for sending to [%s]',self::LOGKEY,$x->count(),$ao->ftn));
// Add Files
foreach ($x as $fo) {
$this->list->push(new Item($fo,self::I_SEND));
$this->list->push(new Tic($ao,$fo,self::I_SEND));
$this->list->push(new File($fo,self::T_FILE));
$this->list->push(new Tic($fo,$ao,self::T_NONE));
}
$file = TRUE;
@@ -247,16 +198,13 @@ final class Send extends Item
{
Log::debug(sprintf('%s:+ Opening file to send',self::LOGKEY));
// If we have mail, we'll send that first
if ($this->sending = $this->packets
->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === FALSE; })
->first())
if ((($this->index=$this->list->search(function($item) { return $item->complete === FALSE; })) !== FALSE)
&& $this->sending->open())
{
Log::debug(sprintf('%s:- Sending [%s]',self::LOGKEY,$this->sending->name));
Log::debug(sprintf('%s:- Sending item [%d] (%s)',self::LOGKEY,$this->index,$this->sending->nameas));
$this->file_pos = 0;
$this->pos = 0;
$this->start = time();
$this->f = TRUE;
/*
if ($compress)
@@ -264,39 +212,9 @@ final class Send extends Item
*/
return TRUE;
}
$this->sending = $this->list
->filter(function($item) { return ($item->action & self::I_SEND) && $item->sent === FALSE; })
->first();
if (! $this->sending)
throw new Exception('No files to open');
$this->file_pos = 0;
$this->start = time();
// If sending->file is a string, then we dont need to actually open anything
if ($this->sending->isType(self::IS_TIC)) {
$this->f = TRUE;
return TRUE;
}
// If sending file is a File::class, then our file is s3
if (! $this->sending->name && $this->sending->filemodel) {
$this->f = Storage::readStream($this->sending->filemodel->full_storage_path);
return TRUE;
} else {
$this->f = fopen($this->sending->name,'rb');
if (! $this->f) {
Log::error(sprintf('%s:! Unable to open file [%s] for reading',self::LOGKEY,$this->sending->name));
return FALSE;
}
Log::info(sprintf('%s:= open - File [%s] opened with size [%d]',self::LOGKEY,$this->sending->name,$this->sending->size));
return TRUE;
throw new Exception('No files to open');
}
}
@@ -304,9 +222,9 @@ final class Send extends Item
* Add our mail to the send queue
*
* @param Address $ao
* @param bool $update
* @return bool
* @throws Exception
* @todo We need to make this into a transaction, incase the transfer fails.
*/
public function mail(Address $ao,bool $update=TRUE): bool
{
@@ -323,7 +241,7 @@ final class Send extends Item
if ($x=$ao->getNetmail($update)) {
Log::debug(sprintf('%s: - Netmail(s) added for sending to [%s]',self::LOGKEY,$ao->ftn));
$this->packets->push(new Mail($x,self::I_SEND));
$this->list->push(new Mail($x,self::T_NETMAIL));
$mail = TRUE;
}
@@ -331,7 +249,7 @@ final class Send extends Item
if ($x=$ao->getEchomail($update)) {
Log::debug(sprintf('%s: - Echomail(s) added for sending to [%s]',self::LOGKEY,$ao->ftn));
$this->packets->push(new Mail($x,self::I_SEND));
$this->list->push(new Mail($x,self::T_ECHOMAIL));
$mail = TRUE;
}
@@ -348,29 +266,18 @@ final class Send extends Item
*/
public function read(int $length): ?string
{
if (! $this->f)
if (! $this->fd)
throw new Exception('No file open for read');
// We are sending mail
if ($this->sending instanceof Mail) {
$data = $this->sending->read($this->file_pos,$length);
// We are sending a tic file
} else if ($this->sending->isType(self::IS_TIC)) {
$data = $this->sending->read($this->file_pos,$length);
} else {
$data = fread($this->f,$length);
}
$this->file_pos += strlen($data);
if ($this->DEBUG)
Log::debug(sprintf('%s: - Read [%d] bytes, file pos now [%d]',self::LOGKEY,strlen($data),$this->file_pos));
$data = $this->sending->read($length);
if ($data === FALSE)
throw new UnreadableFileEncountered('Error reading file: '.$this->sending->name);
$this->pos += strlen($data);
Log::debug(sprintf('%s:- Read [%d] bytes, file pos now [%d]',self::LOGKEY,strlen($data),$this->pos));
return $data;
}
@@ -383,22 +290,20 @@ final class Send extends Item
*/
public function seek(int $pos): bool
{
if (! $this->f)
if (! $this->fd)
throw new Exception('No file open for seek');
if (($this->sending instanceof Mail) || $this->sending->isType(self::IS_TIC)) {
$pos = ($pos < $this->size) ? $pos : $this->size;
$rc = TRUE;
if ($this->sending->seek($pos)) {
$this->pos = $pos;
Log::debug(sprintf('%s:= Seeked to [%d]',self::LOGKEY,$this->pos));
return TRUE;
} else {
$rc = (fseek($this->f,$pos,SEEK_SET) === 0);
Log::error(sprintf('%s:! Failed to seek to [%d]',self::LOGKEY,$pos));
return FALSE;
}
if ($rc)
$this->file_pos = $pos;
Log::debug(sprintf('%s:= Seeked to [%d]',self::LOGKEY,$this->file_pos));
return $rc;
}
}

82
app/Classes/File/Tic.php
View File

@@ -2,38 +2,84 @@
namespace App\Classes\File;
use App\Models\Address;
use App\Models\File;
use App\Classes\FTN\Tic as FTNTic;
use App\Models\{Address,File};
class Tic extends Item
final class Tic extends Send
{
private string $file;
/** @var int Our internal position counter */
private int $readpos;
private Address $ao;
private string $tic;
/**
* @throws \Exception
*/
public function __construct(Address $ao,File $fo,int $action)
public function __construct(File $file,Address $ao,int $type)
{
switch ($action) {
case self::I_SEND:
$tic = new FTNTic;
$this->file = $tic->generate($ao,$fo);
$this->file_name = sprintf('%s.tic',sprintf('%08x',$fo->id));
$this->file_size = strlen($this->file);
$this->file_mtime = $fo->created_at->timestamp;
parent::__construct();
break;
$this->f = $file;
$this->ao = $ao;
$this->ftype = ((($type&0xff)<<8)|self::IS_TIC);
$this->readpos = 0;
$this->tic = FTNTic::generate($ao,$file);
}
public function __get($key) {
switch ($key) {
case 'dbids':
return collect([$this->f->id]);
case 'name':
return sprintf('%08x',timew($this->f->created_at));
case 'nameas':
return sprintf('%s.tic',$this->name);
case 'mtime':
return $this->f->datetime->timestamp;
case 'size':
return strlen($this->tic);
case 'type':
return ($this->ftype&0xff00)>>8;
default:
throw new \Exception('Unknown action: '.$action);
return parent::__get($key);
}
$this->action = $action;
$this->type = self::IS_TIC;
}
public function read(int $start,int $length): string
public function close(bool $successful): void
{
return substr($this->file,$start,$length);
if ($successful)
$this->complete = TRUE;
}
public function feof(): bool
{
return ($this->readpos === $this->size);
}
public function open(string $compress=''): bool
{
return TRUE;
}
public function read(int $length): string
{
$result = substr($this->tic,$this->readpos,$length);
$this->readpos += strlen($result);
return $result;
}
public function seek(int $pos): bool
{
$this->readpos = ($pos < $this->size) ? $pos : $this->size;
return TRUE;
}
}