Added file areas and TIC processing

app/Classes/FTN/Tic.php

@@ -0,0 +1,227 @@
+<?php
+
+namespace App\Classes\FTN;
+
+use Carbon\Carbon;
+use Illuminate\Contracts\Filesystem\FileNotFoundException;
+use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Log;
+use League\Flysystem\UnableToWriteFile;
+
+use App\Classes\FTN as FTNBase;
+use App\Models\{Address, File, Filearea, Setup};
+use App\Traits\EncodeUTF8;
+
+/**
+ * Class TIC
+ *
+ * @package App\Classes
+ */
+class Tic extends FTNBase
+{
+	use EncodeUTF8;
+
+	private const LOGKEY = 'FM-';
+
+	private const cast_utf8 = [
+	];
+
+	// Single value kludge items and whether they are required
+	// http://ftsc.org/docs/fts-5006.001
+	private array $_kludge = [
+		'AREA' => TRUE,
+		'areadesc' => FALSE,
+		'ORIGIN' => TRUE,
+		'FROM' => TRUE,
+		'to' => FALSE,
+		'FILE' => TRUE,			// 8.3 DOS format
+		'lfile' => FALSE,		// alias fullname
+		'fullname' => FALSE,
+		'size' => FALSE,
+		'date' => FALSE,		// File creation date
+		'desc' => FALSE,		// One line description of file
+		'ldesc' => FALSE,		// Can have multiple
+		'created' => FALSE,
+		'magic' => FALSE,
+		'replaces' => FALSE,	// ? and * are wildcards, as per DOS
+		'CRC' => TRUE,			// crc-32
+		'PATH' => TRUE,			// can have multiple: [FTN] [unix timestamp] [datetime human readable] [signature]
+		'SEENBY' => TRUE,
+		'pw' => FALSE,			// Password
+	];
+
+	private File $file;
+	private Filearea $area;
+	private ?string $areadesc = NULL;
+	private ?string $pw = NULL;
+
+	private Address $origin;				// Should be first address in Path
+	private Address $from;					// Should be last address in Path
+	private Address $to;					// Should be me
+
+	public function __construct(private string $filename) {
+		$fo = new File;
+
+		$fo->kludges = collect();
+		$fo->set_path = collect();
+		$fo->set_seenby = collect();
+		$fo->rogue_path = collect();
+		$fo->rogue_seenby = collect();
+
+		list($hex,$name) = explode('-',$filename);
+		$ticfullpath = $this->fullpath($filename);
+
+		if (! file_exists($ticfullpath))
+			throw new FileNotFoundException(sprintf('%s:File [%s] doesnt exist',self::LOGKEY,realpath($ticfullpath)));
+
+		if (! is_writable($ticfullpath))
+			throw new UnableToWriteFile(sprintf('%s:File [%s] is not writable',self::LOGKEY,realpath($ticfullpath)));
+
+		Log::info(sprintf('Processing TIC file [%s]',$ticfullpath));
+
+		$f = fopen($ticfullpath,'rb');
+		if (! $f) {
+			Log::error(sprintf('%s:! Unable to open file [%s] for writing',self::LOGKEY,$ticfullpath));
+			return;
+		}
+
+		while (! feof($f)) {
+			$line = chop(fgets($f));
+			$matches = [];
+
+			if (! $line)
+				continue;
+
+			preg_match('/([a-zA-Z]+)\ (.*)/',$line,$matches);
+
+			if (in_array(strtolower($matches[1]),$this->_kludge)) {
+				switch ($k=strtolower($matches[1])) {
+					case 'area':
+						$this->{$k} = Filearea::singleOrNew(['name'=>strtoupper($matches[2])]);
+						break;
+
+					case 'origin':
+					case 'from':
+					case 'to':
+						$this->{$k} = Address::findFTN($matches[2]);
+
+						// @todo If $this->{$k} is null, we have discovered the system and it should be created
+						break;
+
+					case 'file':
+						if (! file_exists($x=$this->fullpath(sprintf('%s-%s',$hex,$matches[2]))))
+							throw new FileNotFoundException(sprintf('File not found? [%s]',$x));
+
+						$fo->{$k} = $matches[2];
+						$fo->fullname = $x;
+						break;
+
+					case 'areadesc':
+					case 'pw':
+					case 'created':
+						$this->{$k} = $matches[2];
+						break;
+
+					case 'lfile':
+					case 'size':
+					case 'desc':
+					case 'magic':
+					case 'replaces':
+						$fo->{$k} = $matches[2];
+						break;
+
+					case 'fullname':
+						$fo->lfile = $matches[2];
+						break;
+
+					case 'date':
+						$fo->datetime = Carbon::create($matches[2]);
+						break;
+
+					case 'ldesc':
+						$fo->{$k} .= $matches[2];
+						break;
+
+					case 'crc':
+						$fo->{$k} = hexdec($matches[2]);
+						break;
+
+					case 'path':
+						$x = [];
+						preg_match(sprintf('#^[Pp]ath (%s)\ ?([0-9]+)\ ?(.*)$#',Address::ftn_regex),$line,$x);
+						$ao = Address::findFTN($x[1]);
+
+						if (! $ao) {
+							$fo->rogue_path->push($matches[2]);
+						} else {
+							$fo->set_path->push(['address'=>$ao,'datetime'=>Carbon::createFromTimestamp($x[8]),'extra'=>$x[9]]);
+						}
+
+						break;
+
+					case 'seenby':
+						$ao = Address::findFTN($matches[2]);
+
+						if (! $ao) {
+							$fo->rogue_seenby->push($matches[2]);
+						} else {
+							$fo->set_seenby->push($ao->id);
+						}
+
+						break;
+				}
+
+			} else {
+				$fo->kludges->push($line);
+			}
+		}
+
+		fclose($f);
+
+		$f = fopen($fo->fullname,'rb');
+		$stat = fstat($f);
+		fclose($f);
+
+		// Validate Size
+		if ($fo->size !== ($y=$stat['size']))
+			throw new \Exception(sprintf('TIC file size [%d] doesnt match file [%s] (%d)',$fo->size,$fo->fullname,$y));
+
+		// Validate CRC
+		if (sprintf('%x',$fo->crc) !== ($y=hash_file('crc32b',$fo->fullname)))
+			throw new \Exception(sprintf('TIC file CRC [%x] doesnt match file [%s] (%s)',$fo->crc,$fo->fullname,$y));
+
+		// Validate Password
+		if ($this->pw !== ($y=$this->from->session('ticpass')))
+			throw new \Exception(sprintf('TIC file PASSWORD [%s] doesnt match system [%s] (%s)',$this->pw,$this->from->ftn,$y));
+
+		// Validate Sender is linked (and permitted to send)
+		if ($this->from->fileareas->search(function($item) { return $item->id === $this->area->id; }) === FALSE)
+			throw new \Exception(sprintf('Node [%s] is not subscribed to [%s]',$this->from->ftn,$this->area->name));
+
+		// If the filearea is to be autocreated, create it
+		if (! $this->area->exists) {
+			$this->area->description = $this->areadesc;
+			$this->area->active = TRUE;
+			$this->area->public = FALSE;
+			$this->area->notes = 'Autocreated';
+			$this->area->domain_id = $this->from->zone->domain_id;
+			$this->area->save();
+		}
+
+		$fo->filearea_id = $this->area->id;
+		$fo->fftn_id = $this->origin->id;
+
+		// If the file create time is blank, we'll take the files
+		if (! $fo->datetime)
+			$fo->datetime = Carbon::createFromTimestamp($stat['ctime']);
+
+		$fo->save();
+
+		$this->fo = $fo;
+	}
+
+	public function fullpath(string $file,string $prefix=NULL): string
+	{
+		return sprintf('storage/app/%s/%s',config('app.fido'),($prefix ? $prefix.'-' : '').$file);
+	}
+}

app/Classes/File/Item.php

@@ -4,7 +4,7 @@ namespace App\Classes\File;
 
 use Exception;
 use Illuminate\Contracts\Filesystem\FileNotFoundException;
-use League\Flysystem\UnreadableFileException;
+use League\Flysystem\UnreadableFileEncountered;
 
 /**
  * A file we are sending or receiving
@@ -22,6 +22,7 @@ class Item
 	protected const IS_FILE			= (1<<3);
 	protected const IS_FLO			= (1<<4);
 	protected const IS_REQ			= (1<<5);
+	protected const IS_TIC			= (1<<6);
 
 	protected const I_RECV			= (1<<6);
 	protected const I_SEND			= (1<<7);
@@ -38,7 +39,7 @@ class Item
 
 	/**
 	 * @throws FileNotFoundException
-	 * @throws UnreadableFileException
+	 * @throws UnreadableFileEncountered
 	 * @throws Exception
 	 */
 	public function __construct($file,int $action)
@@ -54,7 +55,7 @@ class Item
 					throw new FileNotFoundException('Item doesnt exist: '.$file);
 
 				if (! is_readable($file))
-					throw new UnreadableFileException('Item cannot be read: '.$file);
+					throw new UnreadableFileEncountered('Item cannot be read: '.$file);
 
 				$this->file_name = $file;
 				$x = stat($file);
@@ -130,6 +131,9 @@ class Item
 		if (strcasecmp(substr($x,1),'req') == 0)
 			return self::IS_REQ;
 
+		if (strcasecmp(substr($x,1),'tic') == 0)
+			return self::IS_TIC;
+
 		for ($i=0;$i<count($ext);$i++)
 			if (! strncasecmp($x,'.'.$ext[$i],strlen($ext[$i])) && (preg_match('/^[0-9a-z]/',strtolower(substr($x,3,1)))))
 				return self::IS_ARC;

app/Classes/File/Receive.php

@@ -10,7 +10,7 @@ use Symfony\Component\HttpFoundation\File\Exception\FileException;
 
 use App\Classes\FTN\InvalidPacketException;
 use App\Classes\FTN\Packet;
-use App\Jobs\MessageProcess;
+use App\Jobs\{MessageProcess,TicProcess};
 use App\Models\Address;
 
 /**
@@ -170,6 +170,14 @@ final class Receive extends Item
 
 					break;
 
+				case self::IS_TIC:
+					Log::info(sprintf('%s: - Processing tic file [%s]',self::LOGKEY,$this->file));
+
+					// Queue the tic to be processed later, in case the referenced file hasnt been received yet
+					TicProcess::dispatch($this->file);
+
+					break;
+
 				default:
 					Log::debug(sprintf('%s: - Leaving file [%s] in the inbound dir',self::LOGKEY,$this->file));
 			}

app/Classes/File/Send.php

@@ -6,7 +6,7 @@ use Exception;
 use Illuminate\Contracts\Filesystem\FileNotFoundException;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Log;
-use League\Flysystem\UnreadableFileException;
+use League\Flysystem\UnreadableFileEncountered;
 
 use App\Models\Address;
 
@@ -135,7 +135,7 @@ final class Send extends Item
 			Log::error(sprintf('%s:! Item [%s] doesnt exist',self::LOGKEY,$file));
 			return;
 
-		} catch (UnreadableFileException) {
+		} catch (UnreadableFileEncountered) {
 			Log::error(sprintf('%s:! Item [%s] cannot be read',self::LOGKEY,$file));
 			return;
 
@@ -257,7 +257,7 @@ final class Send extends Item
 	 *
 	 * @param int $length
 	 * @return string|null
-	 * @throws UnreadableFileException
+	 * @throws UnreadableFileEncountered
 	 * @throws Exception
 	 */
 	public function read(int $length): ?string
@@ -276,7 +276,7 @@ final class Send extends Item
 		Log::debug(sprintf('%s: - Read [%d] bytes, file pos now [%d]',self::LOGKEY,strlen($data),$this->file_pos));
 
 		if ($data === FALSE)
-			throw new UnreadableFileException('Error reading file: '.$this->sending->file_name);
+			throw new UnreadableFileEncountered('Error reading file: '.$this->sending->file_name);
 
 		return $data;
 	}

app/Classes/Protocol/Binkp.php

@@ -7,7 +7,7 @@ use Exception;
 use Illuminate\Support\Arr;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Log;
-use League\Flysystem\UnreadableFileException;
+use League\Flysystem\UnreadableFileEncountered;
 
 use App\Classes\Protocol as BaseProtocol;
 use App\Classes\Sock\SocketClient;
@@ -511,7 +511,7 @@ final class Binkp extends BaseProtocol
 				try {
 					$data = $this->send->read(self::BP_BLKSIZE);
 
-				} catch (UnreadableFileException) {
+				} catch (UnreadableFileEncountered) {
 					$this->send->close(FALSE);
 					$this->sessionClear(self::SE_SENDFILE);
 

app/Classes/Protocol/Zmodem.php

@@ -3,7 +3,6 @@
 namespace App\Classes\Protocol;
 
 use Illuminate\Support\Facades\Log;
-use League\Flysystem\UnreadableFileException;
 
 use App\Classes\Protocol;
 use App\Classes\Protocol\Zmodem as ZmodemClass;

app/Console/Commands/TicProcess.php

@@ -0,0 +1,37 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+
+use App\Jobs\TicProcess as Job;
+
+class TicProcess extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'tic:process {file : TIC file}';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Process a TIC file';
+
+    /**
+     * Execute the console command.
+     *
+     * @return int
+     */
+    public function handle()
+    {
+		// Dispatch job.
+		Job::dispatchSync($this->argument('file'));
+
+        return Command::SUCCESS;
+    }
+}

app/Http/Controllers/SystemController.php

@@ -12,7 +12,7 @@ use Illuminate\Support\Facades\Notification;
 use Illuminate\Support\ViewErrorBag;
 
 use App\Http\Requests\SystemRegister;
-use App\Models\{Address,Echoarea,Setup,System,SystemZone,Zone};
+use App\Models\{Address,Echoarea,Filearea,Setup,System,SystemZone,Zone};
 use App\Notifications\AddressLink;
 use App\Rules\{FidoInteger,TwoByteInteger};
 
@@ -410,6 +410,40 @@ class SystemController extends Controller
 			->with('echoareas',$eo);
 	}
 
+	/**
+	 * Update the systems fileareas
+	 *
+	 * @param Request $request
+	 * @param System $o
+	 * @return \Illuminate\Contracts\Foundation\Application|\Illuminate\Contracts\View\Factory|\Illuminate\Contracts\View\View|\Illuminate\Http\RedirectResponse
+	 */
+	public function fileareas(Request $request,System $o)
+	{
+		$ao = $o->addresses->firstWhere('id',$request->address_id);
+
+		if (($request->method() == 'POST') && $request->post()) {
+			session()->flash('accordion','filearea');
+
+			// Ensure we have session details for this address.
+			if (! $ao->session('sespass'))
+				return redirect()->back()->withErrors('System doesnt belong to this network');
+
+			$ao->fileareas()->syncWithPivotValues($request->get('id',[]),['subscribed'=>Carbon::now()]);
+
+			return redirect()->back()->with('success','Fileareas updated');;
+		}
+
+		$fo = Filearea::active()
+			->where('domain_id',$ao->zone->domain_id)
+			->orderBy('name')
+			->get();
+
+		return view('system.widget.filearea')
+			->with('o',$o)
+			->with('ao',$ao)
+			->with('fileareas',$fo);
+	}
+
 	public function home()
 	{
 		return view('system.home');

app/Jobs/TicProcess.php

@@ -0,0 +1,40 @@
+<?php
+
+namespace App\Jobs;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Filesystem\FileNotFoundException;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+use App\Classes\FTN\Tic;
+
+class TicProcess implements ShouldQueue
+{
+	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+	private const LOGKEY = 'JTP';
+
+	/**
+	 * Create a new job instance.
+	 *
+	 * @param string $file
+	 */
+	public function __construct(private string $file)
+	{
+	}
+
+	/**
+	 * Execute the job.
+	 *
+	 * @return void
+	 * @throws FileNotFoundException
+	 */
+	public function handle()
+	{
+		new Tic($this->file);
+	}
+}

app/Models/Address.php

@@ -159,7 +159,29 @@ class Address extends Model
 	public function echomails()
 	{
 		return $this->belongsToMany(Echomail::class,'echomail_seenby')
-			->withPivot(['sent_at','packet']);
+			->withPivot(['sent_at','export_at','packet']);
+	}
+
+	/**
+	 * Files that this address has seen
+	 *
+	 * @return \Illuminate\Database\Eloquent\Relations\BelongsToMany
+	 */
+	public function files()
+	{
+		return $this->belongsToMany(File::class,'file_seenby')
+			->withPivot(['sent_at','export_at']);
+	}
+
+	/**
+	 * Echoareas this address is subscribed to
+	 *
+	 * @return \Illuminate\Database\Eloquent\Relations\BelongsToMany
+	 */
+	public function fileareas()
+	{
+		return $this->belongsToMany(Filearea::class)
+			->withPivot(['subscribed']);
 	}
 
 	/**
@@ -409,7 +431,7 @@ class Address extends Model
 	}
 
 	/**
-	 * Netmail waiting to be sent to this system
+	 * Echomail waiting to be sent to this system
 	 *
 	 * @return Collection
 	 */
@@ -421,6 +443,19 @@ class Address extends Model
 			->get();
 	}
 
+	/**
+	 * Files waiting to be sent to this system
+	 *
+	 * @return Collection
+	 */
+	public function filesWaiting(): Collection
+	{
+		return $this->files()
+			->whereNull('file_seenby.sent_at')
+			->whereNotNull('file_seenby.export_at')
+			->get();
+	}
+
 	/**
 	 * Get echomail for this node
 	 *

app/Models/Echomail.php

@@ -46,7 +46,7 @@ final class Echomail extends Model implements Packet
 
 	protected $dates = ['datetime'];
 
-	public function __set($key, $value)
+	public function __set($key,$value)
 	{
 		switch ($key) {
 			case 'no_export':
@@ -68,7 +68,7 @@ final class Echomail extends Model implements Packet
 		// @todo if the message is updated with new SEEN-BY's from another route, we'll delete the pending export for systems (if there is one)
 		static::created(function($model) {
 			if (! $model->echoarea_id) {
-				Log::alert(sprintf('%s:- Message has no echo area, not exporting',self::LOGKEY,$model->id));
+				Log::alert(sprintf('%s:- Message has no echoarea, not exporting',self::LOGKEY,$model->id));
 				return;
 			}
 

app/Models/File.php

@@ -0,0 +1,157 @@
+<?php
+
+namespace App\Models;
+
+use Carbon\Carbon;
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\SoftDeletes;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
+use Rennokki\QueryCache\Traits\QueryCacheable;
+
+use App\Casts\{CollectionOrNull,CompressedString};
+use App\Traits\EncodeUTF8;
+
+class File extends Model
+{
+	use SoftDeletes,EncodeUTF8,QueryCacheable;
+
+	private const LOGKEY = 'MF-';
+	private bool $no_export = FALSE;
+
+	protected $casts = [
+		'kludges' => CollectionOrNull::class,
+		'rogue_seenby' => CollectionOrNull::class,
+		'rogue_path' => CollectionOrNull::class,
+		'desc' => CompressedString::class,
+		'ldesc' => CompressedString::class,
+		'size' => 'int',
+	];
+
+	private const cast_utf8 = [
+		'desc',
+		'ldesc',
+	];
+
+	protected $dates = ['datetime'];
+
+	public function __set($key,$value)
+	{
+		switch ($key) {
+			case 'fullname':
+			case 'replaces':
+			case 'no_export':
+			case 'set_path':
+			case 'set_packet':
+			case 'set_seenby':
+				$this->{$key} = $value;
+				break;
+
+			default:
+				parent::__set($key,$value);
+		}
+	}
+
+	public static function boot()
+	{
+		parent::boot();
+
+		static::creating(function($model) {
+			Log::debug(sprintf('%s:- Storing file [%s]',self::LOGKEY,$model->file));
+
+			// Store file
+
+			// Delete file from inbound
+
+			// Delete anything being replaced
+		});
+
+		// @todo if the file is updated with new SEEN-BY's from another route, we'll delete the pending export for systems (if there is one)
+		static::created(function($model) {
+			if (! $model->filearea_id) {
+				Log::alert(sprintf('%s:- File has no filearea, not exporting',self::LOGKEY,$model->id));
+				return;
+			}
+
+			$so = Setup::findOrFail(config('app.id'));
+
+			// Our address
+			$ftns = $so
+				->system
+				->match($model->fftn->zone,Address::NODE_ACTIVE|Address::NODE_PVT|Address::NODE_HOLD);
+
+			// Add our address to the seenby;
+			$model->set_seenby = $model->set_seenby->merge($ftns->pluck('id'))->unique();
+			$model->set_path = $model->set_path->merge([[
+				'address'=>$ftns->first(),
+				'datetime'=>($x=Carbon::now())->timestamp,
+				'extra'=>sprintf('%s %s (%s)',$x->toRfc7231String(),$so::PRODUCT_NAME,$so->version),
+			]]);
+
+			// Save the seenby
+			$model->seenby()->sync($model->set_seenby);
+
+			// Save the Path
+			$ppoid = NULL;
+			foreach ($model->set_path as $path) {
+				$po = DB::select('INSERT INTO file_path (file_id,address_id,parent_id,datetime,extra) VALUES (?,?,?,?,?) RETURNING id',[
+					$model->id,
+					$path['address']->id,
+					$ppoid,
+					Carbon::createFromTimestamp($path['datetime']),
+					$path['extra'],
+				]);
+
+				$ppoid = $po[0]->id;
+			}
+
+			// See if we need to export this message.
+			$exportto = $model->filearea->addresses->pluck('id')->diff($model->set_seenby);
+
+			if ($exportto->count()) {
+				if ($model->no_export) {
+					Log::debug(sprintf('%s:- NOT processing exporting of message by configuration [%s] to [%s]',self::LOGKEY,$model->id,$exportto->join(',')));
+					return;
+				}
+
+				Log::debug(sprintf('%s:- Exporting file [%s] to [%s]',self::LOGKEY,$model->id,$exportto->join(',')));
+
+				// Save the seenby for the exported systems
+				$model->seenby()->syncWithPivotValues($exportto,['export_at'=>Carbon::now()],FALSE);
+			}
+		});
+	}
+
+	/* RELATIONS */
+
+	public function filearea()
+	{
+		return $this->belongsTo(Filearea::class);
+	}
+
+	public function fftn()
+	{
+		return $this->belongsTo(Address::class)
+			->withTrashed();
+	}
+
+	public function seenby()
+	{
+		return $this->belongsToMany(Address::class,'file_seenby')
+			->ftnOrder();
+	}
+
+	public function path()
+	{
+		return $this->belongsToMany(Address::class,'file_path')
+			->withPivot(['id','parent_id','extra']);
+	}
+
+	/* METHODS */
+
+	public function jsonSerialize(): array
+	{
+		return $this->encode();
+	}
+}

app/Models/Filearea.php

@@ -11,6 +11,10 @@ class Filearea extends Model
 {
 	use SoftDeletes,ScopeActive;
 
+	protected $fillable = [
+		'name',
+	];
+
 	/* RELATIONS */
 
 	public function addresses()

app/Models/System.php

@@ -92,6 +92,14 @@ class System extends Model
 			->where('addresses.system_id',$this->id);
 	}
 
+	public function fileareas()
+	{
+		return Filearea::select('fileareas.*')
+			->join('address_filearea',['address_filearea.filearea_id'=>'fileareas.id'])
+			->join('addresses',['addresses.id'=>'address_filearea.address_id'])
+			->where('addresses.system_id',$this->id);
+	}
+
 	/**
 	 * Return the system name, or role name for the zone
 	 *