Rework TIC processing to use Storage::disk(). Implemented handling of replaces and files that already exist

app/Classes/FTN/Tic.php

@@ -4,10 +4,11 @@ namespace App\Classes\FTN;
 
 use Carbon\Carbon;
 use Illuminate\Contracts\Filesystem\FileNotFoundException;
+use Illuminate\Support\Arr;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\Storage;
-use League\Flysystem\UnableToWriteFile;
+use League\Flysystem\UnableToReadFile;
 
 use App\Classes\FTN as FTNBase;
 use App\Models\{Address,File,Filearea,Setup,System};
@@ -104,6 +105,8 @@ class Tic extends FTNBase
 		$result->put('SIZE',$fo->size);
 		if ($fo->description)
 			$result->put('DESC',$fo->description);
+		if ($fo->replaces)
+			$result->put('REPLACES',$fo->replaces);
 		$result->put('AREA',$fo->filearea->name);
 		$result->put('AREADESC',$fo->filearea->description);
 		if ($x=$ao->session('ticpass'))
@@ -117,9 +120,14 @@ class Tic extends FTNBase
 		foreach ($fo->path as $o)
 			$out .= sprintf("PATH %s %s %s\r\n",$o->ftn3d,$o->pivot->datetime,$o->pivot->extra);
 
+		// Add ourself to the path:
+		$out .= sprintf("PATH %s %s\r\n",$sysaddress->ftn3d,Carbon::now());
+
 		foreach ($fo->seenby as $o)
 			$out .= sprintf("SEENBY %s\r\n",$o->ftn3d);
 
+		$out .= sprintf("SEENBY %s\r\n",$sysaddress->ftn3d);
+
 		return $out;
 	}
 
@@ -137,33 +145,38 @@ class Tic extends FTNBase
 	/**
 	 * Load a TIC file from an existing filename
 	 *
-	 * @param string $filename
+	 * @param string $sp
+	 * @param string $filename Relative to filesystem
 	 * @return void
 	 * @throws FileNotFoundException
 	 */
 	public function load(string $filename): void
 	{
 		Log::info(sprintf('%s:+ Processing TIC file [%s]',self::LOGKEY,$filename));
+		$fs = Storage::disk(config('fido.local_disk'));
 
 		if (str_contains($filename,'-')) {
 			list($hex,$name) = explode('-',$filename);
 			$hex = basename($hex);
+
 		} else {
 			$hex = '';
 		}
 
-		if (! file_exists($filename))
+		if (! $fs->exists($filename))
 			throw new FileNotFoundException(sprintf('File [%s] doesnt exist',$filename));
 
-		if (! is_readable($filename))
+		if (! is_readable($fs->path($filename)))
-			throw new UnableToWriteFile(sprintf('File [%s] is not readable',realpath($filename)));
+			throw new UnableToReadFile(sprintf('File [%s] is not readable',realpath($filename)));
 
-		$f = fopen($filename,'rb');
+		$f = $fs->readStream($filename);
 		if (! $f) {
-			Log::error(sprintf('%s:! Unable to open file [%s] for writing',self::LOGKEY,$filename));
+			Log::error(sprintf('%s:! Unable to open file [%s] for reading',self::LOGKEY,$filename));
 			return;
 		}
 
+		$ldesc = '';
+
 		while (! feof($f)) {
 			$line = chop(fgets($f));
 			$matches = [];
@@ -171,12 +184,13 @@ class Tic extends FTNBase
 			if (! $line)
 				continue;
 
-			preg_match('/([a-zA-Z]+)\ (.*)/',$line,$matches);
+			preg_match('/([a-zA-Z]+)\ ?(.*)?/',$line,$matches);
 
-			if (in_array(strtolower($matches[1]),$this->_kludge)) {
+			if (in_array(strtolower(Arr::get($matches,1,'-')),$this->_kludge)) {
 				switch ($k=strtolower($matches[1])) {
 					case 'area':
 						$this->{$k} = Filearea::singleOrNew(['name'=>strtoupper($matches[2])]);
+
 						break;
 
 					case 'origin':
@@ -184,19 +198,26 @@ class Tic extends FTNBase
 					case 'to':
 						$this->{$k} = Address::findFTN($matches[2]);
 
-						// @todo If $this->{$k} is null, we have discovered the system and it should be created
+						if (! $this->{$k})
+							Log::alert(sprintf('%s:! Unable to find an FTN for [%s] for the (%s)',self::LOGKEY,$matches[2],$k));
+
 						break;
 
 					case 'file':
-						if (! Storage::disk('local')->exists($x=sprintf('%s/%s-%s',config('app.fido'),$hex,$matches[2])))
-							throw new FileNotFoundException(sprintf('File not found? [%s]',$x));
-
 						$this->fo->name = $matches[2];
-						$this->fo->fullname = $x;
+						$this->fo->prefix = $hex;
+
+						if (! $fs->exists($this->fo->recvd_rel_name)) {
+							// @todo Fail this, so that it is rescheduled to try again in 1-24hrs.
+
+							throw new FileNotFoundException(sprintf('File not found? [%s]',$fs->path($this->fo->recvd_rel_name)));
+						}
+
 						break;
 
 					case 'areadesc':
 						$areadesc = $matches[2];
+
 						break;
 
 					case 'created':
@@ -205,10 +226,12 @@ class Tic extends FTNBase
 
 					case 'pw':
 						$pw = $matches[2];
+
 						break;
 
 					case 'lfile':
 						$this->fo->lname = $matches[2];
+
 						break;
 
 					case 'desc':
@@ -216,46 +239,36 @@ class Tic extends FTNBase
 					case 'replaces':
 					case 'size':
 						$this->fo->{$k} = $matches[2];
+
 						break;
 
 					case 'fullname':
 						$this->fo->lfile = $matches[2];
+
 						break;
 
 					case 'date':
-						$this->fo->datetime = Carbon::create($matches[2]);
+						$this->fo->datetime = Carbon::createFromTimestamp($matches[2]);
+
 						break;
 
 					case 'ldesc':
-						$this->fo->{$k} .= $matches[2];
+						$ldesc .= ($ldesc ? "\r" : '').$matches[2];
+
 						break;
 
 					case 'crc':
 						$this->fo->{$k} = hexdec($matches[2]);
+
 						break;
 
 					case 'path':
-						$x = [];
+						$this->fo->set_path->push($matches[2]);
-						preg_match(sprintf('#^[Pp]ath (%s)\ ?([0-9]+)\ ?(.*)$#',Address::ftn_regex),$line,$x);
-						$ao = Address::findFTN($x[1]);
-
-						if (! $ao)
-							$ao = Address::createFTN($x[1],System::createUnknownSystem());
-
-						$this->fo->set_path->push(['address'=>$ao,'datetime'=>Carbon::createFromTimestamp($x[8]),'extra'=>$x[9]]);
 
 						break;
 
 					case 'seenby':
-						$ao = Address::findFTN($matches[2]);
+						$this->fo->set_seenby->push($matches[2]);
-
-						if (! $ao)
-							$ao = Address::createFTN($x[1],System::createUnknownSystem());
-
-						if (! $ao)
-							$this->fo->rogue_seenby->push($matches[2]);
-						else
-							$this->fo->set_seenby->push($ao->id);
 
 						break;
 				}
@@ -265,19 +278,24 @@ class Tic extends FTNBase
 			}
 		}
 
-		fclose($f);
+		if ($ldesc)
+			$this->fo->ldesc = $ldesc;
 
-		$f = fopen($x=Storage::disk('local')->path($this->fo->fullname),'rb');
-		$stat = fstat($f);
 		fclose($f);
 
 		// @todo Add notifictions back to the system
+		if ($this->fo->replaces && (! preg_match('/^'.$this->fo->replaces.'$/',$this->fo->name))) {
+			Log::alert(sprintf('%s:! Regex [%s] doesnt match file name [%s]',self::LOGKEY,$this->fo->replaces,$this->fo->name));
+
+			$this->fo->replaces = NULL;
+		}
+
 		// Validate Size
-		if ($this->fo->size !== ($y=$stat['size']))
+		if ($this->fo->size !== ($y=$fs->size($this->fo->recvd_rel_name)))
 			throw new \Exception(sprintf('TIC file size [%d] doesnt match file [%s] (%d)',$this->fo->size,$this->fo->fullname,$y));
 
 		// Validate CRC
-		if (sprintf('%08x',$this->fo->crc) !== ($y=hash_file('crc32b',$x)))
+		if (sprintf('%08x',$this->fo->crc) !== ($y=$fs->checksum($this->fo->recvd_rel_name,['checksum_algo'=>'crc32b'])))
 			throw new \Exception(sprintf('TIC file CRC [%08x] doesnt match file [%s] (%s)',$this->fo->crc,$this->fo->fullname,$y));
 
 		// Validate Password
@@ -303,7 +321,7 @@ class Tic extends FTNBase
 
 		// If the file create time is blank, we'll take the files
 		if (! $this->fo->datetime)
-			$this->fo->datetime = Carbon::createFromTimestamp($stat['ctime']);
+			$this->fo->datetime = Carbon::createFromTimestamp($fs->lastModified($this->fo->recvd_rel_name));
 
 		$this->fo->save();
 	}

app/Classes/File/File.php

@@ -86,7 +86,7 @@ final class File extends Send
 	{
 		// If sending file is a File::class, then our file is s3
 		if ($this->nameas && $this->f instanceof FileModel) {
-			$this->fd = Storage::readStream($this->f->full_storage_path);
+			$this->fd = Storage::readStream($this->f->rel_name);
 
 		} else {
 			$this->fd = fopen($this->full_name,'rb');

app/Classes/File/Item.php

@@ -2,19 +2,19 @@
 
 namespace App\Classes\File;
 
+use Illuminate\Contracts\Filesystem\Filesystem;
 use Illuminate\Support\Facades\Storage;
 
 use App\Models\Address;
 
 final class Item extends Receive
 {
-	private const LOCATION = 'local';
-
 	/** @var Address The address that sent us this item */
 	private Address $ao;
 	private string $recvas;
 	private int $recvmtime;
 	private int $recvsize;
+	private Filesystem $fs;
 
 	/**
 	 * @throws \Exception
@@ -29,19 +29,20 @@ final class Item extends Receive
 		$this->recvsize = $size;
 
 		$this->ftype = self::IS_FILE;
+		$this->fs = Storage::disk(config('fido.local_disk'));
 	}
 
 	public function __get($key) {
 		switch ($key) {
 			case 'exists':
-				return Storage::disk(self::LOCATION)->exists($this->rel_name);
+				return $this->fs->exists($this->rel_name);
 
-			case 'stor_name':
+			case 'pref_name':
 				return sprintf('%04X-%s',$this->ao->id,$this->recvas);
 			case 'rel_name':
-				return sprintf('%s/%s',config('fido.dir'),$this->stor_name);
+				return sprintf('%s/%s',config('fido.dir'),$this->pref_name);
 			case 'full_name':
-				return Storage::disk(self::LOCATION)->path($this->rel_name);
+				return $this->fs->path($this->rel_name);
 
 			case 'match_mtime':
 				return $this->mtime === $this->recvmtime;
@@ -59,10 +60,10 @@ final class Item extends Receive
 				return sprintf('%s %lu %lu',$this->recvas,$this->recvsize,$this->recvmtime);
 
 			case 'mtime':
-				return Storage::disk(self::LOCATION)->lastModified($this->rel_name);
+				return $this->fs->lastModified($this->rel_name);
 
 			case 'size':
-				return Storage::disk(self::LOCATION)->size($this->rel_name);
+				return $this->fs->size($this->rel_name);
 
 			default:
 				return parent::__get($key);

app/Classes/File/Receive.php

@@ -60,7 +60,7 @@ class Receive extends Base
 			case 'nameas':
 			case 'size':
 			case 'name_size_time':
-			case 'stor_name':
+			case 'pref_name':
 				return $this->receiving->{$key};
 
 			case 'pos':

app/Jobs/PacketProcess.php

@@ -137,7 +137,7 @@ class PacketProcess implements ShouldQueue
 
 					try {
 						if ($fs->makeDirectory($dir)) {
-							$fs->move($this->file->rel_name,$x=sprintf('%s/%s',$dir,$this->file->stor_name));
+							$fs->move($this->file->rel_name,$x=sprintf('%s/%s',$dir,$this->file->pref_name));
 							Log::info(sprintf('%s:- Moved processed packet [%s] to [%s]',self::LOGKEY,$this->file->rel_name,$x));
 
 						} else

app/Jobs/TicProcess.php

@@ -10,6 +10,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Storage;
 
 use App\Classes\FTN\Tic;
 use App\Models\Domain;
@@ -27,7 +28,7 @@ class TicProcess implements ShouldQueue
 	/**
 	 * Create a new job instance.
 	 *
-	 * @param string $file
+	 * @param string $file Relative to Storage::disk('local'), ie: storage/app
 	 * @param string|null $domain
 	 */
 	public function __construct(private string $file,private ?string $domain=NULL)
@@ -56,12 +57,15 @@ class TicProcess implements ShouldQueue
 	 */
 	public function handle()
 	{
+		$fs = Storage::disk(config('fido.local_disk'));
+
 		$to = new Tic;
-		$to->load(storage_path('app').'/'.$this->file);
+		$to->load($this->file);
 
 		Log::info(sprintf('%s:= Processed [%s] storing [%s] as id [%d]',self::LOGKEY,$this->file,$to->fo->name,$to->fo->id));
 
-		unlink(storage_path('app').'/'.$this->file);
+		if (! $fs->delete($this->file))
+			Log::alert(sprintf('%s:! Failed to delete [%s]',self::LOGKEY,$this->file));
 
 		if ($to->isNodelist())
 			NodelistImport::dispatch($to->fo,$this->domain);

app/Models/File.php

@@ -5,6 +5,8 @@ namespace App\Models;
 use Carbon\Carbon;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Database\Eloquent\SoftDeletes;
+use Illuminate\Support\Arr;
+use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\Storage;
@@ -20,7 +22,8 @@ class File extends Model
 	private const LOGKEY = 'MF-';
 	private bool $no_export = FALSE;
 	public string $prefix = '';
-	public string $replaces = '';
+	public Collection $set_path;
+	public Collection $set_seenby;
 
 	protected $casts = [
 		'kludges' => CollectionOrNull::class,
@@ -37,82 +40,108 @@ class File extends Model
 		'ldesc',
 	];
 
-	public function __set($key,$value)
-	{
-		switch ($key) {
-			case 'fullname':
-			case 'replaces':
-			case 'no_export':
-			case 'set_path':
-			case 'set_seenby':
-				$this->{$key} = $value;
-				break;
-
-			default:
-				parent::__set($key,$value);
-		}
-	}
-
 	public static function boot()
 	{
 		parent::boot();
 
 		static::creating(function($model) {
-			Log::info(sprintf('%s:- Storing file [%s] in [%s]',self::LOGKEY,$model->fullname,$model->full_storage_path));
+			if (! $model->filearea_id) {
-
+				Log::alert(sprintf('%s:- File has no filearea, not processing creating [%s]',self::LOGKEY,$model->name));
-			// Store file
+				return;
-			if (Storage::put($model->full_storage_path,Storage::disk('local')->get($model->fullname),'public')) {
-				unlink(Storage::disk('local')->path($model->fullname));
-			} else {
-				throw new \Exception(sprintf('Unable to move file [%s] to [%s]',$model->fullname,$model->full_storage_path));
 			}
 
+			Log::info(sprintf('%s:- Storing file [%s] in [%s]',self::LOGKEY,$model->recvd_rel_name,$model->rel_name));
+
+			$srcfs = Storage::disk(config('fido.local_disk'));
+			$tgtfs = Storage::disk(config('fido.file_disk'));
+
 			// Delete anything being replaced
-			// @todo implement replace
+			foreach (self::where('name',$model->name)->where('filearea_id',$model->filearea_id)->get() as $fo) {
+				Log::info(sprintf('%s:%% Deleting old file record [%d] for file [%s]',self::LOGKEY,$fo->id,$fo->rel_name));
+
+				$tgtfs->move($fo->rel_name,$fo->relname.'.'.$fo->id);
+				$fo->delete();
+			}
+
+			// Store file
+			if ($tgtfs->put($model->rel_name,$srcfs->get($model->recvd_rel_name),'public')) {
+				$srcfs->delete($model->recvd_rel_name);
+
+			} else {
+				throw new \Exception(sprintf('Unable to move file [%s] to [%s]',$model->recvd_rel_name,$model->rel_name));
+			}
 		});
 
 		// @todo if the file is updated with new SEEN-BY's from another route, we'll delete the pending export for systems (if there is one)
 		static::created(function($model) {
 			if (! $model->filearea_id) {
-				Log::alert(sprintf('%s:- File has no filearea, not exporting',self::LOGKEY,$model->id));
+				Log::alert(sprintf('%s:- File has no filearea, not exporting [%d]',self::LOGKEY,$model->id));
 				return;
 			}
 
+			$rogue = collect();
+			$seenby = collect();
+			$path = collect();
+
 			$so = Setup::findOrFail(config('app.id'));
+			$zone = $model->fftn->zone;
 
-			// Our address
+			// Parse PATH
-			$ftns = $so
+			foreach ($model->set_path as $line) {
-				->system
+				$matches = [];
-				->match($model->fftn->zone);
+				preg_match(sprintf('#^(%s)\ ?([0-9]+)\ ?(.*)$#',Address::ftn_regex),$line,$matches);
 
-			// Add our address to the seenby;
+				if ($x=Arr::get($matches,1)) {
-			$model->set_seenby = $model->set_seenby->merge($ftns->pluck('id'))->unique();
+					$ftn = Address::parseFTN($x);
-			$model->set_path = $model->set_path->merge([[
-				'address'=>$ftns->first(),
-				'datetime'=>$x=Carbon::now(),
-				'extra'=>sprintf('%s %s (%s)',$x->toRfc7231String(),$so::PRODUCT_NAME,$so->version),
-			]]);
 
-			// Make sure all the path is in the seenby
+					// If domain should be flattened, look for node regardless of zone (within the list of zones for the domain)
-			$model->set_seenby = $model->set_seenby->merge($model->set_path->pluck('address.id'))->unique()->filter();
+					$ao = ($zone->domain->flatten)
+						? Address::findZone($zone->domain,$ftn['n'],$ftn['f'],0)
+						: Address::findFTN($x);
 
-			// Save the seenby
+					if (! $ao)
-			$model->seenby()->sync($model->set_seenby);
+						$ao = Address::createFTN($x,System::createUnknownSystem());
+
+					$path->push(['address'=>$ao,'datetime'=>Carbon::createFromTimestamp($matches[9]),'extra'=>$matches[10]]);
+				}
+			}
 
 			// Save the Path
 			$ppoid = NULL;
-			foreach ($model->set_path as $path) {
+			foreach ($path as $item) {
 				$po = DB::select('INSERT INTO file_path (file_id,address_id,parent_id,datetime,extra) VALUES (?,?,?,?,?) RETURNING id',[
 					$model->id,
-					$path['address']->id,
+					$item['address']->id,
 					$ppoid,
-					Carbon::createFromTimestamp($path['datetime']),
+					$item['datetime'],
-					$path['extra'],
+					$item['extra'],
 				]);
 
 				$ppoid = $po[0]->id;
 			}
 
+			// Make sure all the path is in the seenby
+			// Add zone to seenby
+			$model->set_seenby = $model->set_seenby->merge($path->pluck('address.ftn3d'))->unique()->filter();
+
+			foreach ($model->set_seenby as $sb) {
+				$ftn = Address::parseFTN($x);
+
+				$ao = ($zone->domain->flatten)
+					? Address::findZone($zone->domain,$ftn['n'],$ftn['f'],0)
+					: Address::findFTN($x);
+
+				if ($ao)
+					$seenby->push($ao->id);
+				else
+					$rogue->push($sb);
+			}
+
+			$model->rogue_seenby = $rogue;
+
+			$model->seenby()->sync($seenby);
+			$model->save();
+
 			// See if we need to export this file.
 			if ($model->filearea->sec_read) {
 				$exportto = $model
@@ -169,7 +198,7 @@ class File extends Model
 	 *
 	 * @return string
 	 */
-	public function getFullStoragePathAttribute(): string
+	public function getRelNameAttribute(): string
 	{
 		return sprintf('%04X/%s',$this->filearea_id,$this->name);
 	}
@@ -179,12 +208,17 @@ class File extends Model
 	 *
 	 * @return string
 	 */
-	public function getRelNameAttribute(): string
+	public function getRecvdRelNameAttribute(): string
 	{
-		return sprintf('%s/%s',config('fido.dir'),$this->prefix_name);
+		return sprintf('%s/%s',config('fido.dir'),$this->recvd_pref_name);
 	}
 
-	public function getPrefixNameAttribute(): string
+	/**
+	 * This is the name of the file, with the sender prefix
+	 *
+	 * @return string
+	 */
+	public function getRecvdPrefNameAttribute(): string
 	{
 		return sprintf('%s%s',$this->prefix ? $this->prefix.'-' : '',$this->name);
 	}

app/Traits/Import.php

@@ -86,12 +86,12 @@ trait Import
 		if ($file instanceof File) {
 			$path = sprintf('import/%s.%d',$key,$file->id);
 
-			Storage::disk('local')->put($path,Storage::get($file->full_storage_path));
+			Storage::disk(config('fido.local_disk'))->put($path,Storage::get($file->rel_name));
 
-			return Storage::disk('local')->path($path);
+			return Storage::disk(config('fido.local_disk'))->path($path);
 
 		} else {
-			return Storage::disk('local')->path($file);
+			return Storage::disk(config('fido.local_disk'))->path($file);
 		}
 	}
 }

config/fido.php

@@ -10,6 +10,9 @@ return [
 	// Directory to use for any data we want to store locally
 	'dir' => env('FIDO_DIR', 'fido'),
 
+	// Our Storage::disk() for files storage
+	'file_disk' => env('FIDO_DIR_FILES', 's3'),
+
 	// Our Storage::disk() for locally stored data
 	'local_disk' => env('FIDO_LOCAL_DIR','local'),
 

database/migrations/2023_09_22_233947_file_tweeks.php

@@ -0,0 +1,34 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+		Schema::table('files', function (Blueprint $table) {
+			$table->dropUnique(['filearea_id','name']);
+			$table->string('replaces')->nullable();
+		});
+
+		DB::statement("CREATE UNIQUE INDEX files_active ON files (filearea_id, name) WHERE deleted_at IS NULL");
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+		DB::statement("DROP INDEX files_active");
+
+		Schema::table('files', function (Blueprint $table) {
+			$table->unique(['filearea_id','name']);
+			$table->dropColumn('replaces');
+		});
+    }
+};