Switchout DB to CockroachDB

.env.example

@@ -7,12 +7,16 @@ APP_DEBUG=true
 LOG_CHANNEL=stack
 LOG_LEVEL=info
 
-DB_CONNECTION=pgsql
+DB_CONNECTION=cockroach
-DB_HOST=postgres
+DB_HOST=cockroach
-DB_PORT=5432
+DB_PORT=26257
-DB_DATABASE=postgres
+DB_DATABASE=clrghouz
-DB_USERNAME=postgres
+DB_USERNAME=dev
-DB_PASSWORD=password
+DB_PASSWORD=
+DB_SSLMODE=prefer
+DB_SSLROOTCERT=/var/www/html/config/ssl/ca.crt
+DB_SSLCERT=/var/www/html/config/ssl/client.crt
+DB_SSLKEY=/var/www/html/config/ssl/client.key
 
 DB_MONGO_HOST=mongo
 DB_MONGO_USERNAME=mongo

app/Classes/FTN/Message.php

@@ -93,7 +93,7 @@ class Message extends FTNBase
 	private string $subject;					// Message subject
 
 	private string $msgid;						// MSG ID
-	private string $reply;						// Reply ID
+	private string $replyid;						// Reply ID
 
 	private string $echoarea;					// FTS-0004.001
 	private string $intl;						// Netmail details
@@ -182,7 +182,7 @@ class Message extends FTNBase
 		$this->message = '';
 
 		$this->msgid = '';
-		$this->reply = '';
+		$this->replyid = '';
 
 		$this->echoarea = '';
 		$this->intl = '';
@@ -372,7 +372,7 @@ class Message extends FTNBase
 			case 'echoarea':
 
 			case 'msgid':
-			case 'reply':
+			case 'replyid':
 
 			case 'message':
 			case 'message_src':
@@ -412,7 +412,7 @@ class Message extends FTNBase
 			case 'subject':
 
 			case 'msgid':
-			case 'reply':
+			case 'replyid':
 
 			case 'echoarea':
 			case 'intl':
@@ -486,8 +486,8 @@ class Message extends FTNBase
 			// Add some kludges
 			$return .= sprintf("\01MSGID: %s\r",$this->msgid);
 
-			if ($this->reply)
+			if ($this->replyid)
-				$return .= sprintf("\01REPLY: %s\r",$this->reply);
+				$return .= sprintf("\01REPLY: %s\r",$this->replyid);
 
 			foreach ($this->_kludge as $k=>$v) {
 				if ($x=$this->kludge->get($k))

app/Classes/FTN/Process/Echomail/Test.php

@@ -26,6 +26,7 @@ final class Test extends Process
 
 	private const testing = ['test','testing'];
 
+	// @todo add path and other kludges
 	public static function handle(Message $msg): bool
 	{
 		if ((strtolower($msg->user_to) !== 'all') || ! in_array(strtolower($msg->subject),self::testing))
@@ -66,7 +67,7 @@ final class Test extends Process
 		$o->datetime = Carbon::now();
 		$o->tzoffset = $o->datetime->utcOffset();
 		$o->echoarea_id = $eo?->id;
-		$o->reply = $msg->msgid;
+		$o->replyid = $msg->msgid;
 		$o->fftn_id = $ftns->id;
 
 		$o->flags = Message::FLAG_LOCAL;

app/Classes/FTN/Process/Netmail/Ping.php

@@ -50,7 +50,7 @@ final class Ping extends Process
 		$o->datetime = Carbon::now();
 		$o->tzoffset = $o->datetime->utcOffset();
 
-		$o->reply = $msg->msgid;
+		$o->replyid = $msg->msgid;
 		$o->fftn_id = $ftns->id;
 		$o->tftn_id = ($x=$msg->fftn_o) ? $x->id : NULL;
 		$o->flags = Message::FLAG_LOCAL;

app/Console/Commands/ConvertMongo.php

@@ -0,0 +1,160 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Echoarea;
+use App\Models\Echomail;
+use App\Models\Netmail;
+use Carbon\Carbon;
+use Illuminate\Console\Command;
+use Illuminate\Support\Arr;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Notification;
+
+class ConvertMongo extends Command
+{
+	/**
+	 * The name and signature of the console command.
+	 *
+	 * @var string
+	 */
+	protected $signature = 'convert:mongo {start=0}';
+
+	/**
+	 * The console command description.
+	 *
+	 * @var string
+	 */
+	protected $description = 'Convert Mongo to Cockroach';
+
+	/**
+	 * Execute the console command.
+	 *
+	 * @return mixed
+	 * @throws \Exception
+	 */
+	public function handle()
+	{
+		/*
+		// Netmails
+		foreach (DB::connection('mongodb')->collection('netmails')->cursor() as $o) {
+			$o['mid'] = (string)$o['_id'];
+
+			foreach([
+				'reply'=>'replyid',
+				'packet'=>'sent_pkt',
+			] as $key => $newkey) {
+				if (array_key_exists($key,$o)) {
+					$o[$newkey] = $o[$key];
+					unset($o[$key]);
+				}
+			}
+
+			unset($o['_id'],$o['sent'],$o['reply']);
+
+			foreach (['created_at','updated_at','sent_at','datetime','deleted_at'] as $key) {
+				if (Arr::get($o,$key))
+					$o[$key] = Carbon::create($o[$key]->toDatetime());
+			}
+
+			if (! Arr::get($o,'datetime'))
+				$o['datetime'] = $o['created_at'];
+
+			$oo = Netmail::withTrashed()->where('mid',$o['mid'])->firstOrNew();
+			$oo->forceFill($o);
+
+			try {
+				$oo->save(['timestamps' => false]);
+			} catch (\Exception $e) {
+				$this->warn(sprintf('Netmail didnt move over: %s (%s)',$o['mid'],$e->getMessage()));
+			}
+		}
+		*/
+
+		/**/
+		// Echomail
+		$c = 0;
+		foreach (DB::connection('mongodb')->collection('echomails')->cursor() as $o) {
+			if (++$c < $this->argument('start'))
+				continue;
+
+			if (! ($c%100))
+				Log::debug(sprintf('Processed : %d Echomails',$c));
+
+			$o['mid'] = (string)$o['_id'];
+
+			foreach([
+				'reply'=>'replyid',
+				'rogue_seen'=>'rogue_seenby',
+			] as $key => $newkey) {
+				if (array_key_exists($key,$o)) {
+					$o[$newkey] = $o[$key];
+					unset($o[$key]);
+				}
+			}
+
+			$path = (array_key_exists('path',$o) && $o['path']) ? $o['path'] : NULL;
+			$seenby = (array_key_exists('seenby',$o) && $o['seenby']) ? $o['seenby'] : NULL;
+			$packet = (array_key_exists('packet',$o) && $o['packet']) ? $o['packet'] : NULL;
+
+			unset($o['_id'],$o['reply'],$o['path'],$o['seenby'],$o['toexport'],$o['sent_at'],$o['packet'],$o['sent']);
+
+			foreach (['created_at','updated_at','datetime','deleted_at'] as $key) {
+				if (Arr::get($o,$key))
+					$o[$key] = Carbon::create($o[$key]->toDatetime());
+			}
+
+			if (! Arr::get($o,'datetime'))
+				$o['datetime'] = $o['created_at'];
+
+			if (Arr::get($o,'echoarea') && ! Arr::get($o,'echoarea_id')) {
+				$ea = Echoarea::where('name',$o['echoarea'])->single();
+				$o['echoarea_id'] = $ea->id;
+				unset($o['echoarea']);
+			}
+
+			if (! $o['echoarea_id']) {
+				Log::error(sprintf('Echomail didnt move over: %s [%d] - has no echoarea_id',$o['mid'],$c));
+				continue;
+			}
+
+			if (Arr::get($o,'msg_src') && ! Arr::get($o,'msg_crc')) {
+				$o['msg_crc'] = 'x'.md5($o['msg_src']);
+			}
+
+			$oo = Echomail::withTrashed()->where('mid',$o['mid'])->firstOrNew();
+			$oo->forceFill($o);
+			$oo->set_path = $path ? array_filter($path) : [];
+			$oo->set_seenby = $seenby ? array_filter($seenby): [];
+			$oo->set_packet = $packet;
+
+			try {
+				$oo->save(['timestamps'=>FALSE]);
+
+			} catch (\Exception $e) {
+				Log::error(sprintf('Echomail didnt move over: %s [%d] (%s@%d|%s)',$o['mid'],$c,$e->getFile(),$e->getLine(),$e->getMessage()));
+				dd(['e'=>$e,'o'=>$o,'oo'=>$oo]);
+			}
+
+			DB::connection('mongodb')->collection('echomails')->delete($o['mid']);
+		}
+		/**/
+
+		// Update old MID seenby with proper ID
+		foreach (DB::connection('cockroach')->table('echomail_seenby')->whereNotNull('mid')->whereNull('echomail_id')->cursor() as $o)
+		{
+			$eo = Echomail::where('mid',$o->mid)->get();
+
+			if ($eo->count() && $eo->count() == 1) {
+				DB::update('UPDATE echomail_seenby set echomail_id = ?, mid=NULL where echomail_id IS NULL AND mid = ? ',[
+					$eo->first()->id,
+					$o->mid,
+				]);
+
+			} elseif ($eo->count() > 1) {
+				Log::error(sprintf('Echomail [%s] has more than 1 record [%d] - skipped',$o->mid,$eo->count()));
+			}
+		}
+	}
+}

app/Http/Controllers/HomeController.php

@@ -121,6 +121,7 @@ class HomeController extends Controller
 		# Look for Messages
 		foreach (Echomail::select(['id','fftn_id','from'])
 				->where('msgid','like','%'.$request->query('term').'%')
+				->orWhere('replyid','like','%'.$request->query('term').'%')
 				->get() as $o)
 		{
 			$result->push(['id'=>$o->id,'name'=>sprintf('%s (%s)',$o->from,$o->fftn->ftn3d),'value'=>url('echomail/view',[$o->id]),'category'=>'Echomail']);

app/Jobs/MessageProcess.php

@@ -112,7 +112,7 @@ class MessageProcess implements ShouldQueue
 
 					$o->fftn_id = ($x=$this->msg->tftn_o) ? $x->id : NULL;
 					$o->tftn_id = ($x=$this->msg->fftn_o) ? $x->id : NULL;
-					$o->reply = $this->msg->msgid;
+					$o->replyid = $this->msg->msgid;
 					$o->msg = Process::format_msg($reply,$reject);
 
 					$o->tagline = 'Do you think it was fate which brought us together? Nah, bad luck :(';

app/Models/Address.php

@@ -11,7 +11,7 @@ use Illuminate\Support\Facades\DB;
 
 use App\Classes\FTN\Packet;
 use App\Http\Controllers\DomainController;
-use App\Traits\{ScopeActive,UsePostgres};
+use App\Traits\ScopeActive;
 
 /**
  * @todo Need to stop this from happening:
@@ -24,7 +24,9 @@ use App\Traits\{ScopeActive,UsePostgres};
  */
 class Address extends Model
 {
-	use ScopeActive,SoftDeletes,UsePostgres;
+	use ScopeActive,SoftDeletes;
+
+	protected $with = ['zone'];
 
 	/* SCOPES */
 
@@ -337,6 +339,21 @@ class Address extends Model
 		return ($o && $o->system->active) ? $o : NULL;
 	}
 
+	/**
+	 * Netmail waiting to be sent to this system
+	 *
+	 * @return Collection
+	 */
+	public function echomailWaiting(): Collection
+	{
+		return Echomail::select(['echomails.*'])
+			->join('echomail_seenby',['echomail_seenby.echomail_id'=>'echomails.id'])
+			->where('echomail_seenby.address_id',$this->id)
+			->whereNull('echomail_seenby.sent_at')
+			->whereNotNull('echomail_seenby.export_at')
+			->get();
+	}
+
 	/**
 	 * Get echomail for this node
 	 *
@@ -346,23 +363,17 @@ class Address extends Model
 	{
 		$pkt = NULL;
 
-		$echomail = DB::table('address_echomail')
+		if (($x=$this->echomailWaiting())
-			->select('echomail_id')
-			->where('address_id',$this->id)
-			->whereNull('sent_date')
-			->get();
-
-		if (($x=Echomail::select('*')
-			->whereIn('_id',$echomail->pluck('echomail_id')))
 			->count())
 		{
-			$pkt = $this->getPacket($x->get());
+			$pkt = $this->getPacket($x);
 
-			DB::table('address_echomail')
+			DB::table('echomail_seenby')
-				->whereIn('echomail_id',$echomail->pluck('echomail_id'))
+				->whereIn('echomail_id',$x->pluck('id'))
 				->where('address_id',$this->id)
-				->whereNull('sent_date')
+				->whereNull('sent_at')
-				->update(['sent_date'=>Carbon::now()]);
+				->whereNotNull('echomail_seenby.export_at')
+				->update(['sent_at'=>Carbon::now(),'packet'=>$pkt->name]);
 		}
 
 		return $pkt;
@@ -375,18 +386,19 @@ class Address extends Model
 	 */
 	public function getNetmail(): ?Packet
 	{
-		if (($x=Netmail::whereIn('tftn_id',(($x=$this->children) ? $x->pluck('id') : collect())->push($this->id))
+		$pkt = NULL;
-			->where(function($q) {
+
-				return $q->whereNull('sent')
+		if (($x=$this->netmailWaiting())
-					->orWhere('sent',FALSE);
-			}))
-			->whereNull('local')
 			->count())
 		{
-			return $this->getPacket($x->get());
+			$pkt = $this->getPacket($x);
+
+			DB::table('netmails')
+				->whereIn('id',$x->pluck('id'))
+				->update(['sent_at'=>Carbon::now(),'sent_pkt'=>$pkt->name]);
 		}
 
-		return NULL;
+		return $pkt;
 	}
 
 	/**
@@ -411,6 +423,18 @@ class Address extends Model
 		return $o;
 	}
 
+	/**
+	 * Netmail waiting to be sent to this system
+	 *
+	 * @return Collection
+	 */
+	public function netmailWaiting(): Collection
+	{
+		return Netmail::whereIn('tftn_id',(($x=$this->children) ? $x->pluck('id') : collect())->push($this->id))
+			->whereNull('sent_at')
+			->get();
+	}
+
 	/**
 	 * Parse a string and split it out as an FTN array
 	 *

app/Models/Domain.php

@@ -7,13 +7,15 @@ use Illuminate\Database\Eloquent\Factories\HasFactory;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\DB;
 use MongoDB\BSON\UTCDateTime;
 
-use App\Traits\ScopeActive;
+use App\Traits\{QueryCacheableConfig,ScopeActive};
 
 class Domain extends Model
 {
-	use HasFactory,ScopeActive;
+	use HasFactory,ScopeActive,QueryCacheableConfig;
+
 	private const CACHE_TIME = 3600;
 	private const STATS_MONTHS = 6;
 
@@ -65,17 +67,19 @@ class Domain extends Model
 
 		$key = sprintf('%s_%d','daily_area_stats',$this->id);
 
-		return Cache::driver('redis')->remember($key,self::CACHE_TIME,function() {
+Cache::forget($key);
-			$where = [
+		return Cache::remember($key,self::CACHE_TIME,function() {
-				'echoarea_id'=>$this->echoareas->pluck('id')->toArray(),
+			$gb ="CONCAT(EXTRACT('year',datetime)::string,'-',LPAD(EXTRACT('month',datetime)::string,2,'0'),'-',LPAD(EXTRACT('day',datetime)::string,2,'0')) AS datetime";
-				'datetime' => ['$gte',new UTCDateTime(Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth())],
-			];
 
-			$echostats = Echomail::countGroupBy(['datetime',['datetime'=>'%Y-%m-%d']],$where);
+			$echostats = Echomail::select([DB::raw($gb),DB::raw('COUNT(*)')])
+				->whereIn('id',$this->echoareas->pluck('id')->toArray())
+				->where('datetime','>=',Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth())
+				->groupBy('datetime')
+				->orderBy('datetime')
+				->get();
 
 			return $echostats
-				->sortBy(function($item) { return $item->id->datetime; })
+				->map(function($item) { return ['x'=>$item->datetime->timestamp*1000,'y'=>$item->count]; })
-				->map(function($item) { return ['x'=>Carbon::createFromFormat('Y-m-d',$item->id->datetime)->timestamp*1000,'y'=>$item->count]; })
 				->values();
 		});
 	}
@@ -87,17 +91,19 @@ class Domain extends Model
 
 		$key = sprintf('%s_%d-%d','daily_echoarea_stats',$this->id,$o->id);
 
-		return Cache::driver('redis')->remember($key,self::CACHE_TIME,function() use ($o) {
+Cache::forget($key);
-			$where = [
+		return Cache::remember($key,self::CACHE_TIME,function() use ($o) {
-				'echoarea_id'=>[$o->id],
+			$gb ="CONCAT(EXTRACT('year',datetime)::string,'-',LPAD(EXTRACT('month',datetime)::string,2,'0'),'-',LPAD(EXTRACT('day',datetime)::string,2,'0')) AS datetime";
-				'datetime' => ['$gte',new UTCDateTime(Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth())],
-			];
 
-			$echostats = Echomail::countGroupBy(['datetime',['datetime'=>'%Y-%m-%d']],$where);
+			$echostats = Echomail::select([DB::raw($gb),DB::raw('COUNT(*)')])
+				->whereIn('echoarea_id',[$o->id])
+				->where('datetime','>=',Carbon::now()->subMonths(self::STATS_MONTHS)->startOfMonth())
+				->groupBy('datetime')
+				->orderBy('datetime')
+				->get();
 
 			return $echostats
-				->sortBy(function($item) { return $item->id->datetime; })
+				->map(function($item) { return ['x'=>$item->datetime->timestamp*1000,'y'=>$item->count]; })
-				->map(function($item) { return ['x'=>Carbon::createFromFormat('Y-m-d',$item->id->datetime)->timestamp*1000,'y'=>$item->count]; })
 				->values();
 		});
 	}

app/Models/Echoarea.php

@@ -3,14 +3,16 @@
 namespace App\Models;
 
 use Carbon\Carbon;
+use Illuminate\Database\Eloquent\Collection;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Database\Eloquent\SoftDeletes;
 use Illuminate\Support\Facades\Cache;
-use App\Traits\{ScopeActive,UsePostgres};
+
+use App\Traits\ScopeActive;
 
 class Echoarea extends Model
 {
-	use SoftDeletes,ScopeActive,UsePostgres;
+	use SoftDeletes,ScopeActive;
 
 	private const CACHE_TIME = 3600;
 
@@ -28,28 +30,24 @@ class Echoarea extends Model
 
 	public function echomail()
 	{
-		return Echomail::select('*')
+		return $this->hasMany(Echomail::class)
-			->where('echoarea_id',$this->id);
+			->orderBy('datetime','ASC');
 	}
 
 	/* ATTRIBUTES */
 
 	public function getLastMessageAttribute(): ?Carbon
 	{
-		$key = sprintf('%s_%d','echo_last_message',$this->id);
+		return $this->echomail?->last()->datetime;
-
-		return Cache::driver('redis')->remember($key,self::CACHE_TIME,function() {
-			return ($x=$this->echomail()->orderBy('datetime','DESC')->first()) ? $x->datetime : NULL;
-		});
 	}
 
 	/* METHODS */
 
 	public function messages_count(int $period): int
 	{
-		$key = sprintf('%s_%d_%d','echo_mesages_count',$this->id,$period);
+		$key = sprintf('%s_%d_%d','echo_messages_count',$this->id,$period);
 
-		return Cache::driver('redis')->remember($key,self::CACHE_TIME,function() use ($period) {
+		return Cache::remember($key,self::CACHE_TIME,function() use ($period) {
 			switch ($period) {
 				case 1: // day
 					return $this->echomail()->where('datetime','>=',Carbon::now()->startOfday()->subDay())->count();
@@ -62,4 +60,20 @@ class Echoarea extends Model
 			}
 		});
 	}
+
+	/**
+	 * Number of messages waiting for address
+	 *
+	 * @param Address $ao
+	 * @return Collection
+	 */
+	public function waiting(Address $ao): Collection
+	{
+		return $this->echomail()
+			->join('echomail_seenby',['echomail_seenby.echomail_id'=>'echomails.id'])
+			->whereNull('sent_at')
+			->whereNotNull('export_at')
+			->where('address_id',$ao->id)
+			->get();
+	}
 }

app/Models/Echomail.php

@@ -3,25 +3,29 @@
 namespace App\Models;
 
 use Carbon\Carbon;
-use Illuminate\Support\Collection;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\SoftDeletes;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
-use Jenssegers\Mongodb\Eloquent\Model;
-use Jenssegers\Mongodb\Eloquent\SoftDeletes;
 
 use App\Classes\FTN\Message;
 use App\Interfaces\Packet;
-use App\Traits\{EncodeUTF8,MsgID,UseMongo};
+use App\Traits\{EncodeUTF8,MsgID};
 
 final class Echomail extends Model implements Packet
 {
-	use SoftDeletes,MsgID,UseMongo,EncodeUTF8;
+	use SoftDeletes,EncodeUTF8,MsgID;
 
 	private const LOGKEY = 'ME-';
+	private array $set_seenby = [];
+	private array $set_path = [];
+	private ?string $set_packet = NULL;
 
-	protected $collection = FALSE;
+	protected $casts = [
-
+		'kludges' => 'json',
-	protected $casts = [ 'kludges' => 'json' ];
+		'rogue_seenby' => 'json',
+		'rogue_path' => 'json',
+	];
 
 	private const cast_utf8 = [
 		'to',
@@ -35,6 +39,20 @@ final class Echomail extends Model implements Packet
 
 	protected $dates = ['datetime'];
 
+	public function __set($key, $value)
+	{
+		switch ($key) {
+			case 'set_path':
+			case 'set_packet':
+			case 'set_seenby':
+				$this->{$key} = $value;
+				break;
+
+			default:
+				parent::__set($key,$value);
+		}
+	}
+
 	public static function boot()
 	{
 		parent::boot();
@@ -46,29 +64,43 @@ final class Echomail extends Model implements Packet
 				return;
 			}
 
-			// See if we need to export this message.
+			// Save the seenby
-			$exportto = $model->echoarea->addresses->pluck('system')->diff($model->seenby->pluck('system'));
+			foreach ($model->set_seenby as $aoid) {
-
+				DB::insert('INSERT INTO echomail_seenby (echomail_id,address_id,packet) VALUES (?,?,?)',[
-			$export_ao = collect();
+					$model->id,
-			foreach ($model->echoarea->domain->zones as $zo) {
+					$aoid,
-				foreach ($exportto as $so) {
+					$model->set_packet,
-					$export_ao = $export_ao->merge($so->match($zo));
-				}
-			}
-
-			// Add to export
-			foreach ($export_ao as $ao) {
-				Log::info(sprintf('%s:- Exporting message [%s] to [%s]',self::LOGKEY,$model->id,$ao->ftn));
-
-				DB::table('address_echomail')->insert([
-					'address_id'=>$ao->id,
-					'echomail_id'=>$model->id,
-					'export_date'=>Carbon::now()
 				]);
 			}
 
-			$model->seenby = $model->seenby->merge($export_ao)->pluck('id')->toArray();
+			// Save the Path
-			$model->save();
+			$ppoid = NULL;
+			foreach ($model->set_path as $aoid) {
+				$po = DB::select('INSERT INTO echomail_path (echomail_id,address_id,parent_id) VALUES (?,?,?) RETURNING id',[
+					$model->id,
+					$aoid,
+					$ppoid,
+				]);
+
+				$ppoid = $po[0]->id;
+			}
+
+			// See if we need to export this message.
+			$exportto = $model->echoarea->addresses->pluck('id')->diff($model->set_seenby);
+
+			if ($exportto->count()) {
+				Log::debug(sprintf('%s:- Exporting message [%s] to [%s]',self::LOGKEY,$model->id,$exportto->join(',')));
+
+				// Save the seenby for the exported systems
+				$export_at = Carbon::now();
+				foreach ($exportto as $aoid) {
+					DB::insert('INSERT INTO echomail_seenby (echomail_id,address_id,export_at) VALUES (?,?,?)',[
+						$model->id,
+						$aoid,
+						$export_at,
+					]);
+				}
+			}
 		});
 	}
 
@@ -81,35 +113,19 @@ final class Echomail extends Model implements Packet
 
 	public function fftn()
 	{
-		return $this
+		return $this->belongsTo(Address::class)
-			->setConnection('pgsql')
-			->belongsTo(Address::class)
 			->withTrashed();
 	}
 
-	/* ATTRIBUTES */
+	public function seenby()
-
-	public function getKludgesAttribute(?string $value): Collection
 	{
-		return collect($this->castAttribute('kludges',$value));
+		return $this->belongsToMany(Address::class,'echomail_seenby')
+			->ftnOrder();
 	}
 
-	public function getPathAttribute(?array $value): Collection
+	public function path()
 	{
-		if (is_null($value))
+		return $this->belongsToMany(Address::class,'echomail_path');
-			return collect();
-
-		return Address::whereIn('id',$value)
-			->orderBy(DB::raw(sprintf("position (id::text in '(%s)')",join(',',$value))))
-			->get();
-	}
-
-	public function getSeenByAttribute(?array $value): Collection
-	{
-		if (is_null($value))
-			return collect();
-
-		return Address::whereIn('id',$value)->get();
 	}
 
 	/* METHODS */
@@ -149,11 +165,11 @@ final class Echomail extends Model implements Packet
 		if ($this->kludges)
 			$o->kludge = $this->kludges;
 
-		$o->kludge->put('mid',$this->id);
+		$o->kludge->put('dbid',$this->id);
 
 		$o->msgid = $this->msgid;
-		if ($this->reply)
+		if ($this->replyid)
-			$o->reply = $this->reply;
+			$o->replyid = $this->replyid;
 
 		$o->message = $this->msg;
 

app/Models/Netmail.php

@@ -3,19 +3,19 @@
 namespace App\Models;
 
 use Carbon\Carbon;
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\SoftDeletes;
 use Illuminate\Support\Facades\Log;
-use Jenssegers\Mongodb\Eloquent\Model;
-use Jenssegers\Mongodb\Eloquent\SoftDeletes;
 
 use App\Classes\FTN\Message;
 use App\Interfaces\Packet;
-use App\Traits\{EncodeUTF8,UseMongo};
+use App\Traits\EncodeUTF8;
 
 final class Netmail extends Model implements Packet
 {
 	private const LOGKEY = 'MN-';
 
-	use SoftDeletes,UseMongo,EncodeUTF8;
+	use SoftDeletes,EncodeUTF8;
 
 	private const cast_utf8 = [
 		'to',
@@ -80,8 +80,8 @@ final class Netmail extends Model implements Packet
 			$o->flags = $this->flags;
 
 			$o->msgid = sprintf('%s %08x',$this->fftn->ftn3d,crc32($this->id));
-			if ($this->reply)
+			if ($this->replyid)
-				$o->reply = $this->reply;
+				$o->replyid = $this->replyid;
 
 			$o->message = $this->msg;
 			$o->tagline = $this->tagline;

app/Models/System.php

@@ -8,10 +8,11 @@ use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Auth;
 
 use App\Http\Controllers\DomainController;
+use App\Traits\QueryCacheableConfig;
 
 class System extends Model
 {
-	use HasFactory;
+	use HasFactory,QueryCacheableConfig;
 
 	protected $dates = ['last_session'];
 
@@ -37,10 +38,7 @@ class System extends Model
 	public function addresses()
 	{
 		return $this->hasMany(Address::class)
-			->orderBy('region_id')
+			->FTNorder();
-			->orderBy('host_id')
-			->orderBy('node_id')
-			->orderBy('point_id');
 	}
 
 	/**

app/Models/Zone.php

@@ -4,11 +4,11 @@ namespace App\Models;
 
 use Illuminate\Database\Eloquent\Model;
 
-use App\Traits\ScopeActive;
+use App\Traits\{QueryCacheableConfig,ScopeActive};
 
 class Zone extends Model
 {
-	use ScopeActive;
+	use ScopeActive,QueryCacheableConfig;
 
 	/* SCOPES */
 

app/Traits/MsgID.php

@@ -7,13 +7,15 @@ namespace App\Traits;
 
 use Illuminate\Support\Arr;
 
+use App\Classes\FTN\Message;
 use App\Models\Setup;
 
 trait MsgID
 {
 	public function save(array $options = [])
 	{
-		if (! $this->exists) {
+		// Only create a MSGID for locally generated conetnt
+		if ((! $this->exists) && ($this->flags & Message::FLAG_LOCAL)) {
 			$ftns = Setup::findOrFail(config('app.id'))->system->match($this->fftn->zone)->first();
 
 			if (is_null(Arr::get($this->attributes,'msgid')))

app/Traits/QueryCacheableConfig.php

@@ -0,0 +1,17 @@
+<?php
+
+/**
+ * Set defaults of QueryCacheable
+ */
+namespace App\Traits;
+
+use Rennokki\QueryCache\Traits\QueryCacheable;
+
+trait QueryCacheableConfig
+{
+	use QueryCacheable;
+
+	public $cacheFor = 900; // cache time, in seconds
+	protected static $flushCacheOnUpdate = TRUE;
+	public $cacheDriver = 'memcached';
+}

app/Traits/UseMongo.php

@@ -1,104 +0,0 @@
-<?php
-
-/**
- * Models that store data in Mongo
- */
-namespace App\Traits;
-
-use Illuminate\Database\Eloquent\Collection;
-use Illuminate\Support\Arr;
-
-trait UseMongo
-{
-	/**
-	 * Resolve a connection instance.
-	 * We need to do this, because our relations are in pgsql and somehow loading a relation changes this models
-	 * connection information. Using protected $connection is not enough.
-	 *
-	 * @param  string|null  $connection
-	 */
-	public static function resolveConnection($connection = null)
-	{
-		return static::$resolver->connection('mongodb');
-	}
-
-	/* ATTRIBUTES */
-
-	public function getMsgAttribute($value): string
-	{
-		return utf8_decode($value);
-	}
-
-	public function setMsgAttribute($value): void
-	{
-		$this->attributes['msg'] = utf8_encode($value);
-	}
-
-	public function getSubjectAttribute($value): string
-	{
-		return utf8_decode($value);
-	}
-
-	public function setSubjectAttribute($value): void
-	{
-		$this->attributes['subject'] = utf8_encode($value);
-	}
-
-	/* METHODS */
-
-	public static function countGroupBy(array $fields,array $where=[]): Collection
-	{
-		$query = collect();
-
-		if (count($where)) {
-			$where_condition = [];
-
-			foreach ($where as $key => $values) {
-				if (! is_array($values))
-					throw new \Exception('Where values must be an array.');
-
-				switch ($x=Arr::get($values,0)) {
-					case '$gt':
-					case '$gte':
-						$where_condition[$key] = [ $x => Arr::get($values,1)];
-						break;
-
-					case '$in':
-					default:
-						$where_condition[$key] = ['$in' => $values];
-				}
-
-			}
-
-			$query->push([
-				'$match' => [ '$and'=> [$where_condition]]
-			]);
-		}
-
-		$gb = collect();
-		foreach ($fields as $field)
-			if (is_array($field)) {
-				foreach ($field as $k=>$v) {
-					$gb->put('datetime',['$dateToString'=>['format'=>$v,'date'=>'$'.$k]]);
-				}
-
-			} else {
-				$gb->put($field,'$'.$field);
-			}
-
-		$query->push([
-			'$group' => [
-				'_id' => $gb->toArray(),
-				'count' => ['$sum' => 1]
-			]
-		]);
-
-		return (new self)
-			->groupBy($field)
-			->raw(function($collection) use ($query) {
-				return $collection->aggregate(
-					$query->toArray()
-				);
-			});
-	}
-}

app/Traits/UsePostgres.php

@@ -1,21 +0,0 @@
-<?php
-
-/**
- * Models that store data in Postgres
- */
-namespace App\Traits;
-
-trait UsePostgres
-{
-	/**
-	 * Resolve a connection instance.
-	 * We need to do this, because our relations are in pgsql and somehow loading a relation changes this models
-	 * connection information. Using protected $connection is not enough.
-	 *
-	 * @param  string|null  $connection
-	 */
-	public static function resolveConnection($connection = null)
-	{
-		return static::$resolver->connection('pgsql');
-	}
-}

composer.json

@@ -17,6 +17,7 @@
         "laravel/framework": "^8.0",
         "laravel/passport": "^10.1",
         "laravel/ui": "^3.2",
+        "leenooks/cockroachdb-laravel": "^0.1.0",
         "rennokki/laravel-eloquent-query-cache": "^3.1",
         "repat/laravel-job-models": "^0.5.1"
     },
@@ -54,6 +55,12 @@
             "Tests\\": "tests/"
         }
     },
+    "repositories": [
+        {
+            "type": "vcs",
+            "url": "https://github.com/leenooks/cockroachdb-laravel"
+        }
+    ],
     "minimum-stability": "dev",
     "prefer-stable": true,
     "scripts": {

config/database.php

@@ -67,7 +67,10 @@ return [
             'prefix' => '',
             'prefix_indexes' => true,
             'schema' => 'public',
-            'sslmode' => 'prefer',
+            'sslmode' => 'disable', //depends on your security level https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS
+            'sslrootcert' => env('DB_SSLROOTCERT', 'config/ssl/ca.crt'),
+            'sslcert' => env('DB_SSLCERT', 'config/ssl/client.crt'),
+            'sslkey' => env('DB_SSLKEY', 'config/ssl/client.key'),
         ],
 
         'sqlsrv' => [
@@ -92,13 +95,13 @@ return [
             'charset' => 'utf8',
             'prefix' => '',
             'schema' => 'public',
-            'sslmode' => 'prefer',
+            'sslmode' => env('DB_SSLMODE', 'prefer'),
 
             // Only set these keys if you want to run en secure mode
             // otherwise you can them out of the configuration array
-            #'sslcert' => env('DB_SSLCERT', 'client.crt'),
+            'sslcert' => env('DB_SSLCERT', 'config/ssl/client.crt'),
-            #'sslkey' => env('DB_SSLKEY', 'client.key'),
+            'sslkey' => env('DB_SSLKEY', 'config/ssl/client.key'),
-            #'sslrootcert' => env('DB_SSLROOTCERT', 'ca.crt'),
+            'sslrootcert' => env('DB_SSLROOTCERT', 'config/ssl/ca.crt'),
         ],
 
         'mongodb' => [

database/migrations/2021_06_18_112022_update_zones.php

@@ -16,11 +16,10 @@ class UpdateZones extends Migration
 		DB::statement('ALTER TABLE zones ALTER COLUMN domain_id SET NOT NULL');
 
 		Schema::table('zones', function (Blueprint $table) {
+			$table->integer('system_id');
 			$table->dropColumn(['description','ipv4','ipv4_mask','ipv6','ipv6_mask','zt_id']);
 			$table->string('ztid')->nullable();
 			$table->dropUnique(['domain_id']);
-
-			$table->integer('system_id');
 			$table->foreign('system_id')->references('id')->on('systems');
 		});
     }

database/migrations/2021_06_21_121910_add_system_to_setups.php

@@ -14,8 +14,8 @@ class AddSystemToSetups extends Migration
 	public function up()
 	{
 		Schema::table('setups', function (Blueprint $table) {
-			$table->integer('options');
 			$table->integer('system_id');
+			$table->integer('options');
 			$table->foreign('system_id')->references('id')->on('systems');
 		});
 	}

database/migrations/2022_01_01_235631_create_mail.php

@@ -0,0 +1,137 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateMail extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('echomails', function (Blueprint $table) {
+            $table->id();
+            $table->timestamps();
+			$table->softDeletes();
+			$table->string('mid')->nullable();
+
+			$table->string('to',128);
+			$table->string('from',128);
+			$table->string('subject',256);
+			$table->dateTimeTz('datetime');
+			$table->tinyInteger('tzoffset');
+
+			$table->string('msgid')->nullable();
+			$table->string('replyid')->nullable();
+
+			$table->text('msg');
+			$table->string('msg_src')->nullable();
+			$table->string('msg_crc')->nullable();
+			$table->string('tagline')->nullable();
+			$table->string('tearline')->nullable();
+			$table->string('origin')->nullable();
+
+			$table->tinyInteger('flags')->nullable();
+			$table->jsonb('rogue_path')->nullable();
+			$table->jsonb('rogue_seenby')->nullable();
+			$table->jsonb('kludges')->nullable();
+
+			$table->integer('echoarea_id');
+			$table->foreign('echoarea_id')->references('id')->on('echoareas');
+
+			$table->integer('fftn_id');
+			$table->foreign('fftn_id')->references('id')->on('addresses');
+		});
+
+		DB::statement("ALTER TABLE address_echomail RENAME COLUMN echomail_id TO mid");
+		DB::statement("ALTER TABLE address_echomail ALTER COLUMN mid DROP NOT NULL");
+		DB::statement("ALTER TABLE address_echomail RENAME COLUMN export_date TO export_at");
+		DB::statement("ALTER TABLE address_echomail ALTER COLUMN export_at DROP NOT NULL");
+		DB::statement("ALTER TABLE address_echomail RENAME COLUMN sent_date TO sent_at");
+		DB::statement("ALTER TABLE address_echomail RENAME TO echomail_seenby");
+		Schema::table('echomail_seenby', function (Blueprint $table) {
+			$table->integer('echomail_id')->nullable()->first();
+			$table->string('packet')->nullable();
+
+			$table->foreign('echomail_id')->references('id')->on('echomails');
+		});
+
+		Schema::create('echomail_path', function (Blueprint $table) {
+			$table->id();
+
+			$table->integer('address_id');
+			$table->foreign('address_id')->references('id')->on('addresses');
+
+			$table->unique(['id','echomail_id']);
+			$table->unique(['echomail_id','address_id','parent_id']);
+
+			$table->integer('parent_id')->nullable();
+			$table->foreign(['parent_id','echomail_id'])->references(['id','echomail_id'])->on('echomail_path');
+
+			$table->integer('echomail_id');
+			$table->foreign('echomail_id')->references('id')->on('echomails');
+		});
+
+		Schema::create('netmails', function (Blueprint $table) {
+			$table->id();
+			$table->timestamps();
+			$table->softDeletes();
+			$table->string('mid')->nullable();
+
+			$table->string('to',128);
+			$table->string('from',128);
+			$table->string('subject',256);
+
+			$table->dateTimeTz('datetime');
+			$table->tinyInteger('tzoffset')->nullable();
+
+			$table->tinyInteger('flags')->nullable();
+			$table->tinyInteger('cost')->nullable();
+			$table->string('msgid')->nullable();
+			$table->string('replyid')->nullable();
+
+			$table->text('msg');
+			$table->text('msg_src')->nullable();
+			$table->integer('msg_crc')->nullable();
+
+			$table->string('tagline')->nullable();
+			$table->string('tearline')->nullable();
+			$table->boolean('local')->default(FALSE);
+
+			$table->string('recv_pkt')->nullable();
+			$table->string('sent_pkt')->nullable();
+			$table->dateTimeTz('sent_at')->nullable();
+
+			$table->integer('fftn_id');
+			$table->integer('tftn_id');
+			$table->foreign('fftn_id')->references('id')->on('addresses');
+			$table->foreign('tftn_id')->references('id')->on('addresses');
+		});
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+		Schema::table('echomail_seenby', function (Blueprint $table) {
+			$table->dropForeign(['echomail_id']);
+			$table->dropColumn(['packet','echomail_id']);
+		});
+		DB::commit();
+		DB::statement("ALTER TABLE echomail_seenby RENAME TO address_echomail");
+		DB::statement("ALTER TABLE address_echomail RENAME COLUMN mid TO echomail_id;");
+		DB::statement("ALTER TABLE address_echomail RENAME COLUMN export_at TO export_date;");
+		DB::statement("ALTER TABLE address_echomail RENAME COLUMN sent_at TO sent_date;");
+
+        Schema::dropIfExists('echomail_path');
+		Schema::dropIfExists('echomails');
+		Schema::dropIfExists('netmails');
+    }
+}

resources/views/domain/view.blade.php

@@ -261,7 +261,13 @@
 				},
 			},
 			legend: {
-				symbolWidth: 40
+				align: 'right',
+				layout: 'vertical',
+				symbolWidth: 40,
+				floating: false,
+				navigation: {
+					arrowSize: 10
+				},
 			},
 			plotOptions: {
 				series: {

resources/views/echoarea/addedit.blade.php

@@ -111,5 +111,46 @@
 				</div>
 			</div>
 		</div>
+
+		<div class="row">
+			<div class="col-12">
+				<h4>Info</h4>
+				<p>There are <strong class="highlight">{{ number_format($o->echomail->count()) }}</strong> messages in this area, and the oldest message in this area is <strong class="highlight">{{ $o->last_message ? $o->last_message->format('Y-m-d H:i:s') : 'None' }}</strong>.</p>
+			</div>
+		</div>
+
+		@can('admin',$o)
+			@if($o->addresses->count())
+				<div class="row">
+					<div class="col-12">
+						<h3>Exporting to the following adresses:</h3>
+
+						<table class="table monotable" id="system">
+							<thead>
+							<tr>
+								<th>System</th>
+								<th>Address</th>
+								<th>Last Session</th>
+								<th>Oldest Message</th>
+								<th class="text-right">Messages Waiting</th>
+							</tr>
+							</thead>
+
+							<tbody>
+							@foreach ($o->addresses as $ao)
+								<tr>
+									<td><a href="{{ url('ftn/system/addedit',[$ao->system_id]) }}">{{ $ao->system->full_name($ao) }}</a> @auth<span class="float-end"><small>@if($ao->session('sespass'))<sup>{{ $ao->session('default') ? '**' : '*' }}</sup>@elseif($ao->system->setup)<sup class="success">+</sup>@endif[{{ $ao->system_id }}]</small></span>@endauth</td>
+									<td>{{ $ao->ftn_3d }}</td>
+									<td>{{ $ao->system->last_session ? $ao->system->last_session->format('Y-m-d H:i') : '-' }}</td>
+									<td>{{ ($x=$o->waiting($ao))->count() ? $x->first()->datetime->format('Y-m-d H:i') : '-' }}</td>
+									<td class="text-right">{{ number_format($x->count()) }}</td>
+								</tr>
+							@endforeach
+							</tbody>
+						</table>
+					</div>
+				</div>
+			@endif
+		@endcan
 	</form>
 @endsection

resources/views/system/addedit.blade.php

@@ -217,7 +217,7 @@ use App\Http\Controllers\DomainController as DC;
 				<div class="accordion-item">
 					<h3 class="accordion-header" id="routing" data-bs-toggle="collapse" data-bs-target="#collapse_routing" aria-expanded="false" aria-controls="collapse_routing">Mail Routing</h3>
 
-					<div id="collapse_routing" class="accordion-collapse collapse {{ ($flash=='routing') ? 'show' : '' }}" aria-labelledby="addresses" data-bs-parent="#accordion_homepage">
+					<div id="collapse_routing" class="accordion-collapse collapse {{ ($flash=='routing') ? 'show' : '' }}" aria-labelledby="routing" data-bs-parent="#accordion_homepage">
 						<div class="accordion-body">
 							<div class="row">
 								<!-- Zone mail sent to -->
@@ -288,6 +288,61 @@ use App\Http\Controllers\DomainController as DC;
 						</div>
 					</div>
 				</div>
+
+				<!-- Waiting Mail -->
+				<div class="accordion-item">
+					<h3 class="accordion-header" id="mail" data-bs-toggle="collapse" data-bs-target="#collapse_mail" aria-expanded="false" aria-controls="collapse_mail">Mail Waiting</h3>
+
+					<div id="collapse_mail" class="accordion-collapse collapse {{ ($flash=='mail') ? 'show' : '' }}" aria-labelledby="mail" data-bs-parent="#accordion_homepage">
+						<div class="accordion-body">
+							<div class="row">
+								<!-- Netmail -->
+								<div class="col-6">
+									Netmails are waiting for these addresses:
+									<table class="table monotable">
+										<thead>
+										<tr>
+											<th>Address</th>
+											<th>Netmail</th>
+										</tr>
+										</thead>
+
+										<tbody>
+										@foreach ($o->addresses->sortBy('zone.zone_id') as $ao)
+											<tr>
+												<td>{{ $ao->ftn3d }}</td>
+												<td>{{ $ao->netmailWaiting()->count() }}</td>
+											</tr>
+										@endforeach
+										</tbody>
+									</table>
+								</div>
+
+								<!-- Echomail -->
+								<div class="col-6">
+									Echomail waiting for these addresses:
+									<table class="table monotable">
+										<thead>
+										<tr>
+											<th>Address</th>
+											<th>Echomail</th>
+										</tr>
+										</thead>
+
+										<tbody>
+										@foreach ($o->addresses->sortBy('zone.zone_id') as $ao)
+											<tr>
+												<td>{{ $ao->ftn3d }}</td>
+												<td>{{ $ao->echomailWaiting()->count() }}</td>
+											</tr>
+										@endforeach
+										</tbody>
+									</table>
+								</div>
+							</div>
+						</div>
+					</div>
+				</div>
 			@endif
 		@else
 			@include('system.form-system')

resources/views/widgets/message.blade.php

@@ -1,22 +1,22 @@
 <div class="row">
+	<div class="col-4">
+		TO: <strong class="highlight">{!! \App\Classes\FTN\Message::tr($msg->to) !!}</strong>
+	</div>
 	<div class="col-4">
 		DATE: <strong class="highlight">{{ $msg->datetime->format('Y-m-d H:i:s') }}</strong>
 	</div>
-	<div class="col-4">
-		MSGID: <strong class="highlight">{{ $msg->msgid }}</strong>@if($x=\App\Models\Echomail::where('reply',$msg->msgid)->count()) (<strong class="highlight">{{$x}}</strong> replies)@endif @if($msg->reply)<br>REPLY: <strong class="highlight">{{ $msg->reply }}</strong>@endif
-	</div>
 </div>
 
-<div class="row pb-2">
+<div class="row pt-1">
 	<div class="col-4">
 		FROM: <strong class="highlight">{!! \App\Classes\FTN\Message::tr($msg->from) !!}</strong> (<strong class="highlight">{{ $msg->fftn->ftn }}</strong>)
 	</div>
 	<div class="col-4">
-		TO: <strong class="highlight">{!! \App\Classes\FTN\Message::tr($msg->to) !!}</strong>
+		MSGID: <strong class="highlight">{{ $msg->msgid }}</strong>@if($x=\App\Models\Echomail::where('replyid',$msg->msgid)->count()) (<strong class="highlight">{{$x}}</strong> replies)@endif @if($msg->replyid)<br>REPLY: <strong class="highlight">{{ $msg->replyid }}</strong>@endif
 	</div>
 </div>
 
-<div class="row pb-2">
+<div class="row pt-1 pb-2">
 	<div class="col-8">
 		SUBJECT: <strong class="highlight">{!! \App\Classes\FTN\Message::tr($msg->subject) !!}</strong>
 	</div>