Optimize the SQL queries that finds unsent echomail,netmail and files

app/Classes/Dynamic/HubStats.php

@@ -9,6 +9,7 @@ use Illuminate\Support\Facades\Log;
 
 use App\Classes\Dynamic;
 use App\Models\Address;
+use App\Traits\HubStats as HubStatsTrait;
 
 /**
  * This method will generate the hub status for an upstream Host/RC/ZC
@@ -18,6 +19,8 @@ use App\Models\Address;
  */
 class HubStats extends Dynamic
 {
+	use HubStatsTrait;
+
 	private const LOGKEY = 'DHS';
 
 	private string $name = '';
@@ -32,41 +35,8 @@ class HubStats extends Dynamic
 	{
 		$date = Carbon::now()->yesterday()->endOfday();
 
-		$r = Address::select([
+		$r = $this->HubStats($date)
-			'a.id',
+			->where('zones.id',$this->ao->zone_id);
-			'addresses.system_id',
-			'addresses.zone_id',
-			'addresses.region_id',
-			'addresses.host_id',
-			'addresses.node_id',
-			'addresses.point_id',
-			'addresses.hub_id',
-			'addresses.role',
-			DB::raw('sum(a.uncollected_echomail) as uncollected_echomail'),
-			DB::raw('sum(a.uncollected_netmail) as uncollected_netmail'),
-			DB::raw('sum(a.uncollected_files) as uncollected_files')
-		])
-			->from(
-				Address::UncollectedEchomailTotal()
-					->where('echomails.created_at','<',$date)
-					->union(Address::UncollectedNetmailTotal()
-						->where('netmails.created_at','<',$date)
-					)
-					->union(Address::UncollectedFilesTotal()
-						->where('files.created_at','<',$date)
-					),'a')
-			->where('systems.active',true)
-			->where('addresses.active',TRUE)
-			->where('zones.active',TRUE)
-			->where('domains.active',TRUE)
-			->where('zones.id',$this->ao->zone_id)
-			->join('addresses',['addresses.id'=>'a.id'])
-			->join('systems',['systems.id'=>'addresses.system_id'])
-			->join('zones',['zones.id'=>'addresses.zone_id'])
-			->join('domains',['domains.id'=>'zones.domain_id'])
-			->ftnOrder()
-			->groupBy('addresses.system_id','a.id','addresses.zone_id','addresses.region_id','addresses.host_id','addresses.node_id','addresses.point_id','addresses.hub_id','addresses.role')
-			->with(['system','zone.domain']);
 
 		$header = "| %-12s | %4d | %3d | %3d | %16s | %5s | %5s |\r\n";
 

app/Http/Controllers/HomeController.php

@@ -14,9 +14,12 @@ use App\Classes\FTN\{Message,Packet};
 use App\Http\Requests\SetupRequest;
 use App\Models\File as FileModel;
 use App\Models\{Address,Echomail,Netmail,Setup,System};
+use App\Traits\HubStats;
 
 class HomeController extends Controller
 {
+	use HubStats;
+
 	public function home()
 	{
 		return redirect(Auth::check() ? 'dashboard' : 'about');
@@ -246,49 +249,8 @@ class HomeController extends Controller
 	{
 		$date = Carbon::now()->yesterday()->endOfday();
 
-		$r = Address::select([
-				'a.id',
-				'a.system_id',
-				'a.zone_id',
-				'addresses.region_id',
-				'a.host_id',
-				'a.node_id',
-				'a.point_id',
-				'addresses.hub_id',
-				'addresses.role',
-				DB::raw('sum(a.uncollected_echomail) as uncollected_echomail'),
-				DB::raw('sum(a.uncollected_netmail) as uncollected_netmail'),
-				DB::raw('sum(a.uncollected_files) as uncollected_files')
-			])
-			->from(
-				Address::UncollectedEchomailTotal()
-					->where('echomails.created_at','<',$this->yesterdayEOD())
-					->union(Address::UncollectedNetmailTotal()
-						->where('netmails.created_at','<',$this->yesterdayEOD())
-					)
-					->union(Address::UncollectedFilesTotal()
-						->where('files.created_at','<',$this->yesterdayEOD())
-					),'a')
-			->where('systems.active',TRUE)
-			->where('addresses.active',TRUE)
-			->where('zones.active',TRUE)
-			->where('domains.active',TRUE)
-			->when(! ($x=Auth::user()) || (! $x->isAdmin()),function($query) { return $query->where('domains.public',TRUE); })
-			->join('addresses',['addresses.id'=>'a.id'])
-			->join('systems',['systems.id'=>'a.system_id'])
-			->join('zones',['zones.id'=>'addresses.zone_id'])
-			->join('domains',['domains.id'=>'zones.domain_id'])
-			->ftnOrder()
-			->groupBy('a.system_id','a.id','a.zone_id','addresses.region_id','a.host_id','a.node_id','a.point_id','addresses.hub_id','addresses.role')
-			->with(['system','zone.domain']);
-
 		return view('status')
 			->with('date',$date)
-			->with('uncollected',$r->get());
+			->with('uncollected',$this->HubStats($date)->get());
-	}
-
-	private function yesterdayEOD(): Carbon
-	{
-		return Carbon::now()->yesterday()->endOfday();
 	}
 }

app/Jobs/MailSend.php

@@ -2,16 +2,16 @@
 
 namespace App\Jobs;
 
+use Carbon\Carbon;
 use Illuminate\Foundation\Bus\Dispatchable;
-use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
 use Repat\LaravelJobs\Job;
 
-use App\Models\Address;
+use App\Traits\HubStats;
 
 class MailSend #implements ShouldQueue
 {
-	use Dispatchable;
+	use HubStats,Dispatchable;
 
 	private const LOGKEY = 'JMS';
 
@@ -25,46 +25,22 @@ class MailSend #implements ShouldQueue
 	 */
 	public function handle(): void
 	{
-		$u = Address::select([
-			'a.id',
-			'a.system_id',
-			'a.zone_id',
-			'addresses.region_id',
-			'a.host_id',
-			'a.node_id',
-			'a.point_id',
-			'addresses.hub_id',
-			'addresses.role',
-			DB::raw('sum(a.uncollected_echomail) as uncollected_echomail'),
-			DB::raw('sum(a.uncollected_netmail) as uncollected_netmail'),
-			DB::raw('sum(a.uncollected_files) as uncollected_files')
-		])
-			->from(Address::UncollectedEchomailTotal()->union(Address::UncollectedNetmailTotal())->union(Address::UncollectedFilesTotal()),'a')
-			->where('systems.active',true)
-			->where('addresses.active',TRUE)
-			->where('zones.active',TRUE)
-			->where('domains.active',TRUE)
-			->join('addresses',['addresses.id'=>'a.id'])
-			->join('systems',['systems.id'=>'a.system_id'])
-			->join('zones',['zones.id'=>'a.zone_id'])
-			->join('domains',['domains.id'=>'zones.domain_id'])
-			->groupBy('a.system_id','a.id','a.zone_id','addresses.region_id','a.host_id','a.node_id','a.point_id','addresses.hub_id','addresses.role')
-			->with(['system','zone.domain'])
-			->get();
-
 		// Return the system we poll
-		$u = $u->transform(function($item) {
+		$u = $this
-			if ($x=$item->uplink()) {
+			->HubStats(Carbon::now())
-				$x->uncollected_echomail = $item->uncollected_echomail;
+			->get()
-				$x->uncollected_netmail = $item->uncollected_netmail;
+			->transform(function($item) {
-				$x->uncollected_files = $item->uncollected_files;
+				if ($x=$item->uplink()) {
+					$x->uncollected_echomail = $item->uncollected_echomail;
+					$x->uncollected_netmail = $item->uncollected_netmail;
+					$x->uncollected_files = $item->uncollected_files;
 
-				return $x;
+					return $x;
 
-			} else {
+				} else {
-				return $item;
+					return $item;
-			}
+				}
-		})
+			})
 			->filter(function($item) {
 				if ($item->system->autohold)
 					return NULL;

app/Models/Address.php

@@ -450,115 +450,6 @@ class Address extends Model
 			->orderBy('point_id');
 	}
 
-	/**
-	 * Return a list of addresses and the amount of uncollected echomail
-	 *
-	 * @param $query
-	 * @return mixed
-	 */
-	public function scopeUncollectedEchomail($query)
-	{
-		return $query
-			->join('echomail_seenby',['echomail_seenby.address_id'=>'addresses.id'])
-			->join('echomails',['echomails.id'=>'echomail_seenby.echomail_id'])
-			->whereNotNull('export_at')
-			->whereNull('sent_at')
-			->whereNull('echomails.deleted_at')
-			->groupBy('addresses.id')
-			->dontCache();
-	}
-
-	public function scopeUncollectedEchomailTotal($query)
-	{
-		return $query
-			->select([
-				'addresses.id',
-				'zone_id',
-				'host_id',
-				'node_id',
-				'point_id',
-				'system_id',
-				DB::raw('count(addresses.id) as uncollected_echomail'),
-				DB::raw('0 as uncollected_netmail'),
-				DB::raw('0 as uncollected_files'),
-			])
-			->UncollectedEchomail();
-	}
-
-	/**
-	 * Return a list of addresses and the amount of uncollected files
-	 *
-	 * @param $query
-	 * @return mixed
-	 */
-	public function scopeUncollectedFiles($query)
-	{
-		return $query
-			->join('file_seenby',['file_seenby.address_id'=>'addresses.id'])
-			->join('files',['files.id'=>'file_seenby.file_id'])
-			->whereNotNull('export_at')
-			->whereNull('sent_at')
-			->whereNull('files.deleted_at')
-			->groupBy('addresses.id')
-			->dontCache();
-	}
-
-	public function scopeUncollectedFilesTotal($query)
-	{
-		return $query
-			->select([
-				'addresses.id',
-				'zone_id',
-				'host_id',
-				'node_id',
-				'point_id',
-				'system_id',
-				DB::raw('0 as uncollected_echomail'),
-				DB::raw('0 as uncollected_netmail'),
-				DB::raw('count(addresses.id) as uncollected_files')
-			])
-			->UncollectedFiles();
-	}
-
-	public function scopeUncollectedNetmail($query)
-	{
-		return $query
-			->join('netmails',['netmails.tftn_id'=>'addresses.id'])
-			->where(function($query) {
-				return $query->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_INTRANSIT))
-					->orWhereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL));
-			})
-			->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT))
-			->whereNull('sent_pkt')
-			->whereNull('sent_at')
-			->whereNull('netmails.deleted_at')
-			->groupBy('addresses.id')
-			->dontCache();
-	}
-
-	/**
-	 * Return a list of addresses and the amount of uncollected netmail
-	 *
-	 * @param $query
-	 * @return mixed
-	 */
-	public function scopeUncollectedNetmailTotal($query)
-	{
-		return $query
-			->select([
-				'addresses.id',
-				'zone_id',
-				'host_id',
-				'node_id',
-				'point_id',
-				'system_id',
-				DB::raw('0 as uncollected_echomail'),
-				DB::raw('count(addresses.id) as uncollected_netmail'),
-				DB::raw('0 as uncollected_files')
-			])
-			->UncollectedNetmail();
-	}
-
 	/* RELATIONS */
 
 	public function domain()
@@ -1077,55 +968,55 @@ class Address extends Model
 	}
 
 	/**
-	 * Echomail waiting to be sent to this system
+	 * Echomail waiting to be sent to this address
 	 *
-	 * @return Builder
+	 * @return Collection
 	 */
-	public function echomailWaiting(): Builder
+	public function echomailWaiting(): Collection
 	{
-		return Echomail::select('echomails.*')
+		return Echomail::Uncollected()
-			->join('echomail_seenby',['echomail_seenby.echomail_id'=>'echomails.id'])
 			->where('address_id',$this->id)
-			->whereNull('echomails.deleted_at')
-			->whereNotNull('export_at')
-			->whereNull('sent_at')
 			->orderby('id')
-			->with([
+			->get();
-				'tagline:id,value',
-				'tearline:id,value',
-				'origin:id,value',
-				'echoarea:id,name,domain_id',
-				'echoarea.domain:id,name',
-				'fftn:id,zone_id,region_id,host_id,node_id,point_id',
-				'fftn.zone:id,domain_id,zone_id',
-				'fftn.zone.domain:id,name',
-			]);
 	}
 
 	/**
-	 * Files waiting to be sent to this system
+	 * Count of echomail waiting to be sent to this address
+	 *
+	 * @return int
+	 */
+	public function echomailWaitingCount(): int
+	{
+		return Echomail::Uncollected()
+			->where('address_id',$this->id)
+			->count();
+	}
+
+	/**
+	 * Files waiting to be sent to this address
 	 *
 	 * @return Collection
 	 */
 	public function filesWaiting(): Collection
 	{
-		return File::select('files.*')
+		return File::Uncollected()
-			->join('file_seenby',['file_seenby.file_id'=>'files.id'])
 			->where('address_id',$this->id)
-			->whereNull('files.deleted_at')
-			->whereNotNull('export_at')
-			->whereNull('sent_at')
 			->orderby('id')
-			->with([
-				'filearea:id,name,domain_id',
-				'filearea.domain:id,name',
-				'fftn:id,zone_id,region_id,host_id,node_id,point_id',
-				'fftn.zone:id,domain_id,zone_id',
-				'fftn.zone.domain:id,name',
-			])
 			->get();
 	}
 
+	/**
+	 * Count of files waiting to be sent to this address
+	 *
+	 * @return int
+	 */
+	public function filesWaitingCount(): int
+	{
+		return File::Uncollected()
+			->where('address_id',$this->id)
+			->count();
+	}
+
 	/**
 	 * Work out what role this FTN should have
 	 *
@@ -1265,43 +1156,40 @@ class Address extends Model
 	 */
 	public function netmailWaiting(): Builder
 	{
-		// Addresses that our downstream of this address, except anybody that has session details with us
+		return Netmail::Uncollected()
-		$ours = our_nodes($this->zone->domain)->pluck('id');
+			->whereIn('tftn_id',$this->downlinks()
-
+				->add($this)
-		$addresses = $this->downlinks()
+				->pluck('id'));
-			->filter(fn($item)=>(! $ours->contains($item->id)))
-			->merge($this->system->match($this->zone,Address::NODE_ALL));
-
-		$netmails = $this
-			->UncollectedNetmail()
-			->select('netmails.id')
-			->whereIn('addresses.id',$addresses->pluck('id'))
-			->groupBy(['netmails.id'])
-			->get();
-
-		return Netmail::whereIn('id',$netmails->pluck('id'));
 	}
 
 	/**
-	 * Netmail alerts waiting to be sent to this system
+	 * Count of echomail waiting to be sent to this address
+	 *
+	 * @return int
+	 */
+	public function netmailWaitingCount(): int
+	{
+		return Netmail::Uncollected()
+			->whereIn('tftn_id',$this->downlinks()
+				->add($this)
+				->pluck('id'))
+			->count();
+	}
+
+	/**
+	 * Netmail alerts waiting to be sent to this address
 	 *
 	 * @return Builder
 	 * @throws \Exception
 	 * @note The packet password to use is on the subject line for these alerts
+	 * @todo To Test
 	 */
 	public function netmailAlertWaiting(): Builder
 	{
-		$netmails = $this
+		return Netmail::UncollectedAlerts()
-			->UncollectedNetmail()
+			->whereIn('tftn_id',$this->downlinks()
-			->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL))
+				->add($this)
-			->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_PKTPASSWD))
+				->pluck('id'));
-			->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT))
-			->select('netmails.id')
-			->whereIn('addresses.id',$this->downlinks()->add($this)->pluck('id'))
-			->groupBy(['netmails.id'])
-			->get();
-
-		return Netmail::whereIn('id',$netmails->pluck('id'));
 	}
 
 	public function nodes(): Collection

app/Models/Echomail.php

@@ -3,6 +3,7 @@
 namespace App\Models;
 
 use Carbon\Carbon;
+use Illuminate\Database\Eloquent\Builder;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Database\Eloquent\SoftDeletes;
 use Illuminate\Support\Collection;
@@ -279,6 +280,33 @@ final class Echomail extends Model implements Packet
 		});
 	}
 
+	/* SCOPES */
+
+	/**
+	 * Base query to find uncollected echomails
+	 *
+	 * @param Builder $query
+	 * @return Builder
+	 */
+	public function scopeUncollected(Builder $query): Builder
+	{
+		return $this
+			->join('echomail_seenby',['echomail_seenby.echomail_id'=>'echomails.id'])
+			->whereNull('echomails.deleted_at')
+			->whereNotNull('export_at')
+			->whereNull('sent_at')
+			->with([
+				'tagline:id,value',
+				'tearline:id,value',
+				'origin:id,value',
+				'echoarea:id,name,domain_id',
+				'echoarea.domain:id,name',
+				'fftn:id,zone_id,region_id,host_id,node_id,point_id',
+				'fftn.zone:id,domain_id,zone_id',
+				'fftn.zone.domain:id,name',
+			]);
+	}
+
 	/* RELATIONS */
 
 	public function echoarea()

app/Models/File.php

@@ -3,6 +3,7 @@
 namespace App\Models;
 
 use Carbon\Carbon;
+use Illuminate\Database\Eloquent\Builder;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Database\Eloquent\SoftDeletes;
 use Illuminate\Support\Arr;
@@ -174,6 +175,30 @@ class File extends Model
 		});
 	}
 
+	/* SCOPES */
+
+	/**
+	 * Base query to find uncollected files
+	 *
+	 * @param Builder $query
+	 * @return Builder
+	 */
+	public function scopeUncollected(Builder $query): Builder
+	{
+		return $query
+			->join('file_seenby',['file_seenby.file_id'=>'files.id'])
+			->whereNotNull('export_at')
+			->whereNull('sent_at')
+			->whereNull('files.deleted_at')
+			->with([
+				'filearea:id,name,domain_id',
+				'filearea.domain:id,name',
+				'fftn:id,zone_id,region_id,host_id,node_id,point_id',
+				'fftn.zone:id,domain_id,zone_id',
+				'fftn.zone.domain:id,name',
+			]);
+	}
+
 	/* RELATIONS */
 
 	public function filearea()

app/Models/Netmail.php

@@ -3,6 +3,7 @@
 namespace App\Models;
 
 use Carbon\Carbon;
+use Illuminate\Database\Eloquent\Builder;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Database\Eloquent\SoftDeletes;
 use Illuminate\Support\Arr;
@@ -10,6 +11,7 @@ use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
 
+use App\Classes\FTN\Message;
 use App\Interfaces\Packet;
 use App\Models\Casts\{CompressedStringOrNull,CollectionOrNull,UTF8StringOrNull};
 use App\Models\Pivots\ViaPivot;
@@ -237,6 +239,37 @@ final class Netmail extends Model implements Packet
 		});
 	}
 
+	/* SCOPES */
+
+	/**
+	 * Base query to find uncollected netmails
+	 *
+	 * @param Builder $query
+	 * @return Builder
+	 */
+	public function scopeUncollected(Builder $query): Builder
+	{
+		return $query
+			->where(fn($query)=>
+				$query->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_INTRANSIT))
+					->orWhereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL)))
+			->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT))
+			->whereNull('sent_pkt')
+			->whereNull('sent_at')
+			->whereNull('netmails.deleted_at');
+	}
+
+	public function scopeUncollectedAlerts(Builder $query): Builder
+	{
+		return $query
+			->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL))
+			->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_PKTPASSWD))
+			->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT))
+			->whereNull('sent_pkt')
+			->whereNull('sent_at')
+			->whereNull('netmails.deleted_at');
+	}
+
 	/* RELATIONS */
 
 	public function path()

app/Traits/HubStats.php

@@ -0,0 +1,91 @@
+<?php
+
+namespace App\Traits;
+
+use Carbon\Carbon;
+use Illuminate\Database\Eloquent\Builder;
+use Illuminate\Support\Facades\Auth;
+use Illuminate\Support\Facades\DB;
+
+use App\Models\{Address,Echomail,File,Netmail};
+
+trait HubStats
+{
+	private function HubStats(Carbon $asat): Builder
+	{
+		return Address::select([
+			'a.id',
+			'a.system_id',
+			'a.zone_id',
+			'addresses.region_id',
+			'a.host_id',
+			'a.node_id',
+			'a.point_id',
+			'addresses.hub_id',
+			'addresses.role',
+			DB::raw('sum(a.uncollected_echomail) as uncollected_echomail'),
+			DB::raw('sum(a.uncollected_netmail) as uncollected_netmail'),
+			DB::raw('sum(a.uncollected_files) as uncollected_files')
+		])
+			->from(
+				Echomail::Uncollected()
+					->select([
+						'addresses.id',
+						'zone_id',
+						'host_id',
+						'node_id',
+						'point_id',
+						'system_id',
+						DB::raw('count(addresses.id) as uncollected_echomail'),
+						DB::raw('0 as uncollected_netmail'),
+						DB::raw('0 as uncollected_files'),
+					])
+					->join('addresses',['addresses.id'=>'echomail_seenby.address_id'])
+					->where('echomails.created_at','<',$asat)
+					->groupBy('addresses.id')
+					->union(Netmail::Uncollected()
+						->select([
+							'addresses.id',
+							'zone_id',
+							'host_id',
+							'node_id',
+							'point_id',
+							'system_id',
+							DB::raw('0 as uncollected_echomail'),
+							DB::raw('count(addresses.id) as uncollected_netmail'),
+							DB::raw('0 as uncollected_files')
+						])
+						->join('addresses',['addresses.id'=>'netmails.tftn_id'])
+						->where('netmails.created_at','<',$asat)
+						->groupBy('addresses.id')
+					)
+					->union(File::Uncollected()
+						->select([
+							'addresses.id',
+							'zone_id',
+							'host_id',
+							'node_id',
+							'point_id',
+							'system_id',
+							DB::raw('0 as uncollected_echomail'),
+							DB::raw('0 as uncollected_netmail'),
+							DB::raw('count(addresses.id) as uncollected_files')
+						])
+						->join('addresses',['addresses.id'=>'file_seenby.address_id'])
+						->where('files.created_at','<',$asat)
+						->groupBy('addresses.id')
+					),'a')
+			->where('systems.active',TRUE)
+			->where('addresses.active',TRUE)
+			->where('zones.active',TRUE)
+			->where('domains.active',TRUE)
+			->when(! ($x=Auth::user()) || (! $x->isAdmin()),fn($query)=>$query->where('domains.public',TRUE))
+			->join('addresses',['addresses.id'=>'a.id'])
+			->join('systems',['systems.id'=>'a.system_id'])
+			->join('zones',['zones.id'=>'addresses.zone_id'])
+			->join('domains',['domains.id'=>'zones.domain_id'])
+			->ftnOrder()
+			->groupBy('a.system_id','a.id','a.zone_id','addresses.region_id','a.host_id','a.node_id','a.point_id','addresses.hub_id','addresses.role')
+			->with(['system','zone.domain']);
+	}
+}