bbs/clrghouz
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Optimize the SQL queries that finds unsent echomail,netmail and files

Browse Source
This commit is contained in:
Deon George 2025-04-16 23:36:55 +10:00
parent 128e333deb
commit 3f0e17e20b
8 changed files with 257 additions and 284 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
app/Classes/Dynamic/HubStats.php
View File

@ -9,6 +9,7 @@ use Illuminate\Support\Facades\Log;
use App\Classes\Dynamic;
use App\Models\Address;
use App\Traits\HubStats as HubStatsTrait;
/**
* This method will generate the hub status for an upstream Host/RC/ZC
@ -18,6 +19,8 @@ use App\Models\Address;
*/
class HubStats extends Dynamic
{
use HubStatsTrait;
private const LOGKEY = 'DHS';
private string $name = '';
@ -32,41 +35,8 @@ class HubStats extends Dynamic
{
$date = Carbon::now()->yesterday()->endOfday();
$r = Address::select([
'a.id',
'addresses.system_id',
'addresses.zone_id',
'addresses.region_id',
'addresses.host_id',
'addresses.node_id',
'addresses.point_id',
'addresses.hub_id',
'addresses.role',
DB::raw('sum(a.uncollected_echomail) as uncollected_echomail'),
DB::raw('sum(a.uncollected_netmail) as uncollected_netmail'),
DB::raw('sum(a.uncollected_files) as uncollected_files')
])
->from(
Address::UncollectedEchomailTotal()
->where('echomails.created_at','<',$date)
->union(Address::UncollectedNetmailTotal()
->where('netmails.created_at','<',$date)
)
->union(Address::UncollectedFilesTotal()
->where('files.created_at','<',$date)
),'a')
->where('systems.active',true)
->where('addresses.active',TRUE)
->where('zones.active',TRUE)
->where('domains.active',TRUE)
->where('zones.id',$this->ao->zone_id)
->join('addresses',['addresses.id'=>'a.id'])
->join('systems',['systems.id'=>'addresses.system_id'])
->join('zones',['zones.id'=>'addresses.zone_id'])
->join('domains',['domains.id'=>'zones.domain_id'])
->ftnOrder()
->groupBy('addresses.system_id','a.id','addresses.zone_id','addresses.region_id','addresses.host_id','addresses.node_id','addresses.point_id','addresses.hub_id','addresses.role')
->with(['system','zone.domain']);
$r = $this->HubStats($date)
->where('zones.id',$this->ao->zone_id);
$header = "| %-12s | %4d | %3d | %3d | %16s | %5s | %5s |\r\n";

46
app/Http/Controllers/HomeController.php
View File

@ -14,9 +14,12 @@ use App\Classes\FTN\{Message,Packet};
use App\Http\Requests\SetupRequest;
use App\Models\File as FileModel;
use App\Models\{Address,Echomail,Netmail,Setup,System};
use App\Traits\HubStats;
class HomeController extends Controller
{
use HubStats;
public function home()
{
return redirect(Auth::check() ? 'dashboard' : 'about');
@ -246,49 +249,8 @@ class HomeController extends Controller
{
$date = Carbon::now()->yesterday()->endOfday();
$r = Address::select([
'a.id',
'a.system_id',
'a.zone_id',
'addresses.region_id',
'a.host_id',
'a.node_id',
'a.point_id',
'addresses.hub_id',
'addresses.role',
DB::raw('sum(a.uncollected_echomail) as uncollected_echomail'),
DB::raw('sum(a.uncollected_netmail) as uncollected_netmail'),
DB::raw('sum(a.uncollected_files) as uncollected_files')
])
->from(
Address::UncollectedEchomailTotal()
->where('echomails.created_at','<',$this->yesterdayEOD())
->union(Address::UncollectedNetmailTotal()
->where('netmails.created_at','<',$this->yesterdayEOD())
)
->union(Address::UncollectedFilesTotal()
->where('files.created_at','<',$this->yesterdayEOD())
),'a')
->where('systems.active',TRUE)
->where('addresses.active',TRUE)
->where('zones.active',TRUE)
->where('domains.active',TRUE)
->when(! ($x=Auth::user()) || (! $x->isAdmin()),function($query) { return $query->where('domains.public',TRUE); })
->join('addresses',['addresses.id'=>'a.id'])
->join('systems',['systems.id'=>'a.system_id'])
->join('zones',['zones.id'=>'addresses.zone_id'])
->join('domains',['domains.id'=>'zones.domain_id'])
->ftnOrder()
->groupBy('a.system_id','a.id','a.zone_id','addresses.region_id','a.host_id','a.node_id','a.point_id','addresses.hub_id','addresses.role')
->with(['system','zone.domain']);
return view('status')
->with('date',$date)
->with('uncollected',$r->get());
}
private function yesterdayEOD(): Carbon
{
return Carbon::now()->yesterday()->endOfday();
->with('uncollected',$this->HubStats($date)->get());
}
}

56
app/Jobs/MailSend.php
View File

@ -2,16 +2,16 @@
namespace App\Jobs;
use Carbon\Carbon;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Repat\LaravelJobs\Job;
use App\Models\Address;
use App\Traits\HubStats;
class MailSend #implements ShouldQueue
{
use Dispatchable;
use HubStats,Dispatchable;
private const LOGKEY = 'JMS';
@ -25,46 +25,22 @@ class MailSend #implements ShouldQueue
*/
public function handle(): void
{
$u = Address::select([
'a.id',
'a.system_id',
'a.zone_id',
'addresses.region_id',
'a.host_id',
'a.node_id',
'a.point_id',
'addresses.hub_id',
'addresses.role',
DB::raw('sum(a.uncollected_echomail) as uncollected_echomail'),
DB::raw('sum(a.uncollected_netmail) as uncollected_netmail'),
DB::raw('sum(a.uncollected_files) as uncollected_files')
])
->from(Address::UncollectedEchomailTotal()->union(Address::UncollectedNetmailTotal())->union(Address::UncollectedFilesTotal()),'a')
->where('systems.active',true)
->where('addresses.active',TRUE)
->where('zones.active',TRUE)
->where('domains.active',TRUE)
->join('addresses',['addresses.id'=>'a.id'])
->join('systems',['systems.id'=>'a.system_id'])
->join('zones',['zones.id'=>'a.zone_id'])
->join('domains',['domains.id'=>'zones.domain_id'])
->groupBy('a.system_id','a.id','a.zone_id','addresses.region_id','a.host_id','a.node_id','a.point_id','addresses.hub_id','addresses.role')
->with(['system','zone.domain'])
->get();
// Return the system we poll
$u = $u->transform(function($item) {
if ($x=$item->uplink()) {
$x->uncollected_echomail = $item->uncollected_echomail;
$x->uncollected_netmail = $item->uncollected_netmail;
$x->uncollected_files = $item->uncollected_files;
$u = $this
->HubStats(Carbon::now())
->get()
->transform(function($item) {
if ($x=$item->uplink()) {
$x->uncollected_echomail = $item->uncollected_echomail;
$x->uncollected_netmail = $item->uncollected_netmail;
$x->uncollected_files = $item->uncollected_files;
return $x;
return $x;
} else {
return $item;
}
})
} else {
return $item;
}
})
->filter(function($item) {
if ($item->system->autohold)
return NULL;

222
app/Models/Address.php
View File

@ -450,115 +450,6 @@ class Address extends Model
->orderBy('point_id');
}
/**
* Return a list of addresses and the amount of uncollected echomail
*
* @param $query
* @return mixed
*/
public function scopeUncollectedEchomail($query)
{
return $query
->join('echomail_seenby',['echomail_seenby.address_id'=>'addresses.id'])
->join('echomails',['echomails.id'=>'echomail_seenby.echomail_id'])
->whereNotNull('export_at')
->whereNull('sent_at')
->whereNull('echomails.deleted_at')
->groupBy('addresses.id')
->dontCache();
}
public function scopeUncollectedEchomailTotal($query)
{
return $query
->select([
'addresses.id',
'zone_id',
'host_id',
'node_id',
'point_id',
'system_id',
DB::raw('count(addresses.id) as uncollected_echomail'),
DB::raw('0 as uncollected_netmail'),
DB::raw('0 as uncollected_files'),
])
->UncollectedEchomail();
}
/**
* Return a list of addresses and the amount of uncollected files
*
* @param $query
* @return mixed
*/
public function scopeUncollectedFiles($query)
{
return $query
->join('file_seenby',['file_seenby.address_id'=>'addresses.id'])
->join('files',['files.id'=>'file_seenby.file_id'])
->whereNotNull('export_at')
->whereNull('sent_at')
->whereNull('files.deleted_at')
->groupBy('addresses.id')
->dontCache();
}
public function scopeUncollectedFilesTotal($query)
{
return $query
->select([
'addresses.id',
'zone_id',
'host_id',
'node_id',
'point_id',
'system_id',
DB::raw('0 as uncollected_echomail'),
DB::raw('0 as uncollected_netmail'),
DB::raw('count(addresses.id) as uncollected_files')
])
->UncollectedFiles();
}
public function scopeUncollectedNetmail($query)
{
return $query
->join('netmails',['netmails.tftn_id'=>'addresses.id'])
->where(function($query) {
return $query->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_INTRANSIT))
->orWhereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL));
})
->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT))
->whereNull('sent_pkt')
->whereNull('sent_at')
->whereNull('netmails.deleted_at')
->groupBy('addresses.id')
->dontCache();
}
/**
* Return a list of addresses and the amount of uncollected netmail
*
* @param $query
* @return mixed
*/
public function scopeUncollectedNetmailTotal($query)
{
return $query
->select([
'addresses.id',
'zone_id',
'host_id',
'node_id',
'point_id',
'system_id',
DB::raw('0 as uncollected_echomail'),
DB::raw('count(addresses.id) as uncollected_netmail'),
DB::raw('0 as uncollected_files')
])
->UncollectedNetmail();
}
/* RELATIONS */
public function domain()
@ -1077,55 +968,55 @@ class Address extends Model
}
/**
* Echomail waiting to be sent to this system
* Echomail waiting to be sent to this address
*
* @return Builder
* @return Collection
*/
public function echomailWaiting(): Builder
public function echomailWaiting(): Collection
{
return Echomail::select('echomails.*')
->join('echomail_seenby',['echomail_seenby.echomail_id'=>'echomails.id'])
return Echomail::Uncollected()
->where('address_id',$this->id)
->whereNull('echomails.deleted_at')
->whereNotNull('export_at')
->whereNull('sent_at')
->orderby('id')
->with([
'tagline:id,value',
'tearline:id,value',
'origin:id,value',
'echoarea:id,name,domain_id',
'echoarea.domain:id,name',
'fftn:id,zone_id,region_id,host_id,node_id,point_id',
'fftn.zone:id,domain_id,zone_id',
'fftn.zone.domain:id,name',
]);
->get();
}
/**
* Files waiting to be sent to this system
* Count of echomail waiting to be sent to this address
*
* @return int
*/
public function echomailWaitingCount(): int
{
return Echomail::Uncollected()
->where('address_id',$this->id)
->count();
}
/**
* Files waiting to be sent to this address
*
* @return Collection
*/
public function filesWaiting(): Collection
{
return File::select('files.*')
->join('file_seenby',['file_seenby.file_id'=>'files.id'])
return File::Uncollected()
->where('address_id',$this->id)
->whereNull('files.deleted_at')
->whereNotNull('export_at')
->whereNull('sent_at')
->orderby('id')
->with([
'filearea:id,name,domain_id',
'filearea.domain:id,name',
'fftn:id,zone_id,region_id,host_id,node_id,point_id',
'fftn.zone:id,domain_id,zone_id',
'fftn.zone.domain:id,name',
])
->get();
}
/**
* Count of files waiting to be sent to this address
*
* @return int
*/
public function filesWaitingCount(): int
{
return File::Uncollected()
->where('address_id',$this->id)
->count();
}
/**
* Work out what role this FTN should have
*
@ -1265,43 +1156,40 @@ class Address extends Model
*/
public function netmailWaiting(): Builder
{
// Addresses that our downstream of this address, except anybody that has session details with us
$ours = our_nodes($this->zone->domain)->pluck('id');
$addresses = $this->downlinks()
->filter(fn($item)=>(! $ours->contains($item->id)))
->merge($this->system->match($this->zone,Address::NODE_ALL));
$netmails = $this
->UncollectedNetmail()
->select('netmails.id')
->whereIn('addresses.id',$addresses->pluck('id'))
->groupBy(['netmails.id'])
->get();
return Netmail::whereIn('id',$netmails->pluck('id'));
return Netmail::Uncollected()
->whereIn('tftn_id',$this->downlinks()
->add($this)
->pluck('id'));
}
/**
* Netmail alerts waiting to be sent to this system
* Count of echomail waiting to be sent to this address
*
* @return int
*/
public function netmailWaitingCount(): int
{
return Netmail::Uncollected()
->whereIn('tftn_id',$this->downlinks()
->add($this)
->pluck('id'))
->count();
}
/**
* Netmail alerts waiting to be sent to this address
*
* @return Builder
* @throws \Exception
* @note The packet password to use is on the subject line for these alerts
* @todo To Test
*/
public function netmailAlertWaiting(): Builder
{
$netmails = $this
->UncollectedNetmail()
->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL))
->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_PKTPASSWD))
->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT))
->select('netmails.id')
->whereIn('addresses.id',$this->downlinks()->add($this)->pluck('id'))
->groupBy(['netmails.id'])
->get();
return Netmail::whereIn('id',$netmails->pluck('id'));
return Netmail::UncollectedAlerts()
->whereIn('tftn_id',$this->downlinks()
->add($this)
->pluck('id'));
}
public function nodes(): Collection

28
app/Models/Echomail.php
View File

@ -3,6 +3,7 @@
namespace App\Models;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Collection;
@ -279,6 +280,33 @@ final class Echomail extends Model implements Packet
});
}
/* SCOPES */
/**
* Base query to find uncollected echomails
*
* @param Builder $query
* @return Builder
*/
public function scopeUncollected(Builder $query): Builder
{
return $this
->join('echomail_seenby',['echomail_seenby.echomail_id'=>'echomails.id'])
->whereNull('echomails.deleted_at')
->whereNotNull('export_at')
->whereNull('sent_at')
->with([
'tagline:id,value',
'tearline:id,value',
'origin:id,value',
'echoarea:id,name,domain_id',
'echoarea.domain:id,name',
'fftn:id,zone_id,region_id,host_id,node_id,point_id',
'fftn.zone:id,domain_id,zone_id',
'fftn.zone.domain:id,name',
]);
}
/* RELATIONS */
public function echoarea()

25
app/Models/File.php
View File

@ -3,6 +3,7 @@
namespace App\Models;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Arr;
@ -174,6 +175,30 @@ class File extends Model
});
}
/* SCOPES */
/**
* Base query to find uncollected files
*
* @param Builder $query
* @return Builder
*/
public function scopeUncollected(Builder $query): Builder
{
return $query
->join('file_seenby',['file_seenby.file_id'=>'files.id'])
->whereNotNull('export_at')
->whereNull('sent_at')
->whereNull('files.deleted_at')
->with([
'filearea:id,name,domain_id',
'filearea.domain:id,name',
'fftn:id,zone_id,region_id,host_id,node_id,point_id',
'fftn.zone:id,domain_id,zone_id',
'fftn.zone.domain:id,name',
]);
}
/* RELATIONS */
public function filearea()

33
app/Models/Netmail.php
View File

@ -3,6 +3,7 @@
namespace App\Models;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Arr;
@ -10,6 +11,7 @@ use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use App\Classes\FTN\Message;
use App\Interfaces\Packet;
use App\Models\Casts\{CompressedStringOrNull,CollectionOrNull,UTF8StringOrNull};
use App\Models\Pivots\ViaPivot;
@ -237,6 +239,37 @@ final class Netmail extends Model implements Packet
});
}
/* SCOPES */
/**
* Base query to find uncollected netmails
*
* @param Builder $query
* @return Builder
*/
public function scopeUncollected(Builder $query): Builder
{
return $query
->where(fn($query)=>
$query->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_INTRANSIT))
->orWhereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL)))
->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT))
->whereNull('sent_pkt')
->whereNull('sent_at')
->whereNull('netmails.deleted_at');
}
public function scopeUncollectedAlerts(Builder $query): Builder
{
return $query
->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL))
->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_PKTPASSWD))
->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT))
->whereNull('sent_pkt')
->whereNull('sent_at')
->whereNull('netmails.deleted_at');
}
/* RELATIONS */
public function path()

91
app/Traits/HubStats.php Normal file
View File

@ -0,0 +1,91 @@
<?php
namespace App\Traits;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Support\Facades\Auth;
use Illuminate\Support\Facades\DB;
use App\Models\{Address,Echomail,File,Netmail};
trait HubStats
{
private function HubStats(Carbon $asat): Builder
{
return Address::select([
'a.id',
'a.system_id',
'a.zone_id',
'addresses.region_id',
'a.host_id',
'a.node_id',
'a.point_id',
'addresses.hub_id',
'addresses.role',
DB::raw('sum(a.uncollected_echomail) as uncollected_echomail'),
DB::raw('sum(a.uncollected_netmail) as uncollected_netmail'),
DB::raw('sum(a.uncollected_files) as uncollected_files')
])
->from(
Echomail::Uncollected()
->select([
'addresses.id',
'zone_id',
'host_id',
'node_id',
'point_id',
'system_id',
DB::raw('count(addresses.id) as uncollected_echomail'),
DB::raw('0 as uncollected_netmail'),
DB::raw('0 as uncollected_files'),
])
->join('addresses',['addresses.id'=>'echomail_seenby.address_id'])
->where('echomails.created_at','<',$asat)
->groupBy('addresses.id')
->union(Netmail::Uncollected()
->select([
'addresses.id',
'zone_id',
'host_id',
'node_id',
'point_id',
'system_id',
DB::raw('0 as uncollected_echomail'),
DB::raw('count(addresses.id) as uncollected_netmail'),
DB::raw('0 as uncollected_files')
])
->join('addresses',['addresses.id'=>'netmails.tftn_id'])
->where('netmails.created_at','<',$asat)
->groupBy('addresses.id')
)
->union(File::Uncollected()
->select([
'addresses.id',
'zone_id',
'host_id',
'node_id',
'point_id',
'system_id',
DB::raw('0 as uncollected_echomail'),
DB::raw('0 as uncollected_netmail'),
DB::raw('count(addresses.id) as uncollected_files')
])
->join('addresses',['addresses.id'=>'file_seenby.address_id'])
->where('files.created_at','<',$asat)
->groupBy('addresses.id')
),'a')
->where('systems.active',TRUE)
->where('addresses.active',TRUE)
->where('zones.active',TRUE)
->where('domains.active',TRUE)
->when(! ($x=Auth::user()) || (! $x->isAdmin()),fn($query)=>$query->where('domains.public',TRUE))
->join('addresses',['addresses.id'=>'a.id'])
->join('systems',['systems.id'=>'a.system_id'])
->join('zones',['zones.id'=>'addresses.zone_id'])
->join('domains',['domains.id'=>'zones.domain_id'])
->ftnOrder()
->groupBy('a.system_id','a.id','a.zone_id','addresses.region_id','a.host_id','a.node_id','a.point_id','addresses.hub_id','addresses.role')
->with(['system','zone.domain']);
}
}