bbs/clrghouz
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Rework crash polling, using optimised scope queries

Browse Source
This commit is contained in:
Deon George 2023-11-25 21:52:05 +11:00
parent b5e5decfdf
commit 2b2482ba71
4 changed files with 141 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
app/Http/Controllers/HomeController.php
View File

@ -248,55 +248,31 @@ class HomeController extends Controller
{
$date = Carbon::now()->yesterday()->endOfday();
// Nodes with uncollected mail
$e = Address::select(['addresses.id',DB::raw('count(*) as e'),DB::raw('0 as n'),DB::raw('0 as f')])
->join('echomail_seenby',['echomail_seenby.address_id'=>'addresses.id'])
->join('echomails',['echomails.id'=>'echomail_seenby.echomail_id'])
->whereNotNull('export_at')
->whereNull('sent_at')
->whereNull('echomails.deleted_at')
->where('echomails.created_at','<',$date)
->groupBy('addresses.id');
$n = Address::select(['addresses.id',DB::raw('0 as e'),DB::raw('count(*) as n'),DB::raw('0 as f')])
->join('netmails',['netmails.tftn_id'=>'addresses.id'])
->where(function($query) {
return $query->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_INTRANSIT))
->orWhereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL));
})
->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT))
->whereNull('netmails.deleted_at')
->where('netmails.created_at','<',$date)
->groupBy('addresses.id');
$f = Address::select(['addresses.id',DB::raw('0 as e'),DB::raw('0 as n'),DB::raw('count(*) as f')])
->join('file_seenby',['file_seenby.address_id'=>'addresses.id'])
->join('files',['files.id'=>'file_seenby.file_id'])
->whereNotNull('export_at')
->whereNull('sent_at')
->whereNull('files.deleted_at')
->where('files.created_at','<',$date)
->groupBy('addresses.id');
$r = Address::select([
'a.id',
'addresses.system_id',
'addresses.zone_id',
'addresses.region_id',
'addresses.host_id',
'addresses.node_id',
DB::raw('sum(a.e) as e'),
DB::raw('sum(a.n) as n'),
DB::raw('sum(a.f) as f')
'addresses.point_id',
'addresses.hub_id',
'addresses.role',
DB::raw('sum(a.uncollected_echomail) as uncollected_echomail'),
DB::raw('sum(a.uncollected_netmail) as uncollected_netmail'),
DB::raw('sum(a.uncollected_files) as uncollected_files')
])
->from($e->union($n)->union($f),'a')
->from(Address::UncollectedEchomail()->union(Address::UncollectedNetmail())->union(Address::UncollectedFiles()),'a')
->where('systems.active',true)
->where('addresses.active',TRUE)
->where('zones.active',TRUE)
->where('domains.active',TRUE)
->when(! ($x=Auth::user()) || (! $x->isAdmin()),function($query) { return $query->where('domains.public',TRUE); })
->join('addresses',['addresses.id'=>'a.id'])
->join('systems',['systems.id'=>'addresses.system_id'])
->join('zones',['zones.id'=>'addresses.zone_id'])
->join('domains',['domains.id'=>'zones.domain_id'])
->groupBy('addresses.system_id','a.id','addresses.zone_id','addresses.host_id','addresses.node_id')
->groupBy('addresses.system_id','a.id','addresses.zone_id','addresses.region_id','addresses.host_id','addresses.node_id','addresses.point_id','addresses.hub_id','addresses.role')
->with(['system','zone.domain']);
return view('status')

147
app/Jobs/MailSend.php
View File

@ -12,7 +12,6 @@ use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Repat\LaravelJobs\Job;
use App\Classes\FTN\Message;
use App\Models\Address;
class MailSend implements ShouldQueue
@ -47,24 +46,33 @@ class MailSend implements ShouldQueue
*/
public function handle(): void
{
// Netmail waiting by node (only netmail is routed)
$netmails = Address::select(['addresses.id','addresses.zone_id','region_id','host_id','node_id','point_id','role','addresses.system_id',DB::raw('count(netmails.id) AS nm')])
->join('zones',['zones.id'=>'addresses.zone_id'])
->join('domains',['domains.id'=>'zones.domain_id'])
->join('netmails',['netmails.tftn_id'=>'addresses.id'])
->join('systems',['systems.id'=>'addresses.system_id'])
$u = Address::select([
'a.id',
'addresses.system_id',
'addresses.zone_id',
'addresses.region_id',
'addresses.host_id',
'addresses.node_id',
'addresses.point_id',
'addresses.hub_id',
'addresses.role',
DB::raw('sum(a.uncollected_echomail) as uncollected_echomail'),
DB::raw('sum(a.uncollected_netmail) as uncollected_netmail'),
DB::raw('sum(a.uncollected_files) as uncollected_files')
])
->from(Address::UncollectedEchomail()->union(Address::UncollectedNetmail())->union(Address::UncollectedFiles()),'a')
->where('systems.active',true)
->where('addresses.active',TRUE)
->where('zones.active',TRUE)
->where('domains.active',TRUE)
->join('addresses',['addresses.id'=>'a.id'])
->join('systems',['systems.id'=>'addresses.system_id'])
->join('zones',['zones.id'=>'addresses.zone_id'])
->join('domains',['domains.id'=>'zones.domain_id'])
->where(function($query) {
return $query->whereNull('autohold')
->orWhere('autohold',FALSE);
})
->where(function($query) {
return $query->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_INTRANSIT))
->orWhereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL));
})
->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT))
->when(! is_null($this->crash),function($query) {
return $query->when(
$this->crash,
@ -76,115 +84,40 @@ class MailSend implements ShouldQueue
}
);
})
->groupBy('addresses.id')
->havingRaw('count(*) > 0')
->groupBy('addresses.system_id','a.id','addresses.zone_id','addresses.region_id','addresses.host_id','addresses.node_id','addresses.point_id','addresses.hub_id','addresses.role')
->with(['system','zone.domain'])
->get();
// Echomail waiting by node
$echomails = Address::select(['addresses.id','addresses.zone_id','region_id','host_id','node_id','point_id','role','addresses.system_id',DB::raw('count(*) AS em')])
->distinct()
->join('zones',['zones.id'=>'addresses.zone_id'])
->join('domains',['domains.id'=>'zones.domain_id'])
->join('echomail_seenby',['echomail_seenby.address_id'=>'addresses.id'])
->join('systems',['systems.id'=>'addresses.system_id'])
->where('addresses.active',TRUE)
->where('zones.active',TRUE)
->where('domains.active',TRUE)
->where(function($query) {
return $query->whereNull('autohold')
->orWhere('autohold',FALSE);
})
->whereNotNull('export_at')
->whereNull('sent_at')
->when(! is_null($this->crash),function($query) {
return $query->when(
$this->crash,
function($query) {
return $query->where('pollmode',$this->crash);
},
function($query) {
return $query->whereNotNull('pollmode');
}
);
})
->groupBy(['addresses.id'])
->havingRaw('count(*) > 0')
->FTNOrder()
->get();
// Return the system we poll
$u = $u->transform(function($item) {
if ($x=$item->parent()) {
$x->uncollected_echomail = $item->uncollected_echomail;
$x->uncollected_netmail = $item->uncollected_netmail;
$x->uncollected_files = $item->uncollected_files;
// Files waiting by node
$files = Address::select(['addresses.id','addresses.zone_id','region_id','host_id','node_id','point_id','role','addresses.system_id',DB::raw('count(*) AS fs')])
->distinct()
->join('zones',['zones.id'=>'addresses.zone_id'])
->join('domains',['domains.id'=>'zones.domain_id'])
->join('file_seenby',['file_seenby.address_id'=>'addresses.id'])
->join('systems',['systems.id'=>'addresses.system_id'])
->where('addresses.active',TRUE)
->where('zones.active',TRUE)
->where('domains.active',TRUE)
->where(function($query) {
return $query->whereNull('autohold')
->orWhere('autohold',FALSE);
})
->whereNotNull('export_at')
->whereNull('sent_at')
->when(! is_null($this->crash),function($query) {
return $query->when(
$this->crash,
function($query) {
return $query->where('pollmode',$this->crash);
},
function($query) {
return $query->whereNotNull('pollmode');
}
);
})
->groupBy(['addresses.id'])
->havingRaw('count(*) > 0')
->FTNOrder()
->get();
return $x;
// Merge our netmails
foreach ($echomails as $ao) {
if (($x=$netmails->search(function($item) use ($ao) { return $item->id === $ao->id; })) !== FALSE) {
$netmails->get($x)->em = $ao->em;
} else {
$netmails->push($ao);
}
return $item;
}
});
// Merge our files
foreach ($files as $ao) {
if (($x=$netmails->search(function($item) use ($ao) { return $item->id === $ao->id; })) !== FALSE) {
$netmails->get($x)->fs = $ao->fs;
} else {
$netmails->push($ao);
}
}
// Remove direct links
$netmails = $netmails->filter(function($item) { return $item->parent(); });
foreach ($netmails->groupBy(function($item) { return $item->parent()->ftn; }) as $oo) {
$ao = $oo->first();
foreach ($u->groupBy('ftn') as $oo) {
if (Job::where('queue','poll')->get()->pluck('command.address.id')->search(($x=$oo->first())->id) === FALSE) {
Log::info(sprintf('%s:- Polling [%s] - we have mail for [%d] links. (%d Netmail,%d Echomail,%d Files)',
self::LOGKEY,
$ao->ftn,
$x->ftn,
$oo->count(),
$oo->sum('nm'),
$oo->sum('em'),
$oo->sum('fs'),
$oo->sum('uncollected_netmail'),
$oo->sum('uncollected_echomail'),
$oo->sum('uncollected_files'),
));
// @todo Only send crash mail - send normal mail with a schedule or hold mail
AddressPoll::dispatch($x);
if (Job::where('queue',$this->queue)->get()->pluck('command.address.id')->search($ao->id) !== FALSE) {
Log::alert(sprintf('%s:= Not scheduling poll to [%s], there is already one in the queue',self::LOGKEY,$ao->ftn));
continue;
} else {
Log::notice(sprintf('%s:= Not scheduling poll to [%s], there is already one in the queue',self::LOGKEY,$x->ftn));
}
AddressPoll::dispatch($ao);
}
}
}

87
app/Models/Address.php
View File

@ -2,6 +2,7 @@
namespace App\Models;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\SoftDeletes;
@ -23,7 +24,7 @@ class Address extends Model
protected $with = ['zone'];
// http://ftsc.org/docs/frl-1028.002
public const ftn_regex = '(\d+):(\d+)/(\d+)(?:\.(\d+))?(?:@([a-z0-9\-_~]{0,8}))?';
public const ftn_regex = '(\d+):(\d+)/(\d+)(?:\.(\d+))?(?:@([a-zA-Z0-9\-_~]{0,8}))?';
public const NODE_ZC = 1<<0; // Zone
public const NODE_RC = 1<<1; // Region
@ -65,13 +66,6 @@ class Address extends Model
->FTNorder();
}
public function scopeTrashed($query)
{
return $query->select($this->getTable().'.*')
->withTrashed()
->FTNorder();
}
public function scopeFTNOrder($query)
{
return $query
@ -89,6 +83,72 @@ class Address extends Model
->orderBy('point_id');
}
public function scopeTrashed($query)
{
return $query->select($this->getTable().'.*')
->withTrashed()
->FTNorder();
}
/**
* Return a list of addresses and the amount of uncollected echomail
*
* @param $query
* @return mixed
*/
public function scopeUncollectedEchomail($query)
{
return $query
->select(['addresses.id',DB::raw('count(*) as uncollected_echomail'),DB::raw('0 as uncollected_netmail'),DB::raw('0 as uncollected_files')])
->join('echomail_seenby',['echomail_seenby.address_id'=>'addresses.id'])
->join('echomails',['echomails.id'=>'echomail_seenby.echomail_id'])
->whereNotNull('export_at')
->whereNull('sent_at')
->whereNull('echomails.deleted_at')
->where('echomails.created_at','<',$this->yesterdayEOD())
->groupBy('addresses.id');
}
/**
* Return a list of addresses and the amount of uncollected files
*
* @param $query
* @return mixed
*/
public function scopeUncollectedFiles($query)
{
return $query
->select(['addresses.id',DB::raw('0 as uncollected_echomail'),DB::raw('0 as uncollected_netmail'),DB::raw('count(*) as uncollected_files')])
->join('file_seenby',['file_seenby.address_id'=>'addresses.id'])
->join('files',['files.id'=>'file_seenby.file_id'])
->whereNotNull('export_at')
->whereNull('sent_at')
->whereNull('files.deleted_at')
->where('files.created_at','<',$this->yesterdayEOD())
->groupBy('addresses.id');
}
/**
* Return a list of addresses and the amount of uncollected netmail
*
* @param $query
* @return mixed
*/
public function scopeUncollectedNetmail($query)
{
return $query
->select(['addresses.id',DB::raw('0 as uncollected_echomail'),DB::raw('count(*) as uncollected_netmail'),DB::raw('0 as uncollected_files')])
->join('netmails',['netmails.tftn_id'=>'addresses.id'])
->where(function($query) {
return $query->whereRaw(sprintf('(flags & %d) > 0',Message::FLAG_INTRANSIT))
->orWhereRaw(sprintf('(flags & %d) > 0',Message::FLAG_LOCAL));
})
->whereRaw(sprintf('(flags & %d) = 0',Message::FLAG_SENT))
->whereNull('netmails.deleted_at')
->where('netmails.created_at','<',$this->yesterdayEOD())
->groupBy('addresses.id');
}
/* RELATIONS */
/**
@ -285,13 +345,13 @@ class Address extends Model
// Normal Nodes
case self::NODE_ACTIVE:
// If we are a child of a hub, then check our hub
$parent = (($this->hub_id)
$parent = ($this->hub_id
? self::where('id',$this->hub_id)
: self::where('zone_id',$this->zone_id)
->where('region_id',$this->region_id)
->where('host_id',$this->host_id)
->where('node_id',0))
->where('role','<',$this->role)
->where('node_id',0)
->where('role','<',$this->role))
->single();
break;
@ -865,4 +925,9 @@ class Address extends Model
{
return ($this->exists && ($x=$this->system->sessions->where('id',$this->zone_id)->first())) ? ($x->pivot->{$type} ?: '') : NULL;
}
private function yesterdayEOD(): Carbon
{
return Carbon::now()->yesterday()->endOfday();
}
}

13
resources/views/status.blade.php
View File

@ -33,11 +33,16 @@
@foreach($uncollected as $o)
<tr>
<td>{{ $o->zone->domain->name }}</td>
<td><a href="{{ url('system/addedit',$o->system_id) }}">{{ $o->system->name }}</a></td>
<td>
<a href="{{ url('system/addedit',$o->system_id) }}">{{ $o->system->name }}</a>
@if (($x=$o->parent()) && ($x->id !== $o->id))
<br><small>[via <a href="{{ url('system/addedit',$x->system_id) }}">{{ $x->ftn4d }}</a>]</small>
@endif
</td>
<td>{{ $o->ftn4d }}</td>
<td class="text-right">{{ number_format($o->e ?? 0) }}</td>
<td class="text-right">{{ number_format($o->n ?? 0) }}</td>
<td class="text-right">{{ number_format($o->f ?? 0) }}</td>
<td class="text-right">{{ number_format($o->uncollected_echomail ?? 0) }}</td>
<td class="text-right">{{ number_format($o->uncollected_netmail ?? 0) }}</td>
<td class="text-right">{{ number_format($o->uncollected_files ?? 0) }}</td>
<td>{{ $o->system->last_session?->format('Y-m-d H:i') }}</td>
<td>{{ is_null($o->system->pollmode) ? 'HOLD' : ($o->system->pollmode ? 'CRASH' : 'DAILY') }}</td>
<td>{{ $o->system->autohold ? 'YES' : 'NO' }}</td>