Implemented system heartbeat, to poll systems regularly that we havent heard from

2023-11-26 13:10:23 +11:00
parent 6e7e09ab50
commit 1ac3583479
8 changed files with 343 additions and 90 deletions
--- a/app/Jobs/SystemHeartbeat.php
+++ b/app/Jobs/SystemHeartbeat.php
@@ -0,0 +1,91 @@
+<?php
+
+namespace App\Jobs;
+
+use Carbon\Carbon;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Support\Facades\Log;
+use Repat\LaravelJobs\Job;
+
+use App\Models\{Address,Setup};
+
+class SystemHeartbeat #implements ShouldQueue
+{
+	use Dispatchable;
+
+	private const LOGKEY = 'JSH';
+
+	/**
+	 * @param bool $force Include systems that are autohold
+	 */
+	public function __construct(private ?bool $force=NULL) {}
+
+	/**
+	 * Execute the job.
+	 */
+	public function handle(): void
+	{
+		// Our zones
+		$our_zones = Setup::findOrFail(config('app.id'))->system->addresses->pluck('zone_id')->unique();
+
+		// Find our uplinks that are hubs, NC, RC or ZCs
+		// Find any system that also has heartbeat configured
+		$l = Address::select(['addresses.id','addresses.zone_id','addresses.region_id','addresses.host_id','addresses.node_id','addresses.point_id','role','addresses.system_id'])
+			->distinct('systems.id')
+			->join('systems',['systems.id'=>'addresses.system_id'])
+			->join('system_zone',['system_zone.system_id'=>'systems.id'])
+			->join('zones',['zones.id'=>'addresses.zone_id'])
+			->join('domains',['domains.id'=>'zones.domain_id'])
+			->where('systems.active',true)
+			->where('addresses.active',TRUE)
+			->where('zones.active',TRUE)
+			->where('domains.active',TRUE)
+			->whereIN('addresses.zone_id',$our_zones)
+			->whereNotNull('pollmode')
+			->where(function($query) {
+				return $query
+					->where('role','<',Address::NODE_ACTIVE)
+					->orWhereNotNull('heartbeat');
+			})
+			->when(! $this->force,function($query) {
+				return $query
+					->where(function($query) {
+						return $query->whereNull('autohold')
+							->orWhere('autohold',FALSE);
+				});
+			})
+			->with(['system','zone.domain'])
+			->get();
+
+		// If we havent polled in heatbeat hours, poll system
+		foreach ($l as $oo) {
+			if (Job::where('queue','poll')->get()->pluck('command.address.id')->search($oo->id) === FALSE) {
+				if ((! $oo->system->last_session)
+					|| ($oo->system->hearbeat && ($oo->system->last_session->addHours($oo->system->heartbeat) < Carbon::now()))
+					|| ((! $oo->system->hearbeat) && ($oo->role < Address::NODE_ACTIVE) && ($oo->system->last_session->addHours(6) < Carbon::now())))
+				{
+					Log::info(sprintf('%s:- Polling [%s] (%s) - we havent seen them since [%s], heartbeat [%d]',
+						self::LOGKEY,
+						$oo->ftn,
+						$oo->system->name,
+						$oo->system->last_session ?: 'Never',
+						$oo->system->heartbeat,
+					));
+
+					AddressPoll::dispatch($oo);
+
+				} else {
+					Log::debug(sprintf('%s:= Not scheduling poll to [%s], we saw them [%s], heartbeat [%d]',
+						self::LOGKEY,
+						$oo->ftn,
+						$oo->system->last_session,
+						$oo->system->heartbeat
+					));
+				}
+
+			} else {
+				Log::notice(sprintf('%s:= Not scheduling poll to [%s], there is already one in the queue',self::LOGKEY,$oo->ftn));
+			}
+		}
+	}
+}