Update and fix broadband traffic

app/Classes/External/Supplier.php

@@ -17,6 +17,8 @@ abstract class Supplier
 	protected $o = NULL;
 	protected $_columns = [];
 
+	public const traffic_connection_keys = ['user','pass','url'];
+
 	public function __construct(Model $o)
 	{
 		$this->o = $o;
@@ -26,24 +28,29 @@ abstract class Supplier
 	/**
 	 * Connect and pull down traffic data
 	 *
+	 * @param array $connection
+	 * @param string $type
 	 * @return Collection
+	 * @throws \Exception
 	 */
-	public function fetch(): Collection
+	public function fetch(array $connection,string $type): Collection
 	{
+		if (count(array_intersect(array_keys($connection),self::traffic_connection_keys)) !== 3)
+			throw new \Exception('No or missing connection details for:'.$type);
+
 		if ($x=$this->mustPause()) {
 			Log::notice(sprintf('%s:API Throttle, waiting [%s]...',self::LOGKEY,$x),['m'=>__METHOD__]);
 			sleep($x);
 		}
 
-		Log::debug(sprintf('%s:Supplier [%d], fetch data for [%s]...',self::LOGKEY,$this->o->id,$this->o->stats_lastupdate),['m'=>__METHOD__]);
+		Log::debug(sprintf('%s:Supplier [%d], fetch data for [%s]...',self::LOGKEY,$this->o->id,Arr::get($connection,'last')),['m'=>__METHOD__]);
-		$key = 'Supplier:'.$this->o->id.$this->o->stats_lastupdate;
+		$key = 'Supplier:'.$this->o->id.Arr::get($connection,'last');
-		$result = Cache::remember($key,86400,function() {
-			$client = $this->getClient();
 
-			$response = Http::get($this->o->stats_url,[
+		$result = Cache::remember($key,86400,function() use ($connection) {
-				$this->login_user_field => $this->o->stats_username,
+			$response = Http::get(Arr::get($connection,'url'),[
-				$this->login_pass_field => $this->o->stats_password,
+				$this->login_user_field => Arr::get($connection,'user'),
-				$this->date_field => $this->o->stats_lastupdate->format('Y-m-d'),
+				$this->login_pass_field => Arr::get($connection,'pass'),
+				$this->date_field => Arr::get($connection,'last'),
 			]);
 
 			// @todo These API rate limiting is untested.
@@ -55,7 +62,6 @@ abstract class Supplier
 				Cache::put('api_throttle',$api_reset,now()->addSeconds($api_reset));
 			}
 
-			//dd($response->header('Content-Type'),$response->headers());
 			// Assume the supplier provides an ASCII output for text/html
 			if (preg_match('#^text/html;#',$x=$response->header('Content-Type'))) {
 				return collect(explode("\n",$response->body()))->filter();

app/Console/Commands/BroadbandTraffic.php

@@ -5,7 +5,7 @@ namespace App\Console\Commands;
 use Illuminate\Console\Command;
 
 use App\Jobs\BroadbandTraffic as Job;
-use App\Models\AdslSupplier;
+use App\Models\Supplier;
 
 class BroadbandTraffic extends Command
 {
@@ -14,7 +14,8 @@ class BroadbandTraffic extends Command
 	 *
 	 * @var string
 	 */
-	protected $signature = 'broadband:traffic:import';
+	protected $signature = 'broadband:traffic:import'.
+		' {--s|supplier= : Supplier Name}';
 
 	/**
 	 * The console command description.
@@ -30,7 +31,14 @@ class BroadbandTraffic extends Command
 	 */
 	public function handle()
 	{
-		foreach (AdslSupplier::active()->get() as $o)
+		if ($this->option('supplier')) {
+			$o = Supplier::where('name','like',$this->option('supplier'))->singleOrFail();
+
+			Job::dispatch($o);
+			return;
+		}
+
+		foreach (Supplier::active()->get() as $o)
 			Job::dispatch($o);
 	}
 }

app/Console/Kernel.php

@@ -5,8 +5,8 @@ namespace App\Console;
 use Illuminate\Console\Scheduling\Schedule;
 use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
 
-use App\Models\AdslSupplier;
 use App\Jobs\BroadbandTraffic;
+use App\Models\Supplier;
 
 class Kernel extends ConsoleKernel
 {
@@ -29,7 +29,7 @@ class Kernel extends ConsoleKernel
     {
         // @todo This needs to be more generic and dynamic
         // Exetel Traffic
-        $schedule->job(new BroadbandTraffic(AdslSupplier::find(1)))->timezone('Australia/Melbourne')->dailyAt('10:00');
+        $schedule->job(new BroadbandTraffic(Supplier::find(1)))->timezone('Australia/Melbourne')->dailyAt('10:00');
     }
 
     /**

app/Interfaces/ServiceUsage.php

@@ -6,6 +6,13 @@ use Illuminate\Support\Collection;
 
 interface ServiceUsage
 {
+	/**
+	 * Our model that holds traffic information
+	 *
+	 * @return mixed
+	 */
+	public function traffic();
+
 	/**
 	 * This service provides usage information
 	 *

app/Jobs/BroadbandTraffic.php

@@ -9,14 +9,16 @@ use Illuminate\Database\Eloquent\Model;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Arr;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\Mail;
 
+use App\Classes\External\Supplier as ExternalSupplier;
 use App\Mail\TrafficMismatch;
-use App\Models\Service\Adsl;
+use App\Models\Supplier;
-use App\Models\Service\AdslTraffic;
+use App\Models\Service\Broadband as ServiceBroadband;
-use App\Models\AdslSupplier;
+use App\Models\Usage\Broadband as UsageBroadband;
 
 /**
  * Class BroadbandTraffic
@@ -33,7 +35,9 @@ final class BroadbandTraffic implements ShouldQueue
 	protected Model $o;                     // The supplier we are updating from
 	private const class_prefix = 'App\Classes\External\Supplier\\';
 
-	public function __construct(AdslSupplier $o)
+	private const traffic = 'broadband';
+
+	public function __construct(Supplier $o)
 	{
 		$this->o = $o;
 	}
@@ -43,12 +47,14 @@ final class BroadbandTraffic implements ShouldQueue
 	 *
 	 * @return void
 	 * @throws \Exception
-	 * @todo The column stats_lastupdate is actually the "next" date that stats should be retrieved. Rename it.
 	 */
 	public function handle()
 	{
 		Log::info(sprintf('%s:Importing Broadband Traffic from [%s]',self::LOGKEY,$this->o->name));
 
+		if ((! $connection=$this->o->detail->connections->get('broadband')) || (count(array_intersect(array_keys($connection),ExternalSupplier::traffic_connection_keys)) !== 3))
+			throw new \Exception('No or missing connection details for:'.self::traffic);
+
 		$u = 0;
 
 		// Load our class for this supplier
@@ -57,21 +63,24 @@ final class BroadbandTraffic implements ShouldQueue
 			$o = new $class($this->o);
 
 		} else {
-			Log::error(sprintf('%s: Class doesnt exist: %d',get_class($this),$class));
+			Log::error(sprintf('%s: Class doesnt exist: %s',self::LOGKEY,$class));
 			exit(1);
 		}
 
+		$last_update = Carbon::create(Arr::get($connection,'last'))->addDay();
+		Arr::set($connection,'last',$last_update->format('Y-m-d'));
+
 		// Repeat pull traffic data until yesterday
-		while ($this->o->stats_lastupdate < Carbon::now()->subDay()) {
+		while ($last_update < Carbon::now()->subDay()) {
-			Log::notice(sprintf('%s:Next update is [%s]',self::LOGKEY,$this->o->stats_lastupdate->format('Y-m-d')));
+			Log::notice(sprintf('%s:Next update is [%s]',self::LOGKEY,$last_update->format('Y-m-d')));
 
 			// Delete traffic, since we'll refresh it.
-			AdslTraffic::where('supplier_id',$this->o->id)
+			UsageBroadband::where('supplier_id',$this->o->id)
-				->where('date',$this->o->stats_lastupdate)
+				->where('date',$last_update->format('Y-m-d'))
 				->delete();
 
 			$c = 0;
-			foreach ($o->fetch() as $line) {
+			foreach ($o->fetch($connection,self::traffic) as $line) {
 				// The first row is our header
 				if (! $c++) {
 					$fields = $o->getColumns(preg_replace('/,\s+/',',',$line),collect($o->header()));
@@ -79,28 +88,26 @@ final class BroadbandTraffic implements ShouldQueue
 				}
 
 				if (! $fields->count())
-					abort(500,'? No fields in data exportupda');
+					abort(500,'? No fields in data export');
 
 				$row = str_getcsv(trim($line));
 
 				try {
-					// @todo Put the date format in the DB.
 					$date = Carbon::createFromFormat('Y-m-d',$row[$o->getColumnKey('Date')]);
-
+					// Find the right service dependent on the dates we supplied the service
-					// Find the right service dependant on the dates we supplied the service
+					$oo = ServiceBroadband::where('service_username',$row[$o->getColumnKey('Login')])
-					$oo = Adsl::where('service_username',$row[$o->getColumnKey('Login')])
+						->select(DB::raw('service_broadband.*'))
-						->select(DB::raw('ab_service__adsl.*'))
 						->join('services','services.id','=','service_id')
 						->where('services.start_at','<=',$date)
 						->where(function($query) use ($date) {
 							$query->whereNULL('services.stop_at')
 								->orWhere('services.stop_at','<=',$date);
 						})
-						->get();
+						->single();
 
-					$to = new AdslTraffic;
+					$to = new UsageBroadband;
-					$to->site_id = 1; // @todo TO ADDRESS
+					$to->site_id = $oo->site_id;
-					$to->date = $this->o->stats_lastupdate;
+					$to->date = $last_update;
 					$to->supplier_id = $this->o->id;
 					$to->up_peak = $row[$o->getColumnKey('Peak upload')];
 					$to->up_offpeak = $row[$o->getColumnKey('Off peak upload')];
@@ -111,30 +118,35 @@ final class BroadbandTraffic implements ShouldQueue
 					$to->time = '24:00'; // @todo
 
 					// If we have no records
-					if ($oo->count() != 1) {
+					if (! $oo) {
 						Log::error(sprintf('%s:Too many services return for [%s]',self::LOGKEY,$row[$o->getColumnKey('Login')]),['date'=>$date,'count'=>$oo->count()]);
 
 						$to->service = $row[$o->getColumnKey('Login')];
-						$to->save();
 
 					} else {
-						$oo->first()->traffic()->save($to);
+						$to->service_item_id = $oo->id;
 					}
 
-					$u++;
+					if ($to->save())
+						$u++;
 
 				} catch (\Exception $e) {
+					dump($to);
 					Log::error(sprintf('%s:Exception occurred when storing traffic record for [%s].',self::LOGKEY,$row[$o->getColumnKey('Login')]),['row'=>$row,'line'=>$line]);
 					throw new \Exception('Error while storing traffic date');
 				}
 			}
+			Log::info(sprintf('%s: Records Imported [%d] for [%s]',self::LOGKEY,$u,$last_update->format('Y-m-d')));
 
-			Log::info(sprintf('%s: Records Imported [%d] for [%s]',self::LOGKEY,$u,$this->o->stats_lastupdate->format('Y-m-d')));
+			// Save our current progress.
+			$this->o->detail->connections = $this->o->detail->connections->put(self::traffic,array_merge($connection,['last'=>$last_update->format('Y-m-d')]));
+			$this->o->detail->save();
+
+			// Update our details for the next iteration.
+			$last_update = $last_update->addDay();
+			Arr::set($connection,'last',$last_update->format('Y-m-d'));
 
 			if ($u) {
-				$this->o->stats_lastupdate = $this->o->stats_lastupdate->addDay();
-				$this->o->save();
-
 				if ($this->o->trafficMismatch($date)->count())
 					Mail::to('deon@graytech.net.au')        // @todo To change
 						->send(new TrafficMismatch($this->o,$date));

app/Mail/TrafficMismatch.php

@@ -8,13 +8,13 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Mail\Mailable;
 use Illuminate\Queue\SerializesModels;
 
-use App\Models\{AdslSupplier,Site};
+use App\Models\{Supplier,Site};
 
 class TrafficMismatch extends Mailable
 {
 	use Queueable, SerializesModels;
 
-	public AdslSupplier $aso;
+	public Supplier $aso;
 	public Carbon $date;
 
 	/**
@@ -22,7 +22,7 @@ class TrafficMismatch extends Mailable
 	 *
 	 * @return void
 	 */
-	public function __construct(AdslSupplier $o,Carbon $date)
+	public function __construct(Supplier $o,Carbon $date)
 	{
 		$this->aso = $o;
 		$this->date = $date;

app/Models/AdslSupplier.php

@@ -1,48 +0,0 @@
-<?php
-
-namespace App\Models;
-
-use App\Models\Service\AdslTraffic;
-use Carbon\Carbon;
-use Illuminate\Database\Eloquent\Collection;
-use Illuminate\Database\Eloquent\Model;
-
-/**
- * @deprecated
- */
-class AdslSupplier extends Model
-{
-	protected $table = 'ab_adsl_supplier';
-
-	protected $dates = [
-		'stats_lastupdate',
-	];
-
-	public $timestamps = FALSE;
-
-	/** SCOPES */
-
-	/**
-	 * Only query active categories
-	 */
-	public function scopeActive($query)
-	{
-		return $query->where('active',TRUE);
-	}
-
-	/** METHODS **/
-
-	/**
-	 * Return the traffic records, that were not matched to a service.
-	 *
-	 * @param Carbon $date
-	 * @return Collection
-	 */
-	public function trafficMismatch(Carbon $date): Collection
-	{
-		return AdslTraffic::where('date',$date->format('Y-m-d'))
-			->where('supplier_id',$this->id)
-			->whereNULL('ab_service_adsl_id')
-			->get();
-	}
-}

app/Models/Product/Broadband.php

@@ -22,7 +22,7 @@ final class Broadband extends Type implements ProductItem
 		'options.address'=>[
 			'request'=>'options.address',
 			'key'=>'service_address',
-			'validation'=>'required|string:10|unique:ab_service__adsl,service_address',
+			'validation'=>'required|string:10|unique:service_broadband,service_address',
 			'validation_message'=>'Address is a required field.',
 		],
 		'options.notes'=>[

app/Models/Service/Broadband.php

@@ -10,6 +10,7 @@ use App\Interfaces\ServiceUsage;
 use App\Models\Base\ServiceType;
 use App\Models\Supplier\Broadband as SupplierBroadband;
 use App\Models\Supplier\Type;
+use App\Models\Usage\Broadband as UsageBroadband;
 
 /**
  * Class Broadband (Service)
@@ -65,17 +66,15 @@ class Broadband extends ServiceType implements ServiceUsage
 		return $this->service_number ?: ($this->service_address ?: '-');
 	}
 
-	/* RELATIONS */
-
 	/**
-	 * The accounts that this user manages
+	 * The usage information for broadband
 	 *
 	 * @return \Illuminate\Database\Eloquent\Relations\HasMany
 	 */
 	public function traffic()
 	{
-		// @todo Need to include site_id in this relation
+		return $this->hasMany(UsageBroadband::class,'service_item_id')
-		return $this->hasMany(AdslTraffic::class,'ab_service_adsl_id');
+			->where('site_id',$this->site_id);
 	}
 
 	/* ATTRIBUTES */
@@ -128,8 +127,6 @@ class Broadband extends ServiceType implements ServiceUsage
 		if (! $maxdate)
 			return collect();
 
-		Log::debug(sprintf('%s:Getting Usage data for [%d] days from [%s]',self::LOGKEY,$days,$maxdate->date->format('Y-m-d')),['m'=>__METHOD__]);
-
 		return $this->traffic()
 			->where('date','<=',$maxdate->date->format('Y-m-d'))
 			->where('date','>=',$maxdate->date->subDays($days)->format('Y-m-d'))
@@ -139,9 +136,9 @@ class Broadband extends ServiceType implements ServiceUsage
 	/**
 	 * Find the last date any traffic was recorded for a service
 	 *
-	 * @return AdslTraffic|null
+	 * @return UsageBroadband|null
 	 */
-	private function usage_last_date(): ?AdslTraffic
+	private function usage_last_date(): ?UsageBroadband
 	{
 		return $this->traffic
 			->sortBy('date')

app/Models/Supplier.php

@@ -2,6 +2,7 @@
 
 namespace App\Models;
 
+use Carbon\Carbon;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Support\Arr;
 use Illuminate\Support\Collection;
@@ -62,7 +63,8 @@ class Supplier extends Model
 	// EG: Crazy Domains, "domains" and "hosting".
 	public function detail()
 	{
-		return $this->hasOne(SupplierDetail::class);
+		return $this->hasOne(SupplierDetail::class)
+			->withoutGlobalScope(\App\Models\Scopes\SiteScope::class);
 	}
 
 	/* METHODS */
@@ -103,4 +105,18 @@ class Supplier extends Model
 
 		return $result;
 	}
+
+	/**
+	 * Return the traffic records, that were not matched to a service.
+	 *
+	 * @param Carbon $date
+	 * @return \Illuminate\Database\Eloquent\Collection
+	 */
+	public function trafficMismatch(Carbon $date): Collection
+	{
+		return Usage\Broadband::where('date',$date->format('Y-m-d'))
+			->where('supplier_id',$this->id)
+			->whereNULL('service_item_id')
+			->get();
+	}
 }

app/Models/Usage/Broadband.php

@@ -1,23 +1,29 @@
 <?php
 
-namespace App\Models\Service;
+namespace App\Models\Usage;
 
 use Illuminate\Database\Eloquent\Model;
 use Leenooks\Carbon;
 
-class AdslTraffic extends Model
+use App\Models\Service\Broadband as ServiceBroadband;
+
+class Broadband extends Model
 {
-	protected $table = 'ab_service__adsl_traffic';
-	public $timestamps = FALSE;
 	protected $dates = ['date'];
-	public $dateFormat = 'U';
+	protected $table = 'usage_broadband';
+	public $timestamps = FALSE;
+
 	private $traffic_end = 14;
 
+	/* RELATIONS */
+
 	public function broadband()
 	{
-		return $this->belongsTo(Adsl::class);
+		return $this->belongsTo(ServiceBroadband::class);
 	}
 
+	/* ATTRIBUTES */
+
 	public function getTotalAttribute() {
 		return $this->up_peak+$this->down_peak+$this->up_offpeak+$this->down_offpeak;
 	}

database/migrations/2022_04_19_121452_rename_service_tables.php

@@ -266,6 +266,8 @@ class RenameServiceTables extends Migration
 		DB::statement('ALTER TABLE service_broadband MODIFY service_id int unsigned NOT NULL');
 		DB::statement('ALTER TABLE service_broadband MODIFY service_stats_collect tinyint(1) DEFAULT NULL');
 		DB::statement('ALTER TABLE service_broadband RENAME COLUMN service_stats_lastupdate TO service_stats_at');
+		DB::statement('ALTER TABLE service_broadband RENAME COLUMN provided_adsl_plan_id TO provided_supplier_broadband_id');
+		DB::statement('ALTER TABLE service_broadband MODIFY provided_supplier_broadband_id int unsigned DEFAULT NULL');
 		// @todo drop column provided_adsl_plan_id
 
 		Schema::table('service_broadband', function (Blueprint $table) {
@@ -281,6 +283,7 @@ class RenameServiceTables extends Migration
 			$table->dropIndex('ab_service__adsl_id_site_id_index');
 
 			$table->foreign(['service_id','site_id'])->references(['id','site_id'])->on('services');
+			$table->foreign(['provided_supplier_broadband_id','site_id'])->references(['id','site_id'])->on('supplier_broadband');
 		});
 
 		// Convert our dates

database/migrations/2022_04_20_134953_rename_traffic.php

@@ -0,0 +1,42 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class RenameTraffic extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+		DB::statement('ALTER TABLE ab_service__adsl_traffic RENAME TO usage_broadband');
+		DB::statement('ALTER TABLE usage_broadband RENAME COLUMN ab_service_adsl_id TO service_item_id');
+		DB::statement('ALTER TABLE usage_broadband MODIFY supplier_id int unsigned NOT NULL');	// @todo This should be deleted, it could be obtained by the product
+		DB::statement('ALTER TABLE usage_broadband MODIFY service_item_id int unsigned DEFAULT NULL');
+		DB::statement('ALTER TABLE usage_broadband MODIFY site_id int unsigned NOT NULL');
+
+		Schema::table('usage_broadband', function (Blueprint $table) {
+			$table->unique(['service_item_id','site_id','date','time']);
+			$table->foreign(['service_item_id','site_id'])->references(['id','site_id'])->on('service_broadband');
+			$table->foreign(['supplier_id'])->references(['id'])->on('suppliers');
+		});
+
+		Schema::table('suppliers', function (Blueprint $table) {
+			$table->date('usage_last')->nullable();
+		});
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+		abort(500,'Cant go back');
+    }
+}