deon/osb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update and fix broadband traffic

Browse Source
This commit is contained in:
Deon George 2022-04-20 16:24:58 +10:00
parent 621a132e35
commit 16b7e0b493
13 changed files with 161 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
app/Classes/External/Supplier.php vendored
View File

@ -17,6 +17,8 @@ abstract class Supplier
protected $o = NULL;
protected $_columns = [];
public const traffic_connection_keys = ['user','pass','url'];
public function __construct(Model $o)
{
$this->o = $o;
@ -26,24 +28,29 @@ abstract class Supplier
/**
* Connect and pull down traffic data
*
* @param array $connection
* @param string $type
* @return Collection
* @throws \Exception
*/
public function fetch(): Collection
public function fetch(array $connection,string $type): Collection
{
if (count(array_intersect(array_keys($connection),self::traffic_connection_keys)) !== 3)
throw new \Exception('No or missing connection details for:'.$type);
if ($x=$this->mustPause()) {
Log::notice(sprintf('%s:API Throttle, waiting [%s]...',self::LOGKEY,$x),['m'=>__METHOD__]);
sleep($x);
}
Log::debug(sprintf('%s:Supplier [%d], fetch data for [%s]...',self::LOGKEY,$this->o->id,$this->o->stats_lastupdate),['m'=>__METHOD__]);
$key = 'Supplier:'.$this->o->id.$this->o->stats_lastupdate;
$result = Cache::remember($key,86400,function() {
$client = $this->getClient();
Log::debug(sprintf('%s:Supplier [%d], fetch data for [%s]...',self::LOGKEY,$this->o->id,Arr::get($connection,'last')),['m'=>__METHOD__]);
$key = 'Supplier:'.$this->o->id.Arr::get($connection,'last');
$response = Http::get($this->o->stats_url,[
$this->login_user_field => $this->o->stats_username,
$this->login_pass_field => $this->o->stats_password,
$this->date_field => $this->o->stats_lastupdate->format('Y-m-d'),
$result = Cache::remember($key,86400,function() use ($connection) {
$response = Http::get(Arr::get($connection,'url'),[
$this->login_user_field => Arr::get($connection,'user'),
$this->login_pass_field => Arr::get($connection,'pass'),
$this->date_field => Arr::get($connection,'last'),
]);
// @todo These API rate limiting is untested.
@ -55,7 +62,6 @@ abstract class Supplier
Cache::put('api_throttle',$api_reset,now()->addSeconds($api_reset));
}
//dd($response->header('Content-Type'),$response->headers());
// Assume the supplier provides an ASCII output for text/html
if (preg_match('#^text/html;#',$x=$response->header('Content-Type'))) {
return collect(explode("\n",$response->body()))->filter();

14
app/Console/Commands/BroadbandTraffic.php
View File

@ -5,7 +5,7 @@ namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Jobs\BroadbandTraffic as Job;
use App\Models\AdslSupplier;
use App\Models\Supplier;
class BroadbandTraffic extends Command
{
@ -14,7 +14,8 @@ class BroadbandTraffic extends Command
*
* @var string
*/
protected $signature = 'broadband:traffic:import';
protected $signature = 'broadband:traffic:import'.
' {--s|supplier= : Supplier Name}';
/**
* The console command description.
@ -30,7 +31,14 @@ class BroadbandTraffic extends Command
*/
public function handle()
{
foreach (AdslSupplier::active()->get() as $o)
if ($this->option('supplier')) {
$o = Supplier::where('name','like',$this->option('supplier'))->singleOrFail();
Job::dispatch($o);
return;
}
foreach (Supplier::active()->get() as $o)
Job::dispatch($o);
}
}

4
app/Console/Kernel.php
View File

@ -5,8 +5,8 @@ namespace App\Console;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
use App\Models\AdslSupplier;
use App\Jobs\BroadbandTraffic;
use App\Models\Supplier;
class Kernel extends ConsoleKernel
{
@ -29,7 +29,7 @@ class Kernel extends ConsoleKernel
{
// @todo This needs to be more generic and dynamic
// Exetel Traffic
$schedule->job(new BroadbandTraffic(AdslSupplier::find(1)))->timezone('Australia/Melbourne')->dailyAt('10:00');
$schedule->job(new BroadbandTraffic(Supplier::find(1)))->timezone('Australia/Melbourne')->dailyAt('10:00');
}
/**

7
app/Interfaces/ServiceUsage.php
View File

@ -6,6 +6,13 @@ use Illuminate\Support\Collection;
interface ServiceUsage
{
/**
* Our model that holds traffic information
*
* @return mixed
*/
public function traffic();
/**
* This service provides usage information
*

68
app/Jobs/BroadbandTraffic.php
View File

@ -9,14 +9,16 @@ use Illuminate\Database\Eloquent\Model;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Mail;
use App\Classes\External\Supplier as ExternalSupplier;
use App\Mail\TrafficMismatch;
use App\Models\Service\Adsl;
use App\Models\Service\AdslTraffic;
use App\Models\AdslSupplier;
use App\Models\Supplier;
use App\Models\Service\Broadband as ServiceBroadband;
use App\Models\Usage\Broadband as UsageBroadband;
/**
* Class BroadbandTraffic
@ -33,7 +35,9 @@ final class BroadbandTraffic implements ShouldQueue
protected Model $o; // The supplier we are updating from
private const class_prefix = 'App\Classes\External\Supplier\\';
public function __construct(AdslSupplier $o)
private const traffic = 'broadband';
public function __construct(Supplier $o)
{
$this->o = $o;
}
@ -43,12 +47,14 @@ final class BroadbandTraffic implements ShouldQueue
*
* @return void
* @throws \Exception
* @todo The column stats_lastupdate is actually the "next" date that stats should be retrieved. Rename it.
*/
public function handle()
{
Log::info(sprintf('%s:Importing Broadband Traffic from [%s]',self::LOGKEY,$this->o->name));
if ((! $connection=$this->o->detail->connections->get('broadband')) || (count(array_intersect(array_keys($connection),ExternalSupplier::traffic_connection_keys)) !== 3))
throw new \Exception('No or missing connection details for:'.self::traffic);
$u = 0;
// Load our class for this supplier
@ -57,21 +63,24 @@ final class BroadbandTraffic implements ShouldQueue
$o = new $class($this->o);
} else {
Log::error(sprintf('%s: Class doesnt exist: %d',get_class($this),$class));
Log::error(sprintf('%s: Class doesnt exist: %s',self::LOGKEY,$class));
exit(1);
}
$last_update = Carbon::create(Arr::get($connection,'last'))->addDay();
Arr::set($connection,'last',$last_update->format('Y-m-d'));
// Repeat pull traffic data until yesterday
while ($this->o->stats_lastupdate < Carbon::now()->subDay()) {
Log::notice(sprintf('%s:Next update is [%s]',self::LOGKEY,$this->o->stats_lastupdate->format('Y-m-d')));
while ($last_update < Carbon::now()->subDay()) {
Log::notice(sprintf('%s:Next update is [%s]',self::LOGKEY,$last_update->format('Y-m-d')));
// Delete traffic, since we'll refresh it.
AdslTraffic::where('supplier_id',$this->o->id)
->where('date',$this->o->stats_lastupdate)
UsageBroadband::where('supplier_id',$this->o->id)
->where('date',$last_update->format('Y-m-d'))
->delete();
$c = 0;
foreach ($o->fetch() as $line) {
foreach ($o->fetch($connection,self::traffic) as $line) {
// The first row is our header
if (! $c++) {
$fields = $o->getColumns(preg_replace('/,\s+/',',',$line),collect($o->header()));
@ -79,28 +88,26 @@ final class BroadbandTraffic implements ShouldQueue
}
if (! $fields->count())
abort(500,'? No fields in data exportupda');
abort(500,'? No fields in data export');
$row = str_getcsv(trim($line));
try {
// @todo Put the date format in the DB.
$date = Carbon::createFromFormat('Y-m-d',$row[$o->getColumnKey('Date')]);
// Find the right service dependant on the dates we supplied the service
$oo = Adsl::where('service_username',$row[$o->getColumnKey('Login')])
->select(DB::raw('ab_service__adsl.*'))
// Find the right service dependent on the dates we supplied the service
$oo = ServiceBroadband::where('service_username',$row[$o->getColumnKey('Login')])
->select(DB::raw('service_broadband.*'))
->join('services','services.id','=','service_id')
->where('services.start_at','<=',$date)
->where(function($query) use ($date) {
$query->whereNULL('services.stop_at')
->orWhere('services.stop_at','<=',$date);
})
->get();
->single();
$to = new AdslTraffic;
$to->site_id = 1; // @todo TO ADDRESS
$to->date = $this->o->stats_lastupdate;
$to = new UsageBroadband;
$to->site_id = $oo->site_id;
$to->date = $last_update;
$to->supplier_id = $this->o->id;
$to->up_peak = $row[$o->getColumnKey('Peak upload')];
$to->up_offpeak = $row[$o->getColumnKey('Off peak upload')];
@ -111,30 +118,35 @@ final class BroadbandTraffic implements ShouldQueue
$to->time = '24:00'; // @todo
// If we have no records
if ($oo->count() != 1) {
if (! $oo) {
Log::error(sprintf('%s:Too many services return for [%s]',self::LOGKEY,$row[$o->getColumnKey('Login')]),['date'=>$date,'count'=>$oo->count()]);
$to->service = $row[$o->getColumnKey('Login')];
$to->save();
} else {
$oo->first()->traffic()->save($to);
$to->service_item_id = $oo->id;
}
if ($to->save())
$u++;
} catch (\Exception $e) {
dump($to);
Log::error(sprintf('%s:Exception occurred when storing traffic record for [%s].',self::LOGKEY,$row[$o->getColumnKey('Login')]),['row'=>$row,'line'=>$line]);
throw new \Exception('Error while storing traffic date');
}
}
Log::info(sprintf('%s: Records Imported [%d] for [%s]',self::LOGKEY,$u,$last_update->format('Y-m-d')));
Log::info(sprintf('%s: Records Imported [%d] for [%s]',self::LOGKEY,$u,$this->o->stats_lastupdate->format('Y-m-d')));
// Save our current progress.
$this->o->detail->connections = $this->o->detail->connections->put(self::traffic,array_merge($connection,['last'=>$last_update->format('Y-m-d')]));
$this->o->detail->save();
// Update our details for the next iteration.
$last_update = $last_update->addDay();
Arr::set($connection,'last',$last_update->format('Y-m-d'));
if ($u) {
$this->o->stats_lastupdate = $this->o->stats_lastupdate->addDay();
$this->o->save();
if ($this->o->trafficMismatch($date)->count())
Mail::to('deon@graytech.net.au') // @todo To change
->send(new TrafficMismatch($this->o,$date));

6
app/Mail/TrafficMismatch.php
View File

@ -8,13 +8,13 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Mail\Mailable;
use Illuminate\Queue\SerializesModels;
use App\Models\{AdslSupplier,Site};
use App\Models\{Supplier,Site};
class TrafficMismatch extends Mailable
{
use Queueable, SerializesModels;
public AdslSupplier $aso;
public Supplier $aso;
public Carbon $date;
/**
@ -22,7 +22,7 @@ class TrafficMismatch extends Mailable
*
* @return void
*/
public function __construct(AdslSupplier $o,Carbon $date)
public function __construct(Supplier $o,Carbon $date)
{
$this->aso = $o;
$this->date = $date;

48
app/Models/AdslSupplier.php
View File

@ -1,48 +0,0 @@
<?php
namespace App\Models;
use App\Models\Service\AdslTraffic;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Collection;
use Illuminate\Database\Eloquent\Model;
/**
* @deprecated
*/
class AdslSupplier extends Model
{
protected $table = 'ab_adsl_supplier';
protected $dates = [
'stats_lastupdate',
];
public $timestamps = FALSE;
/** SCOPES */
/**
* Only query active categories
*/
public function scopeActive($query)
{
return $query->where('active',TRUE);
}
/** METHODS **/
/**
* Return the traffic records, that were not matched to a service.
*
* @param Carbon $date
* @return Collection
*/
public function trafficMismatch(Carbon $date): Collection
{
return AdslTraffic::where('date',$date->format('Y-m-d'))
->where('supplier_id',$this->id)
->whereNULL('ab_service_adsl_id')
->get();
}
}

2
app/Models/Product/Broadband.php
View File

@ -22,7 +22,7 @@ final class Broadband extends Type implements ProductItem
'options.address'=>[
'request'=>'options.address',
'key'=>'service_address',
'validation'=>'required|string:10|unique:ab_service__adsl,service_address',
'validation'=>'required|string:10|unique:service_broadband,service_address',
'validation_message'=>'Address is a required field.',
],
'options.notes'=>[

15
app/Models/Service/Broadband.php
View File

@ -10,6 +10,7 @@ use App\Interfaces\ServiceUsage;
use App\Models\Base\ServiceType;
use App\Models\Supplier\Broadband as SupplierBroadband;
use App\Models\Supplier\Type;
use App\Models\Usage\Broadband as UsageBroadband;
/**
* Class Broadband (Service)
@ -65,17 +66,15 @@ class Broadband extends ServiceType implements ServiceUsage
return $this->service_number ?: ($this->service_address ?: '-');
}
/* RELATIONS */
/**
* The accounts that this user manages
* The usage information for broadband
*
* @return \Illuminate\Database\Eloquent\Relations\HasMany
*/
public function traffic()
{
// @todo Need to include site_id in this relation
return $this->hasMany(AdslTraffic::class,'ab_service_adsl_id');
return $this->hasMany(UsageBroadband::class,'service_item_id')
->where('site_id',$this->site_id);
}
/* ATTRIBUTES */
@ -128,8 +127,6 @@ class Broadband extends ServiceType implements ServiceUsage
if (! $maxdate)
return collect();
Log::debug(sprintf('%s:Getting Usage data for [%d] days from [%s]',self::LOGKEY,$days,$maxdate->date->format('Y-m-d')),['m'=>__METHOD__]);
return $this->traffic()
->where('date','<=',$maxdate->date->format('Y-m-d'))
->where('date','>=',$maxdate->date->subDays($days)->format('Y-m-d'))
@ -139,9 +136,9 @@ class Broadband extends ServiceType implements ServiceUsage
/**
* Find the last date any traffic was recorded for a service
*
* @return AdslTraffic|null
* @return UsageBroadband|null
*/
private function usage_last_date(): ?AdslTraffic
private function usage_last_date(): ?UsageBroadband
{
return $this->traffic
->sortBy('date')

18
app/Models/Supplier.php
View File

@ -2,6 +2,7 @@
namespace App\Models;
use Carbon\Carbon;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Arr;
use Illuminate\Support\Collection;
@ -62,7 +63,8 @@ class Supplier extends Model
// EG: Crazy Domains, "domains" and "hosting".
public function detail()
{
return $this->hasOne(SupplierDetail::class);
return $this->hasOne(SupplierDetail::class)
->withoutGlobalScope(\App\Models\Scopes\SiteScope::class);
}
/* METHODS */
@ -103,4 +105,18 @@ class Supplier extends Model
return $result;
}
/**
* Return the traffic records, that were not matched to a service.
*
* @param Carbon $date
* @return \Illuminate\Database\Eloquent\Collection
*/
public function trafficMismatch(Carbon $date): Collection
{
return Usage\Broadband::where('date',$date->format('Y-m-d'))
->where('supplier_id',$this->id)
->whereNULL('service_item_id')
->get();
}
}

18
app/Models/Service/AdslTraffic.php → app/Models/Usage/Broadband.php
View File

@ -1,23 +1,29 @@
<?php
namespace App\Models\Service;
namespace App\Models\Usage;
use Illuminate\Database\Eloquent\Model;
use Leenooks\Carbon;
class AdslTraffic extends Model
use App\Models\Service\Broadband as ServiceBroadband;
class Broadband extends Model
{
protected $table = 'ab_service__adsl_traffic';
public $timestamps = FALSE;
protected $dates = ['date'];
public $dateFormat = 'U';
protected $table = 'usage_broadband';
public $timestamps = FALSE;
private $traffic_end = 14;
/* RELATIONS */
public function broadband()
{
return $this->belongsTo(Adsl::class);
return $this->belongsTo(ServiceBroadband::class);
}
/* ATTRIBUTES */
public function getTotalAttribute() {
return $this->up_peak+$this->down_peak+$this->up_offpeak+$this->down_offpeak;
}

3
database/migrations/2022_04_19_121452_rename_service_tables.php
View File

@ -266,6 +266,8 @@ class RenameServiceTables extends Migration
DB::statement('ALTER TABLE service_broadband MODIFY service_id int unsigned NOT NULL');
DB::statement('ALTER TABLE service_broadband MODIFY service_stats_collect tinyint(1) DEFAULT NULL');
DB::statement('ALTER TABLE service_broadband RENAME COLUMN service_stats_lastupdate TO service_stats_at');
DB::statement('ALTER TABLE service_broadband RENAME COLUMN provided_adsl_plan_id TO provided_supplier_broadband_id');
DB::statement('ALTER TABLE service_broadband MODIFY provided_supplier_broadband_id int unsigned DEFAULT NULL');
// @todo drop column provided_adsl_plan_id
Schema::table('service_broadband', function (Blueprint $table) {
@ -281,6 +283,7 @@ class RenameServiceTables extends Migration
$table->dropIndex('ab_service__adsl_id_site_id_index');
$table->foreign(['service_id','site_id'])->references(['id','site_id'])->on('services');
$table->foreign(['provided_supplier_broadband_id','site_id'])->references(['id','site_id'])->on('supplier_broadband');
});
// Convert our dates

42
database/migrations/2022_04_20_134953_rename_traffic.php Normal file
View File

@ -0,0 +1,42 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class RenameTraffic extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
DB::statement('ALTER TABLE ab_service__adsl_traffic RENAME TO usage_broadband');
DB::statement('ALTER TABLE usage_broadband RENAME COLUMN ab_service_adsl_id TO service_item_id');
DB::statement('ALTER TABLE usage_broadband MODIFY supplier_id int unsigned NOT NULL'); // @todo This should be deleted, it could be obtained by the product
DB::statement('ALTER TABLE usage_broadband MODIFY service_item_id int unsigned DEFAULT NULL');
DB::statement('ALTER TABLE usage_broadband MODIFY site_id int unsigned NOT NULL');
Schema::table('usage_broadband', function (Blueprint $table) {
$table->unique(['service_item_id','site_id','date','time']);
$table->foreign(['service_item_id','site_id'])->references(['id','site_id'])->on('service_broadband');
$table->foreign(['supplier_id'])->references(['id'])->on('suppliers');
});
Schema::table('suppliers', function (Blueprint $table) {
$table->date('usage_last')->nullable();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
abort(500,'Cant go back');
}
}