Import nodelists
This commit is contained in:
Deon George
@@ -2,8 +2,11 @@
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use Carbon\Carbon;
|
||||
use Illuminate\Console\Command;
|
||||
use App\Models\{Flag,Node,Zone};
|
||||
|
||||
use App\Models\{Domain,Nodelist};
|
||||
use App\Jobs\ImportNodelist as Job;
|
||||
|
||||
class ImportNodelist extends Command
|
||||
{
|
||||
@@ -12,7 +15,10 @@ class ImportNodelist extends Command
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'import:nodelist {file : Nodelist File}';
|
||||
protected $signature = 'import:nodelist'
|
||||
.' {domain : Domain Name}'
|
||||
.' {file : Nodelist File}'
|
||||
.' {--D|delete : Delete old data for the date}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
@@ -38,135 +44,9 @@ class ImportNodelist extends Command
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$file = $this->argument('file');
|
||||
$lines = intval(exec("wc -l '$file'"));
|
||||
$do = Domain::where('name',$this->argument('domain'))->singleOrFail();
|
||||
$o = Nodelist::firstOrCreate(['date'=>Carbon::now(),'domain_id'=>$do->id]);
|
||||
|
||||
$file = fopen($file,"r");
|
||||
$bar = $this->output->createProgressBar($lines);
|
||||
$bar->setFormat("%current%/%max% [%bar%] %percent:3s%% (%memory%) (%remaining%) ");
|
||||
$bar->setRedrawFrequency(100);
|
||||
$bar->start();
|
||||
|
||||
$zone = $region = $host = NULL;
|
||||
|
||||
while (! feof($file)) {
|
||||
$line = stream_get_line($file, 0, "\r\n");
|
||||
|
||||
// Lines beginning with a semicolon(;) are comments
|
||||
if (preg_match('/^;/',$line) OR ($line == chr(0x1A)))
|
||||
continue;
|
||||
|
||||
$fields = explode(',',$line);
|
||||
|
||||
$o = new Node();
|
||||
|
||||
// First field is either zone,region,host,hub,down,pvt (or blank)
|
||||
if ($fields[0] AND ! in_array($fields[0],['Zone','Region','Host','Hub','Pvt','Down']))
|
||||
{
|
||||
$this->error(sprintf('Invalid field zero [%s] - IGNORING record (%s)',$fields[0],$line));
|
||||
$bar->advance();
|
||||
continue;
|
||||
}
|
||||
|
||||
$status = NULL;
|
||||
$node = 0;
|
||||
|
||||
switch ($fields[0])
|
||||
{
|
||||
case 'Zone': $zone = $fields[1];
|
||||
$zo = Zone::firstOrCreate([
|
||||
'id'=>$zone
|
||||
]);
|
||||
break;
|
||||
case 'Host': $host = $fields[1]; break;
|
||||
case 'Region': $region = $fields[1]; break;
|
||||
|
||||
case 'Down':
|
||||
case 'Pvt': $status = $fields[0];
|
||||
case '':
|
||||
$node = $fields[1];
|
||||
break;
|
||||
|
||||
default:
|
||||
$this->error(sprintf('Unhandled first field [%s]',$fields[0]));
|
||||
$bar->advance();
|
||||
continue 2;
|
||||
}
|
||||
|
||||
if (! $zone)
|
||||
{
|
||||
$this->error('Zone NOT set, ignoring record...');
|
||||
$bar->advance();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (! $host)
|
||||
{
|
||||
$host = $zone;
|
||||
}
|
||||
|
||||
if (! $region)
|
||||
{
|
||||
$region = 0 ;
|
||||
}
|
||||
|
||||
$o = Node::firstOrNew([
|
||||
'zone_id'=>$zo->id,
|
||||
'host_id'=>$host,
|
||||
'node_id'=>$node,
|
||||
]);
|
||||
|
||||
$o->region_id = $region;
|
||||
$o->system = $fields[2];
|
||||
$o->location = $fields[3];
|
||||
$o->sysop = $fields[4];
|
||||
$o->active = TRUE;
|
||||
$o->status = $status;
|
||||
|
||||
if (! in_array($fields[5],['-Unpublished-']))
|
||||
$o->phone = $fields[5];
|
||||
|
||||
$o->baud = $fields[6];
|
||||
$o->save();
|
||||
|
||||
// Fields 7 onwards are flags
|
||||
$flags = collect();
|
||||
for ($i=7;$i<count($fields);$i++)
|
||||
{
|
||||
$value = NULL;
|
||||
if (preg_match('/^[A-Z]+:/',$fields[$i]))
|
||||
{
|
||||
list($flag,$value) = explode(':',$fields[$i],2);
|
||||
} else {
|
||||
$flag = $fields[$i];
|
||||
}
|
||||
|
||||
$fo = Flag::firstOrNew(
|
||||
['flag'=>$flag]
|
||||
);
|
||||
|
||||
if (! $flag)
|
||||
{
|
||||
$this->warn('Ignoring blank flag on: '.$line);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (! $fo->exists)
|
||||
{
|
||||
$fo->description = 'Auto Created on Import';
|
||||
$fo->save();
|
||||
}
|
||||
|
||||
$flags->put($fo->id,['arguments'=>$value]);
|
||||
}
|
||||
|
||||
$o->flags()->sync($flags);
|
||||
|
||||
$bar->advance();
|
||||
}
|
||||
|
||||
fclose($file);
|
||||
$bar->finish();
|
||||
echo "\n";
|
||||
return Job::dispatchSync($do,$o,$this->argument('file'),$this->option('delete'));
|
||||
}
|
||||
}
|
||||
231
app/Jobs/ImportNodelist.php
Normal file
231
app/Jobs/ImportNodelist.php
Normal file
@@ -0,0 +1,231 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldBeUnique;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
use App\Http\Controllers\DomainController;
|
||||
use App\Models\{Address,Domain,Nodelist,System,Zone};
|
||||
use App\Traits\Import as ImportTrait;
|
||||
|
||||
class ImportNodelist implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
use ImportTrait;
|
||||
|
||||
protected const LOGKEY = 'JIN';
|
||||
private const importkey = 'nodelist';
|
||||
private string $file;
|
||||
private Domain $do;
|
||||
private Nodelist $no;
|
||||
private bool $deletefile = FALSE;
|
||||
|
||||
/**
|
||||
* Import Nodelist constructor.
|
||||
*
|
||||
* @param Domain $do
|
||||
* @param Nodelist $no
|
||||
* @param string $file
|
||||
* @param bool $delete
|
||||
*/
|
||||
public function __construct(Domain $do,Nodelist $no,string $file,bool $delete=TRUE)
|
||||
{
|
||||
$this->do = $do;
|
||||
$this->no = $no;
|
||||
$this->file = $file;
|
||||
|
||||
if ($delete)
|
||||
$no->addresses()->detach();
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*
|
||||
* @return void
|
||||
* @throws \Exception
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
// Get the file from the host
|
||||
$file = $this->getFileFromHost('',self::importkey,$this->file);
|
||||
$lines = $this->getFileLines($file);
|
||||
Log::debug(sprintf('%s:Processing [%d] lines.',static::LOGKEY,$lines));
|
||||
|
||||
$fh = fopen($file,'r');
|
||||
$c =0;
|
||||
|
||||
$region = NULL;
|
||||
$host = NULL;
|
||||
$hub_id = NULL;
|
||||
|
||||
while (! feof($fh)) {
|
||||
$line = stream_get_line($fh, 0, "\r\n");
|
||||
|
||||
// Lines beginning with a semicolon(;) are comments
|
||||
if (preg_match('/^;/',$line) OR ($line == chr(0x1a)))
|
||||
continue;
|
||||
|
||||
// Remove any embedded CR and BOM
|
||||
$line = str_replace("\r",'',$line);
|
||||
$line = preg_replace('/^\x{feff}/u','',$line);
|
||||
$c++;
|
||||
|
||||
$fields = str_getcsv(trim($line));
|
||||
|
||||
// First field is either zone,region,host,hub,down,pvt (or blank)
|
||||
if ($fields[0] AND ! in_array($fields[0],Nodelist::definitions)) {
|
||||
Log::error(sprintf('%s:Invalid field zero [%s] - IGNORING record (%s)',self::LOGKEY,$fields[0],$line));
|
||||
continue;
|
||||
}
|
||||
|
||||
$node = 0;
|
||||
|
||||
switch ($fields[0]) {
|
||||
case 'Zone': $zone = $fields[1];
|
||||
// We ignore the Zone entries, since they are dynamically created.
|
||||
$zo = Zone::firstOrCreate([
|
||||
'zone_id'=>$zone,
|
||||
'domain_id'=>$this->do->id,
|
||||
]);
|
||||
|
||||
$region = 0;
|
||||
$host = 0;
|
||||
$hub_id = NULL;
|
||||
$role = DomainController::NODE_ZC;
|
||||
|
||||
continue 2;
|
||||
|
||||
case 'Region':
|
||||
$region = $fields[1];
|
||||
$role = DomainController::NODE_RC;
|
||||
$host = 0;
|
||||
$hub_id = NULL;
|
||||
|
||||
break;
|
||||
|
||||
case 'Host':
|
||||
$host = $fields[1];
|
||||
$hub_id = NULL;
|
||||
$role = DomainController::NODE_NC;
|
||||
|
||||
// We ignore the Host entries, since they are dynamically created.
|
||||
continue 2;
|
||||
|
||||
case 'Hub':
|
||||
$role = DomainController::NODE_HC;
|
||||
|
||||
break;
|
||||
|
||||
case 'Pvt':
|
||||
$node = $fields[1];
|
||||
$role = DomainController::NODE_PVT;
|
||||
|
||||
break;
|
||||
|
||||
case 'Down':
|
||||
$node = $fields[1];
|
||||
$role = DomainController::NODE_DOWN;
|
||||
|
||||
break;
|
||||
|
||||
case '':
|
||||
$node = $fields[1];
|
||||
break;
|
||||
|
||||
default:
|
||||
Log::error(sprintf('%s:Unhandled first field [%s]',self::LOGKEY,$fields[0]));
|
||||
continue 2;
|
||||
}
|
||||
|
||||
if (! $zone) {
|
||||
Log::error(sprintf('%s:Zone NOT set, ignoring record...',self::LOGKEY));
|
||||
continue;
|
||||
}
|
||||
|
||||
Address::unguard();
|
||||
$ao = Address::firstOrNew([
|
||||
'zone_id' => $zo->id,
|
||||
'region_id' => $region,
|
||||
'host_id' => $host,
|
||||
'node_id' => $node,
|
||||
'point_id' => 0,
|
||||
]);
|
||||
Address::reguard();
|
||||
|
||||
$ao->active = TRUE;
|
||||
$ao->role = $role;
|
||||
$ao->hub_id = $hub_id;
|
||||
|
||||
$role = NULL;
|
||||
|
||||
// Get the System
|
||||
if ($ao->system_id && (($ao->system->sysop === str_replace('_',' ',$fields[4])) || ($ao->system->name !== str_replace('_',' ',$fields[2])))) {
|
||||
$so = $ao->system;
|
||||
|
||||
// If the sysop name is different
|
||||
if ($so->sysop !== str_replace('_',' ',$fields[4])) {
|
||||
$so->sysop = str_replace('_',' ',$fields[4]);
|
||||
$so->location = str_replace('_',' ',$fields[3]);
|
||||
|
||||
// We have the same name has changed.
|
||||
} else {
|
||||
$so->name = str_replace('_',' ',$fields[2]);
|
||||
$so->location = str_replace('_',' ',$fields[3]);
|
||||
}
|
||||
|
||||
// We'll search and see if we already have that system
|
||||
} else {
|
||||
$so = System::where('name',str_replace('_',' ',$fields[2]))
|
||||
->where('sysop',str_replace('_',' ',$fields[4]))
|
||||
->firstOrNew();
|
||||
|
||||
$so->name = str_replace('_',' ',$fields[2]);
|
||||
$so->sysop = str_replace('_',' ',$fields[4]);
|
||||
$so->location = str_replace('_',' ',$fields[3]);
|
||||
$so->active = TRUE;
|
||||
|
||||
if (! $so->exists)
|
||||
$so->notes = sprintf('Created by Nodelist Import: %d',$this->no->id);
|
||||
}
|
||||
|
||||
/*
|
||||
if (! in_array($fields[5],['-Unpublished-']))
|
||||
$so->phone = $fields[5];
|
||||
|
||||
$so->baud = $fields[6];
|
||||
*/
|
||||
|
||||
// Save the system record
|
||||
$so->save();
|
||||
|
||||
try {
|
||||
$so->addresses()->save($ao);
|
||||
if ($role == DomainController::NODE_HC)
|
||||
$hub_id = $ao->id;
|
||||
|
||||
$this->no->addresses()->attach($ao,['role'=>$role]);
|
||||
|
||||
} catch (\Exception $e) {
|
||||
Log::error(sprintf('%s:Error with line [%s] (%s)',self::LOGKEY,$line,$e->getMessage()),['fields'=>$fields]);
|
||||
throw new \Exception($e->getMessage());
|
||||
}
|
||||
|
||||
if (! ($c % 100)) {
|
||||
Log::notice(sprintf('%s:Processed [%s] records',self::LOGKEY,$c),['memory'=>memory_get_usage(TRUE)]);
|
||||
}
|
||||
}
|
||||
|
||||
fclose($fh);
|
||||
|
||||
if ($this->deletefile and $c)
|
||||
unlink($file);
|
||||
|
||||
Log::info(sprintf('%s:Records Updated: %d',self::LOGKEY,$c));
|
||||
}
|
||||
}
|
||||
@@ -59,6 +59,10 @@ class Address extends Model
|
||||
return 'Host';
|
||||
case DomainController::NODE_HC;
|
||||
return 'Hub';
|
||||
case DomainController::NODE_PVT;
|
||||
return 'PVT';
|
||||
case DomainController::NODE_DOWN;
|
||||
return 'DOWN';
|
||||
case NULL:
|
||||
return 'Node';
|
||||
default:
|
||||
|
||||
20
app/Models/Nodelist.php
Normal file
20
app/Models/Nodelist.php
Normal file
@@ -0,0 +1,20 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
|
||||
class Nodelist extends Model
|
||||
{
|
||||
protected $dates = ['date'];
|
||||
protected $fillable = ['date','domain_id'];
|
||||
|
||||
public const definitions = ['Zone','Region','Host','Hub','Pvt','Down'];
|
||||
|
||||
/* RELATIONS */
|
||||
|
||||
public function addresses()
|
||||
{
|
||||
return $this->belongsToMany(Address::class);
|
||||
}
|
||||
}
|
||||
@@ -31,7 +31,7 @@ class System extends Model
|
||||
* @param Address $o
|
||||
* @return string
|
||||
*/
|
||||
public function name(Address $o): string
|
||||
public function full_name(Address $o): string
|
||||
{
|
||||
switch ($o->attributes['role']) {
|
||||
case DomainController::NODE_ZC;
|
||||
|
||||
82
app/Traits/Import.php
Normal file
82
app/Traits/Import.php
Normal file
@@ -0,0 +1,82 @@
|
||||
<?php
|
||||
|
||||
/**
|
||||
* Import from files
|
||||
*/
|
||||
namespace App\Traits;
|
||||
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
|
||||
trait Import
|
||||
{
|
||||
protected Collection $_columns; // Columns in the Import File
|
||||
|
||||
/**
|
||||
* Count the lines in a file
|
||||
*/
|
||||
private function getFileLines(string $file): int
|
||||
{
|
||||
$f = fopen($file,'r');
|
||||
$c = 0;
|
||||
|
||||
while (! feof($f)) {
|
||||
fgets($f);
|
||||
$c++;
|
||||
}
|
||||
|
||||
fclose($f);
|
||||
|
||||
return $c;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the columns from the file that we'll work with
|
||||
*
|
||||
* @param string $line
|
||||
* @return Collection
|
||||
*/
|
||||
private function getColumns(string $line): Collection
|
||||
{
|
||||
$this->_columns = collect(explode(',',strtoupper($line)))->filter();
|
||||
return $this->_columns->intersect($this->columns);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the index for the column in the file
|
||||
*
|
||||
* @param string $key
|
||||
* @return int|null
|
||||
*/
|
||||
private function getColumnKey(string $key): ?int
|
||||
{
|
||||
return ($x=$this->_columns->search(strtoupper($this->columns->get($key)))) !== FALSE ? $x : NULL;
|
||||
}
|
||||
|
||||
private function getFileFromHost(string $host,string $key,string $file): string
|
||||
{
|
||||
$path = 'import/'.$key;
|
||||
|
||||
// If we are doing it locally, we dont need to retrieve it via curl
|
||||
if (! $host) {
|
||||
return preg_match('/^data/',$file) ? $file : sprintf('storage/app/public/%s/%s',$path,basename($file));
|
||||
}
|
||||
|
||||
// Get the file from the host
|
||||
$srcfile = sprintf('http://%s%s',$host,$file);
|
||||
$dstfile = '/tmp/'.basename($host);
|
||||
Log::debug(sprintf('%s:Retrieving [%s] from [%s]',static::LOGKEY,$host,$file));
|
||||
|
||||
$src = fopen($srcfile,'r');
|
||||
$dst = fopen($dstfile,'w');
|
||||
stream_copy_to_stream($src,$dst);
|
||||
fclose($src);
|
||||
fclose($dst);
|
||||
|
||||
// Store the file for later processing
|
||||
Storage::putFileAs($path,$dstfile,basename($file));
|
||||
|
||||
return Storage::path($path.'/'.basename($file));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user