Major refactor of photo processing, video processing still to do
This commit is contained in:
@@ -1,45 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
use App\Traits\Type;
|
||||
|
||||
class CatalogDump extends Command
|
||||
{
|
||||
use Type;
|
||||
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'catalog:dump {type : Photo | Video } {id : Photo ID}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Scan Photo for metadata';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*
|
||||
* @return mixed
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$class = $this->getModelType($this->argument('type'));
|
||||
|
||||
$o = $class::findOrFail($this->argument('id'));
|
||||
|
||||
if (! $o->isReadable()) {
|
||||
$this->warn(sprintf('Ignoring [%s], it is not readable',$o->file_path()));
|
||||
exit;
|
||||
}
|
||||
|
||||
dump($o->properties());
|
||||
}
|
||||
}
|
@@ -4,6 +4,7 @@ namespace App\Console\Commands;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
|
||||
use App\Jobs\CatalogScan as Job;
|
||||
use App\Traits\Type;
|
||||
|
||||
class CatalogScan extends Command
|
||||
@@ -25,7 +26,7 @@ class CatalogScan extends Command
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Scan Photo for metadata';
|
||||
protected $description = 'Scan Photo/Video for metadata';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
@@ -38,54 +39,6 @@ class CatalogScan extends Command
|
||||
|
||||
$o = $class::findOrFail($this->argument('id'));
|
||||
|
||||
if (! $o->isReadable()) {
|
||||
$this->warn(sprintf('Ignoring [%s], it is not readable',$o->file_path()));
|
||||
return;
|
||||
}
|
||||
|
||||
$o->setDateCreated();
|
||||
$o->setSubSecTime();
|
||||
$o->setSignature();
|
||||
$o->setMakeModel();
|
||||
$o->setLocation();
|
||||
$o->setHeightWidth();
|
||||
$o->setThumbnail();
|
||||
|
||||
// If this is a duplicate
|
||||
$x = $o->myduplicates()->get();
|
||||
if (count($x)) {
|
||||
foreach ($x as $oo) {
|
||||
// And that photo is not marked as a duplicate
|
||||
if (! $oo->duplicate) {
|
||||
$o->duplicate = '1';
|
||||
$this->warn(sprintf('Image [%s] marked as a duplicate',$o->filename));
|
||||
|
||||
// If the file signature also matches, we'll mark it for deletion
|
||||
if ($oo->file_signature AND $o->file_signature == $oo->file_signature) {
|
||||
$this->warn(sprintf('Image [%s] marked for deletion',$o->filename));
|
||||
$o->remove = '1';
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$o->scanned = '1';
|
||||
|
||||
if ($o->getDirty()) {
|
||||
$this->warn(sprintf('Image [%s] metadata changed',$o->filename));
|
||||
|
||||
if ($this->option('dirty'))
|
||||
dump(['id'=>$o->id,'data'=>$o->getDirty()]);
|
||||
}
|
||||
|
||||
// If the file signature changed, abort the update.
|
||||
if ($o->getOriginal('file_signature') AND $o->wasChanged('file_signature')) {
|
||||
dump(['old'=>$o->getOriginal('file_signature'),'new'=>$o->file_signature]);
|
||||
abort(500,'File Signature Changed?');
|
||||
}
|
||||
|
||||
$o->save();
|
||||
return Job::dispatchSync($o);
|
||||
}
|
||||
}
|
@@ -2,9 +2,12 @@
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use Carbon\Carbon;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Foundation\Bus\DispatchesJobs;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
|
||||
use App\Jobs\CatalogScan;
|
||||
use App\Traits\Type;
|
||||
@@ -13,14 +16,18 @@ class CatalogScanAll extends Command
|
||||
{
|
||||
use DispatchesJobs,Type;
|
||||
|
||||
private int|bool $depth = true;
|
||||
protected const chunk_size = 5;
|
||||
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'catalog:scanall'.
|
||||
' {type : Photo | Video }'.
|
||||
' {--scanned : Rescan Scanned Photos}';
|
||||
protected $signature = 'catalog:scanall'
|
||||
.' {type : Photo | Video }'
|
||||
.' {--i|ignore : Ignore missing files}'
|
||||
.' {--s|scan : Force Rescan of all files }';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
@@ -30,40 +37,128 @@ class CatalogScanAll extends Command
|
||||
protected $description = '(re)Scan Media';
|
||||
|
||||
/**
|
||||
* Create a new command instance.
|
||||
* Execute the console command.
|
||||
*
|
||||
* @return void
|
||||
* @return int
|
||||
*/
|
||||
public function __construct()
|
||||
public function handle(): int
|
||||
{
|
||||
parent::__construct();
|
||||
$started = Carbon::now();
|
||||
$class = $this->getModelType($this->argument('type'));
|
||||
|
||||
Log::info('Scanning disk: '.Storage::disk('nas')->path(''));
|
||||
|
||||
$c = 0;
|
||||
// Scan files in dir, and make sure file lives in DB, (touch it if it does), otherwise create it
|
||||
foreach (Storage::disk('nas')->directories($class::dir_prefix()) as $dir) {
|
||||
Log::info(sprintf(' - DIR: %s',$dir));
|
||||
|
||||
// Take x files at a time and check the DB
|
||||
foreach ($this->files($dir,$class::config,$class::dir_prefix())->chunk(self::chunk_size) as $chunk) {
|
||||
$list = $class::whereIn('filename',$chunk)->get();
|
||||
|
||||
// If there is a new file found it wont be in the DB
|
||||
if ($list->count() !== self::chunk_size)
|
||||
foreach ($chunk->diff($list->pluck('filename')) as $file) {
|
||||
Log::info(sprintf('Found new file [%s] - queueing scan',$file));
|
||||
|
||||
$o = new $class;
|
||||
$o->filename = $file;
|
||||
$o->file_signature = $o->getObjectOriginal('file_signature');
|
||||
$o->save();
|
||||
|
||||
CatalogScan::dispatch($o)
|
||||
->onQueue('scan');
|
||||
|
||||
$c++;
|
||||
}
|
||||
|
||||
foreach ($list as $o) {
|
||||
// Check the details are valid
|
||||
if ($o->file_signature === $o->getObjectOriginal('file_signature')) {
|
||||
// For sanity, we'll check a couple of other attrs
|
||||
if (($o->width !== $o->getObjectOriginal('width')) || ($o->height !== $o->getObjectOriginal('height')))
|
||||
Log::alert(sprintf('Dimensions [%s] (%s x %s) mismatch for [%s]',
|
||||
$o->dimensions,
|
||||
$o->getObjectOriginal('width'),
|
||||
$o->getObjectOriginal('height'),
|
||||
$o->file_name(FALSE)));
|
||||
|
||||
} else {
|
||||
Log::alert(sprintf('File Signature [%s] doesnt match [%s] for [%s]',
|
||||
$o->getObjectOriginal('file_signature'),
|
||||
$o->file_signature,
|
||||
$o->file_name(FALSE)));
|
||||
}
|
||||
|
||||
if ($o->signature !== $o->getObjectOriginal('signature')) {
|
||||
Log::notice(sprintf('Updating image signature for [%s] to [%s] was [%s]',$o->filename,$o->signature,$o->getObjectOriginal('signature')));
|
||||
|
||||
$o->signature = $o->getObjectOriginal('signature');
|
||||
}
|
||||
|
||||
if ($o->isDirty())
|
||||
$o->save();
|
||||
else
|
||||
$o->touch();
|
||||
|
||||
if ($this->option('scan')) {
|
||||
Log::info(sprintf('Forcing re-scan of [%s] - queued',$o->filename));
|
||||
|
||||
CatalogScan::dispatch($o)
|
||||
->onQueue('scan');
|
||||
}
|
||||
|
||||
$c++;
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
Log::info('Checking for missing files');
|
||||
|
||||
// Find DB records before $started, check they exist (they shouldnt), and delete if not
|
||||
if (! $this->option('ignore'))
|
||||
foreach ($class::select(['id','filename'])->where('updated_at','<',$started)->cursor() as $o)
|
||||
Log::error(sprintf('It appears that file [%s] is missing (%d)',$o->filename,$o->id));
|
||||
|
||||
Log::info(sprintf('Processed [%s]',$c));
|
||||
|
||||
return self::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
* Recursively find files that we should catalog
|
||||
*
|
||||
* @return mixed
|
||||
* @param string $dir Directory to get files from
|
||||
* @param string $type Configuration key to refer to config())
|
||||
* @param string $prefix Remove the prefix from the filename
|
||||
* @return Collection
|
||||
*/
|
||||
public function handle()
|
||||
public function files(string $dir,string $type,string $prefix): Collection
|
||||
{
|
||||
$class = $this->getModelType($this->argument('type'));
|
||||
$files = collect(Storage::disk('nas')->files($dir))
|
||||
->map(fn($item)=>preg_replace('#^'.$prefix.'#','',$item))
|
||||
->filter(function($item) use ($type) {
|
||||
return ((! ($x=strrpos($item,'.')))
|
||||
|| (! in_array(strtolower(substr($item,$x+1)),config($type.'.import.accepted'))))
|
||||
? NULL
|
||||
: $item;
|
||||
});
|
||||
|
||||
if ($this->option('scanned')) {
|
||||
$class::whereNotNull('scanned')
|
||||
->update(['scanned'=>NULL]);
|
||||
}
|
||||
if (! $this->depth)
|
||||
return $files;
|
||||
|
||||
$c = 0;
|
||||
$class::NotScanned()->each(function ($item) use ($c) {
|
||||
if ($item->remove) {
|
||||
Log::warning(sprintf('Not scanning [%s], marked for removal',$item->id));
|
||||
return;
|
||||
}
|
||||
if (is_numeric($this->depth))
|
||||
$this->depth--;
|
||||
|
||||
$this->dispatch((new CatalogScan($item))->onQueue('scan'));
|
||||
$c++;
|
||||
});
|
||||
foreach (Storage::disk('nas')->directories($dir) as $dir)
|
||||
$files = $files->merge($this->files($dir,$type,$prefix));
|
||||
|
||||
Log::info(sprintf('Processed [%s]',$c));
|
||||
if (is_numeric($this->depth))
|
||||
$this->depth++;
|
||||
|
||||
return $files;
|
||||
}
|
||||
}
|
@@ -1,50 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Foundation\Bus\DispatchesJobs;
|
||||
|
||||
use App\Jobs\CatalogVerify as Job;
|
||||
|
||||
class CatalogVerify extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'catalog:verify'.
|
||||
' {type : Photo | Video }';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Verify media on disk and in the DB';
|
||||
|
||||
/**
|
||||
* Create a new command instance.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*
|
||||
* @return mixed
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
if ($this->argument('type')) {
|
||||
Job::dispatch($this->argument('type'))->onQueue('scan');
|
||||
|
||||
} else {
|
||||
}
|
||||
}
|
||||
}
|
@@ -1,107 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use Log;
|
||||
use Illuminate\Console\Command;
|
||||
use App\Model\Photo;
|
||||
|
||||
class PhotoUpdate extends Command
|
||||
{
|
||||
use \App\Traits\Files;
|
||||
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'photo:update
|
||||
{--dir= : Directory to Parse}
|
||||
{--file= : File to import}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Update Signatures';
|
||||
|
||||
/**
|
||||
* Create a new command instance.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*
|
||||
* @return mixed
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$files = $this->getFiles(['dir'=>$this->option('dir'),'file'=>$this->option('file')]);
|
||||
if (! count($files))
|
||||
exit;
|
||||
|
||||
// Show a progress bar
|
||||
$bar = $this->output->createProgressBar(count($files));
|
||||
$bar->setFormat("%current%/%max% [%bar%] %percent:3s%% (%memory%) (%remaining%) ");
|
||||
|
||||
$c = 0;
|
||||
foreach ($files as $file)
|
||||
{
|
||||
$bar->advance();
|
||||
|
||||
if (preg_match('/@__thumb/',$file) OR preg_match('/\/._/',$file))
|
||||
{
|
||||
$this->warn(sprintf('Ignoring file [%s]',$file));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (! in_array(strtolower(pathinfo($file,PATHINFO_EXTENSION)),config('photo.import.accepted')))
|
||||
{
|
||||
$this->warn(sprintf('Ignoring [%s]',$file));
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($this->option('verbose'))
|
||||
$this->info(sprintf('Processing file [%s]',$file));
|
||||
|
||||
$c++;
|
||||
|
||||
$po = Photo::where('filename',$file)->first();
|
||||
|
||||
if (is_null($po))
|
||||
{
|
||||
$this->error(sprintf('File is not in the database [%s]?',$file));
|
||||
Log::error(sprintf('%s: File is not in the database [%s]?',__METHOD__,$file));
|
||||
continue;
|
||||
}
|
||||
|
||||
$po->signature = $po->property('signature');
|
||||
|
||||
try {
|
||||
$po->thumbnail = exif_thumbnail($po->file_path());
|
||||
} catch (\Exception $e) {
|
||||
// @todo Couldnt get the thumbnail, so we should create one.
|
||||
}
|
||||
|
||||
if ($po->isDirty())
|
||||
{
|
||||
if (count($po->getDirty()) > 1 OR ! array_key_exists('signature',$po->getDirty()))
|
||||
$this->error(sprintf('More than the signature changed for [%s] (%s)?',$po->id,join('|',array_keys($po->getDirty()))));
|
||||
|
||||
$this->info(sprintf('Signature update for [%s]',$po->id));
|
||||
$po->save();
|
||||
}
|
||||
}
|
||||
|
||||
$bar->finish();
|
||||
|
||||
return $this->info(sprintf('Images processed: %s',$c));
|
||||
}
|
||||
}
|
Reference in New Issue
Block a user