diff --git a/SQL/SQLite/schema_clear.sql b/SQL/SQLite/schema_clear.sql index f06c00f1f40..543f4e3b625 100644 --- a/SQL/SQLite/schema_clear.sql +++ b/SQL/SQLite/schema_clear.sql @@ -23,6 +23,7 @@ DELETE FROM genre_track; DELETE FROM comments; DELETE FROM scanned_files; +DELETE FROM scanned_pics; DELETE FROM works; diff --git a/SQL/SQLite/schema_optimize.sql b/SQL/SQLite/schema_optimize.sql index 4f3ae990d8d..7882bc7bf92 100644 --- a/SQL/SQLite/schema_optimize.sql +++ b/SQL/SQLite/schema_optimize.sql @@ -3,6 +3,8 @@ -- This is done here as it is faster to do in sql than in the server. -- +DELETE FROM scanned_pics; + -- XXX This appears to not be needed anymore as contributors are properly -- removed by the new scanner diff --git a/SQL/SQLite/schema_scanner.sql b/SQL/SQLite/schema_scanner.sql new file mode 100644 index 00000000000..b25cf70341e --- /dev/null +++ b/SQL/SQLite/schema_scanner.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS scanned_files; +CREATE TABLE scanned_files ( + url text NOT NULL COLLATE NOCASE, -- URL must be case insensitive, or we might duplicate tracks if the filename changes case only (https://github.com/LMS-Community/slimserver/issues/705#issuecomment-1026229542) + timestamp int(10), + filesize int(10) +); +CREATE INDEX scannedUrlIndex ON scanned_files (url); + +DROP TABLE IF EXISTS scanned_pics; +CREATE TABLE scanned_pics ( + url text NOT NULL, + timestamp int(10), + filesize int(10) +); +CREATE INDEX scannedPicUrlIndex ON scanned_pics (url); diff --git a/SQL/mysql/schema_clear.sql b/SQL/mysql/schema_clear.sql index f0a7d886c0f..f1efdc6b6b8 100644 --- a/SQL/mysql/schema_clear.sql +++ b/SQL/mysql/schema_clear.sql @@ -26,6 +26,7 @@ DELETE FROM pluginversion; DELETE FROM unreadable_tracks; DELETE FROM scanned_files; +DELETE FROM scanned_pics; DELETE FROM works; diff --git a/SQL/mysql/schema_optimize.sql b/SQL/mysql/schema_optimize.sql index fca7a1d23af..4727c6b32d0 100644 --- a/SQL/mysql/schema_optimize.sql +++ b/SQL/mysql/schema_optimize.sql @@ -3,6 +3,8 @@ -- This is done here as it is faster to do in sql than in the server. -- +DELETE FROM scanned_pics; + -- XXX This appears to not be needed anymore as contributors are properly -- removed by the new scanner diff --git a/SQL/mysql/schema_scanner.sql b/SQL/mysql/schema_scanner.sql new file mode 100644 index 00000000000..87502086c8d --- /dev/null +++ b/SQL/mysql/schema_scanner.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS scanned_files; +CREATE TABLE scanned_files ( + url text NOT NULL COLLATE NOCASE, -- URL must be case insensitive, or we might duplicate tracks if the filename changes case only (https://github.com/LMS-Community/slimserver/issues/705#issuecomment-1026229542) + timestamp int(10), + filesize int(10) +); +CREATE INDEX scannedUrlIndex ON scanned_files (url); + +DROP TABLE IF EXISTS scanned_pics; +CREATE TABLE scanned_pics ( + url text NOT NULL COLLATE NOCASE, -- URL must be case insensitive, or we might duplicate tracks if the filename changes case only (https://github.com/LMS-Community/slimserver/issues/705#issuecomment-1026229542) + timestamp int(10), + filesize int(10) +); +CREATE INDEX scannedPicUrlIndex ON scanned_pics (url); diff --git a/Slim/Music/Artwork.pm b/Slim/Music/Artwork.pm index 766165737e4..727fbe7601d 100644 --- a/Slim/Music/Artwork.pm +++ b/Slim/Music/Artwork.pm @@ -54,23 +54,28 @@ tie my %lastFile, 'Tie::Cache::LRU', 128; # scans of files in the same directory # Don't use Tie::Cache::LRU as it is a bit too expensive in the scanner my %findArtCache; +my $artFolderRead; +my $imageTypesRegex; # Public class methods sub findStandaloneArtwork { my ( $class, $trackAttributes, $deferredAttributes, $dirurl ) = @_; - return 0 if !Slim::Music::Info::isFileURL($dirurl); + return wantarray ? () : 0 if !Slim::Music::Info::isFileURL($dirurl); my $isInfo = main::INFOLOG && $log->is_info; my $art = $findArtCache{$dirurl}; # Files to look for - my @files = qw(cover folder album thumb); + my @files = qw(cover album folder thumb); # User-defined artwork format my $coverFormat = $prefs->get('coverArt'); + my $artDir = $prefs->get('artfolder'); + my $candidateForArtfolder; + if ( !defined $art ) { my $parentDir = Path::Class::dir( Slim::Utils::Misc::pathFromFileURL($dirurl) ); @@ -80,7 +85,7 @@ sub findStandaloneArtwork { # to generate that pattern. This is nasty. if ( $coverFormat =~ /^%(.*?)(\..*?){0,1}$/ ) { my $formatStr = $1; - my $suffix = $2 ? $2 : '.jpg'; + my $suffix = $2; my $track = $trackAttributes && delete $trackAttributes->{_track}; @@ -88,31 +93,32 @@ sub findStandaloneArtwork { # XXX This may break for some people as it's not using a Track object anymore my $meta = { %{$trackAttributes}, %{$deferredAttributes} } unless $track; - if ( my $prefix = Slim::Music::TitleFormatter::infoFormat( $track, $formatStr, undef, $meta ) ) { - $coverFormat = $prefix . $suffix; + if ( my $coverName = Slim::Music::TitleFormatter::infoFormat( $track, $formatStr, undef, $meta ) ) { + $coverName .= $suffix; if ( main::ISWINDOWS ) { # Remove illegal characters from filename. - $coverFormat =~ s/\\|\/|\:|\*|\?|\"|<|>|\|//g; + $coverName =~ s/\\|\/|\:|\*|\?|\"|<|>|\|//g; } # Generating a pathname from tags is dangerous because the filesystem # encoding may not match the locale, but that is the best guess that we have. - $coverFormat = Slim::Utils::Unicode::encode_locale($coverFormat); + $coverName = Slim::Utils::Unicode::encode_locale($coverName); - my $artPath = $parentDir->file($coverFormat)->stringify; + unshift @files, $coverName; - if ( my $artDir = $prefs->get('artfolder') ) { - $artDir = Path::Class::dir($artDir); - $artPath = $artDir->file($coverFormat)->stringify; - } + if ( $artDir && -d $artDir ) { + $candidateForArtfolder = $coverName; - if ( -e $artPath ) { - $isInfo && $log->info("Found variable cover $coverFormat from $1"); - $art = $artPath; - } - else { - $isInfo && $log->info("No variable cover $coverFormat found from $1"); + # add the content of the artwork folder to our scanned picture table + if (main::SCANNER && !$artFolderRead) { + Slim::Utils::Scanner::Local::Async->find( $artDir, { + types => 'image', + no_async => 1, + }, sub {} ); + + $artFolderRead = 1; + } } } else { @@ -125,34 +131,35 @@ sub findStandaloneArtwork { $coverFormat =~ s/\\|\/|\:|\*|\?|\"|<|>|\|//g; } - push @files, $coverFormat; + unshift @files, $coverFormat; } } - if ( !$art ) { - # Find all image files in the file directory - my $types = qr/\.(?:jpe?g|png|gif)$/i; + $isInfo && $log->info("Looking for artwork in $parentDir: " . join(', ', @files)); - my $files = File::Next::files( { - file_filter => sub { Slim::Utils::Misc::fileFilter($File::Next::dir, $_, $types, undef, 1) }, - descend_filter => sub { 0 }, - }, $parentDir ); - - my @found; - while ( my $image = $files->() ) { - push @found, $image; + if (wantarray) { + my @artFiles = _findStandaloneArtwork($parentDir, \@files); + if ($candidateForArtfolder) { + push @artFiles, _findStandaloneArtwork($artDir, [$candidateForArtfolder]); } + push @artFiles, _findStandaloneArtwork($parentDir); - # Prefer cover/folder/album/thumb, then just take the first image - my $filelist = join( '|', @files ); - if ( my @preferred = grep { basename($_) =~ qr/^(?:$filelist)\./i } @found ) { - $art = $preferred[0]; - } - else { - $art = $found[0] || 0; - } + $isInfo && $log->info("Found artwork files: " . join(', ', @artFiles)); + + return @artFiles; + } + + # look up "artist name" (or whatever the template), cover, album, etc. in music folder first + $art ||= _findStandaloneArtwork($parentDir, \@files); + + # check for "artist name" (or whatever) in the artwork folder (if defined) + if ( !$art && $candidateForArtfolder && $artFolderRead ) { + $art = _findStandaloneArtwork($artDir, [$candidateForArtfolder]); } + # pick any picture in music folder + $art ||= _findStandaloneArtwork($parentDir); + # Cache found artwork for this directory to speed up later tracks # No caching if using a user-defined artwork format, the user may have multiple # files in a single directory with different artwork @@ -167,6 +174,84 @@ sub findStandaloneArtwork { return $art || 0; } +sub _findStandaloneArtwork { + my ($parentDir, $filenameTemplates) = @_; + my $dbh = Slim::Schema->dbh; + + $imageTypesRegex ||= Slim::Music::Info::validTypeExtensions('image'); + + my @candidates = $filenameTemplates ? map { + my $name = $_; + my @variations; + + if ($name =~ $imageTypesRegex) { + push @variations, catfile($parentDir, $name); + } + else { + @variations = map {( + catfile($parentDir, "$name.$_"), + catfile($parentDir, $name . '.' . uc($_)), + )} ('jpg', 'jpeg', 'png', 'gif'); + } + + @variations; + } @$filenameTemplates : (); + + my @images; + + if (main::SCANNER) { + my $sql = 'SELECT url FROM scanned_pics WHERE url '; + if (scalar @candidates) { + $sql .= sprintf('IN (%s)', join(',', map { '?' } @candidates)); + } + else { + # doing a range search helps us avoid a LIKE query, which would result in a scan + $sql .= '>= ? AND url < ?'; + my $pathUrl = Slim::Utils::Misc::fileURLFromPath($parentDir); + push @candidates, $pathUrl, $pathUrl . chr(0xff); + } + + my $sth = Slim::Schema->dbh->prepare_cached($sql); + + @images = Slim::Utils::Misc::uniq(map { + Slim::Utils::Misc::pathFromFileURL($_->[0]); + } @{ + $dbh->selectall_arrayref($sth, undef, map { Slim::Utils::Misc::fileURLFromPath($_) } @candidates) + }); + } + else { + @images = Slim::Utils::Misc::uniq(grep { -f $_ } @candidates); + + # read the folder anyway, as we don't have the full list of images in the database table + if (!scalar @images && !$filenameTemplates) { + my $files = File::Next::files( { + file_filter => sub { Slim::Utils::Misc::fileFilter($File::Next::dir, $_, $imageTypesRegex, undef, 1) }, + descend_filter => sub { 0 }, + }, $parentDir ); + + while ( my $image = $files->() ) { + # just take the first image found... + push @images, $image; + last; + } + } + } + + # keep sort order from the templates list + if (scalar @images > 1) { + my %rank; + @rank{ map { $_ } @$filenameTemplates } = (0 .. $#$filenameTemplates); + + @images = sort { + my $a_rank = $rank{ basename($a) } // 999_999; + my $b_rank = $rank{ basename($b) } // 999_999; + $a_rank <=> $b_rank; + } @images; + } + + return wantarray ? @images : ($images[0] || 0); +} + sub updateStandaloneArtwork { my $class = shift; my $cb = shift; # optional callback when done (main process async mode) @@ -521,114 +606,18 @@ sub _readCoverArtFiles { my $isInfo = main::INFOLOG && $log->is_info; - my @names = qw(cover Cover thumb Thumb album Album folder Folder); - my @ext = qw(png jpg jpeg gif); - - my $file = file($path); - my $parentDir = $file->dir; - my $trackId = $track->id; + my $parentDir = file($path)->dir; $isInfo && $log->info("Looking for image files in $parentDir"); - my %nameslist = map { $_ => [do { my $t = $_; map { "$t.$_" } @ext }] } @names; - - # these seem to be in a particular order - not sure if that means anything. - my @filestotry = map { @{$nameslist{$_}} } @names; - my $artwork = $prefs->get('coverArt'); - - # If the user has specified a pattern to match the artwork on, we need - # to generate that pattern. This is nasty. - if (defined($artwork) && $artwork =~ /^%(.*?)(\..*?){0,1}$/) { - - my $suffix = $2 ? $2 : ".jpg"; - - if (my $prefix = Slim::Music::TitleFormatter::infoFormat( - Slim::Utils::Misc::fileURLFromPath($track->url), $1)) { - - $artwork = $prefix . $suffix; - - $isInfo && $log->info("Variable cover: $artwork from $1"); - - if (main::ISWINDOWS) { - # Remove illegal characters from filename. - $artwork =~ s/\\|\/|\:|\*|\?|\"|<|>|\|//g; - } - - # Generating a pathname from tags is dangerous because the filesystem - # encoding may not match the locale, but that is the best guess that we have. - $artwork = Slim::Utils::Unicode::encode_locale($artwork); - - my $artPath = $parentDir->file($artwork)->stringify; - - my ($body, $contentType) = $class->getImageContentAndType($artPath); - - my $artDir = dir($prefs->get('artfolder')); - - if (!$body && defined $artDir) { - - $artPath = $artDir->file($artwork)->stringify; - - ($body, $contentType) = $class->getImageContentAndType($artPath); - } - - if ($body && $contentType) { - - $isInfo && $log->info("Found image file: $artPath"); - - return ($body, $contentType, $artPath); - } - } else { - - $isInfo && $log->info("Variable cover: no match from $1"); - } + my @candidates = $class->findStandaloneArtwork({ _track => $track }, {}, Slim::Utils::Misc::fileURLFromPath($path)); - } elsif (defined $artwork) { - - unshift @filestotry, $artwork; - } - - if (defined $artworkDir && $artworkDir eq $parentDir) { - - if (exists $lastFile{$trackId} && $lastFile{$trackId} ne 1) { - - $isInfo && $log->info("Using existing image: $lastFile{$trackId}"); - - my ($body, $contentType) = $class->getImageContentAndType($lastFile{$trackId}); - - return ($body, $contentType, $lastFile{$trackId}); - - } elsif (exists $lastFile{$trackId}) { - - $isInfo && $log->info("No image in $artworkDir"); - - return undef; - } - - } else { - - $artworkDir = $parentDir; - %lastFile = (); - } - - for my $file (@filestotry) { - - $file = $parentDir->file($file)->stringify; - - next unless -f $file; - - my ($body, $contentType) = $class->getImageContentAndType($file); + foreach my $artPath (@candidates) { + my ($body, $contentType) = $class->getImageContentAndType($artPath); if ($body && $contentType) { - - $isInfo && $log->info("Found image file: $file"); - - $lastFile{$trackId} = $file; - - return ($body, $contentType, $file); - - } else { - - $lastFile{$trackId} = 1; + $isInfo && $log->info("Found image file: $artPath"); + return ($body, $contentType, $artPath); } } diff --git a/Slim/Music/Import.pm b/Slim/Music/Import.pm index 36fef026b0f..045e4415d68 100644 --- a/Slim/Music/Import.pm +++ b/Slim/Music/Import.pm @@ -499,8 +499,10 @@ sub runScanPostProcessing { } # update standalone artwork if it's been changed without the music file being changed (don't run on a wipe & rescan) - $importsRunning{'updateStandaloneArtwork'} = Time::HiRes::time(); - Slim::Music::Artwork->updateStandaloneArtwork() unless $class->stillScanning =~ /wipe/i; + if ($class->stillScanning !~ /wipe/i) { + $importsRunning{'updateStandaloneArtwork'} = Time::HiRes::time(); + Slim::Music::Artwork->updateStandaloneArtwork(); + } # Pre-cache resized artwork $importsRunning{'precacheArtwork'} = Time::HiRes::time(); diff --git a/Slim/Music/Info.pm b/Slim/Music/Info.pm index 2ebce123add..58dd5adcce3 100644 --- a/Slim/Music/Info.pm +++ b/Slim/Music/Info.pm @@ -1333,6 +1333,17 @@ sub isContainer { return 0; } +sub isImage { + my $pathOrObj = shift; + my $type = shift || _isContentTypeHelper($pathOrObj); + + $type =~ s/jpeg/jpg/i if $type; + + if ($type && $slimTypes{$type} && $slimTypes{$type} eq 'image') { + return $type; + } +} + # Return a list of valid extensions for a particular type as listed in types.conf sub validTypeExtensions { my $findTypes = shift || 'list|audio'; @@ -1430,7 +1441,7 @@ sub typeFromSuffix { my $defaultType = shift || 'unk'; if (defined $path && $path =~ m%\.([^./]+)$%) { - return $suffixes{lc($1)} || $defaultType; + return $suffixes{lc($1)} || $defaultType; } return $defaultType; diff --git a/Slim/Player/Protocols/Volatile.pm b/Slim/Player/Protocols/Volatile.pm index be2b8b047f6..adec3c570ea 100644 --- a/Slim/Player/Protocols/Volatile.pm +++ b/Slim/Player/Protocols/Volatile.pm @@ -78,12 +78,7 @@ sub getArtwork { my ($body, $contentType, $file); eval { - ($body, $contentType, $file) = Slim::Music::Artwork->_readCoverArtTags($track, $path); - - # Nothing there? Look on the file system. - if (!defined $body) { - ($body, $contentType, $file) = Slim::Music::Artwork->_readCoverArtFiles($track, $path); - } + ($body, $contentType, $file) = Slim::Music::Artwork->readCoverArt($track, $path); }; if ($body && defined $file) { diff --git a/Slim/Schema.pm b/Slim/Schema.pm index 29ab9ef606a..0d3ee64cff0 100644 --- a/Slim/Schema.pm +++ b/Slim/Schema.pm @@ -439,6 +439,11 @@ sub migrateDB { my $dbh = $class->storage->dbh; my ($driver, $source, $username, $password) = $class->sourceInformation; + # initialize scanner helper tables + Slim::Utils::SQLHelper->executeSQLFile( + $driver, $class->storage->dbh, "schema_scanner.sql" + ); + # Migrate to the latest schema version - see SQL/$driver/schema_\d+_up.sql my $dbix = DBIx::Migration->new({ dbh => $dbh, diff --git a/Slim/Utils/Scanner/Local/Async.pm b/Slim/Utils/Scanner/Local/Async.pm index d40c7eeb1ba..98f18afa6c7 100644 --- a/Slim/Utils/Scanner/Local/Async.pm +++ b/Slim/Utils/Scanner/Local/Async.pm @@ -13,6 +13,7 @@ package Slim::Utils::Scanner::Local::Async; use strict; +use File::Basename qw(fileparse); use File::Next; use File::Spec (); use Path::Class (); @@ -38,14 +39,21 @@ sub find { # Scanned files are stored in the database, use raw DBI to improve performance here my $dbh = Slim::Schema->dbh; - my $sth = $dbh->prepare_cached( qq{ + my $audioSth = $dbh->prepare_cached( qq{ INSERT INTO scanned_files (url, timestamp, filesize) VALUES (?, ?, ?) } ); - my $types = Slim::Music::Info::validTypeExtensions( $args->{types} || 'audio' ); + my $imageSth = $dbh->prepare_cached( qq{ + INSERT INTO scanned_pics + (url, timestamp, filesize) + VALUES + (?, ?, ?) + } ); + + my $types = Slim::Music::Info::validTypeExtensions( ($args->{types} || 'audio') . '|image' ); my $progress; if ( $args->{progress} ) { @@ -160,6 +168,9 @@ sub find { my $mtime = $stat[9] || 0; my $size = -d $file ? 0 : ($stat[7] || 0); + my ($ext) = $file =~ /\.([^.]+)$/; + my $sth = ($ext && Slim::Music::Info::isImage($file, lc($ext))) ? $imageSth : $audioSth; + $sth->execute( Slim::Utils::Misc::fileURLFromPath($file), $mtime, diff --git a/types.conf b/types.conf index 6d6a53bdd0e..19e78592b43 100644 --- a/types.conf +++ b/types.conf @@ -23,12 +23,12 @@ dsf dsf audio/dsf audio dtd dtd application/xml-dtd - flc flac,flc,fla audio/x-flac,audio/flac audio fec - audio/x-cue-flac playlist -gif gif image/gif - +gif gif image/gif image htm htm,html text/html - htc htc text/x-component - log log text/plain - ico ico image/x-icon - -jpg jpg,jpeg image/jpeg - +jpg jpg,jpeg image/jpeg image jnp jnlp application/x-java-jnlp-file - jar jar application/x-java-archive - js js application/x-javascript - @@ -47,7 +47,7 @@ pcm pcm,l16,l24 audio/L16,audio/L24,audio/x-pcm audio pdf pdf application/pdf - pls pls audio/scpls,audio/x-scpls playlist pod - application/rss+xml - -png png image/png - +png png image/png image gd gd image/gd - sls - audio/x-m4a-sls audio svg svg image/svg+xml -