Fix IMDb matching for original non-English titles

src/Zilean.Database/Functions/SearchImdbProcedureV4.cs

@@ -0,0 +1,44 @@
+namespace Zilean.Database.Functions;
+
+public static class SearchImdbProcedureV4
+{
+    internal const string CreateImdbProcedure =
+        """
+            CREATE OR REPLACE FUNCTION search_imdb_meta(search_term TEXT, category_param TEXT DEFAULT NULL, year_param INT DEFAULT NULL, limit_param INT DEFAULT 10, similarity_threshold REAL DEFAULT 0.85)
+        RETURNS TABLE(imdb_id text, title text, category text, year INT, score REAL) AS $$
+            BEGIN
+        EXECUTE format('SET pg_trgm.similarity_threshold = %L', similarity_threshold);
+        RETURN QUERY
+            SELECT
+                "ImdbId",
+                "Title",
+                "Category",
+                "Year",
+                GREATEST(
+                    similarity("Title", search_term),
+                    similarity(COALESCE("OriginalTitle", ''), search_term)
+                ) as score
+            FROM public."ImdbFiles"
+            WHERE ("Title" % search_term OR COALESCE("OriginalTitle", '') % search_term)
+              AND ("Adult" = FALSE)
+              AND (
+                   category_param IS NULL
+                   OR (
+                       category_param = 'movie' AND "Category" IN ('movie', 'tvMovie')
+                   )
+                   OR (
+                       category_param = 'tvSeries' AND "Category" IN ('tvSeries', 'tvShort', 'tvMiniSeries', 'tvSpecial')
+                   )
+                   OR (
+                       category_param NOT IN ('movie', 'tvSeries') AND "Category" = category_param
+                   )
+               )
+              AND (year_param IS NULL OR "Year" BETWEEN year_param - 1 AND year_param + 1)
+            ORDER BY score DESC
+            LIMIT limit_param;
+            END; $$
+        LANGUAGE plpgsql;
+        """;
+
+    internal const string RemoveImdbProcedure = "DROP FUNCTION IF EXISTS search_imdb_meta(TEXT, TEXT, INT, INT);";
+}

src/Zilean.Database/Migrations/20260415120000_ImdbOriginalTitles.cs

@@ -0,0 +1,42 @@
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Zilean.Database.Functions;
+
+#nullable disable
+
+namespace Zilean.Database.Migrations;
+
+[DbContext(typeof(ZileanDbContext))]
+[Migration("20260415120000_ImdbOriginalTitles")]
+public class ImdbOriginalTitles : Migration
+{
+    private const string BackfillOriginalTitles =
+        """
+        UPDATE public."ImdbFiles"
+        SET "OriginalTitle" = "Title"
+        WHERE "OriginalTitle" IS NULL;
+        """;
+
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.AddColumn<string>(
+            name: "OriginalTitle",
+            table: "ImdbFiles",
+            type: "text",
+            nullable: true);
+
+        migrationBuilder.Sql(BackfillOriginalTitles);
+        migrationBuilder.Sql(SearchImdbProcedureV3.RemoveImdbProcedure);
+        migrationBuilder.Sql(SearchImdbProcedureV4.CreateImdbProcedure);
+    }
+
+    protected override void Down(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.Sql(SearchImdbProcedureV4.RemoveImdbProcedure);
+        migrationBuilder.Sql(SearchImdbProcedureV3.CreateImdbProcedure);
+
+        migrationBuilder.DropColumn(
+            name: "OriginalTitle",
+            table: "ImdbFiles");
+    }
+}

src/Zilean.Database/Migrations/ZileanDbContextModelSnapshot.cs

@@ -338,6 +338,9 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     b.Property<string>("Category")
                         .HasColumnType("text");
 
+                    b.Property<string>("OriginalTitle")
+                        .HasColumnType("text");
+
                     b.Property<string>("Title")
                         .HasColumnType("text");
 

src/Zilean.Database/ModelConfiguration/ImdbFileConfiguration.cs

@@ -17,6 +17,9 @@ public void Configure(EntityTypeBuilder<ImdbFile> builder)
         builder.Property(i => i.Title)
             .HasColumnType("text");
 
+        builder.Property(i => i.OriginalTitle)
+            .HasColumnType("text");
+
         builder.Property(i => i.Adult)
             .HasColumnType("boolean");
 

src/Zilean.Database/Services/FuzzyString/ImdbFuzzyStringMatchingService.cs

@@ -1,5 +1,3 @@
-using Raffinert.FuzzySharp;
-using Raffinert.FuzzySharp.PreProcess;
 using Zilean.Shared.Extensions;
 
 namespace Zilean.Database.Services.FuzzyString;
@@ -9,8 +7,6 @@ public class ImdbFuzzyStringMatchingService(ILogger<ImdbFuzzyStringMatchingServi
     private ConcurrentDictionary<string, string?>? _imdbCache;
     private ConcurrentDictionary<int,List<ImdbFile>>? _imdbTvFiles;
     private ConcurrentDictionary<int,List<ImdbFile>>? _imdbMovieFiles;
-    private const double ExactMatchTitleYearScore = 2.0;
-    private const double CloseMatchTitleYearScore = 1.5;
 
     public async Task PopulateImdbData()
     {
@@ -76,7 +72,7 @@ public Task<ConcurrentQueue<TorrentInfo>> MatchImdbIdsForBatchAsync(IEnumerable<
                             imdb.ImdbId,
                             imdb.Title,
                             imdb.Year,
-                            Score = CalculateScore(torrent, imdb),
+                            Score = ImdbTitleMatching.CalculateScore(torrent, imdb),
                         })
                     .OrderByDescending(match => match.Score)
                     .FirstOrDefault();
@@ -139,14 +135,6 @@ public Task<ConcurrentQueue<TorrentInfo>> MatchImdbIdsForBatchAsync(IEnumerable<
         return Task.FromResult(updatedTorrents);
     }
 
-    private static double CalculateScore(TorrentInfo torrent, ImdbFile imdb) =>
-        torrent.ParsedTitle == imdb.Title && torrent.Year == imdb.Year
-            ? ExactMatchTitleYearScore * 100
-            : torrent.ParsedTitle == imdb.Title && torrent.Year.HasValue &&
-              Math.Abs(torrent.Year.Value - imdb.Year) <= 1
-                ? CloseMatchTitleYearScore * 100
-                : Fuzz.Ratio(torrent.ParsedTitle, imdb.Title, PreprocessMode.Full);
-
     private bool HasFilteredPartitionsWithYear(
         ConcurrentDictionary<int, List<ImdbFile>> imdbTvFilesByYear,
         ConcurrentDictionary<int, List<ImdbFile>> imdbMovieFilesByYear,
@@ -222,7 +210,7 @@ private async Task<ConcurrentDictionary<int, List<ImdbFile>>> GetImdbMovieFiles(
 
         var imdbFiles = sqlConnection.Query<ImdbFile>(
             """
-            SELECT "ImdbId", "Title", "Adult", "Category", "Year" FROM public."ImdbFiles"
+            SELECT "ImdbId", "Title", "OriginalTitle", "Adult", "Category", "Year" FROM public."ImdbFiles"
             WHERE "Category" IN ('movie', 'tvMovie')
             """);
 
@@ -244,7 +232,7 @@ private async Task<ConcurrentDictionary<int, List<ImdbFile>>> GetImdbTvFiles()
 
         var imdbFiles = sqlConnection.Query<ImdbFile>(
             """
-            SELECT "ImdbId", "Title", "Adult", "Category", "Year" FROM public."ImdbFiles"
+            SELECT "ImdbId", "Title", "OriginalTitle", "Adult", "Category", "Year" FROM public."ImdbFiles"
             WHERE "Category" IN ('tvSeries', 'tvShort', 'tvMiniSeries', 'tvSpecial')
             """);
 

src/Zilean.Database/Services/ImdbTitleMatching.cs

@@ -0,0 +1,38 @@
+using Raffinert.FuzzySharp;
+using Raffinert.FuzzySharp.PreProcess;
+
+namespace Zilean.Database.Services;
+
+public static class ImdbTitleMatching
+{
+    private const double ExactMatchTitleYearScore = 2.0;
+    private const double CloseMatchTitleYearScore = 1.5;
+
+    public static double CalculateScore(TorrentInfo torrent, ImdbFile imdb) =>
+        GetCandidateTitles(imdb)
+            .Select(title => CalculateScore(torrent.ParsedTitle, torrent.Year, title, imdb.Year))
+            .DefaultIfEmpty(0)
+            .Max();
+
+    public static IReadOnlyCollection<string> GetCandidateTitles(ImdbFile imdb) =>
+        [..
+            new[] { imdb.Title, imdb.OriginalTitle }
+                .Where(title => !string.IsNullOrWhiteSpace(title))
+                .Select(title => title!)
+                .Distinct(StringComparer.OrdinalIgnoreCase)
+        ];
+
+    private static double CalculateScore(string? torrentTitle, int? torrentYear, string imdbTitle, int imdbYear)
+    {
+        if (string.IsNullOrWhiteSpace(torrentTitle))
+        {
+            return 0;
+        }
+
+        return torrentTitle == imdbTitle && torrentYear == imdbYear
+            ? ExactMatchTitleYearScore * 100
+            : torrentTitle == imdbTitle && torrentYear.HasValue && Math.Abs(torrentYear.Value - imdbYear) <= 1
+                ? CloseMatchTitleYearScore * 100
+                : Fuzz.Ratio(torrentTitle, imdbTitle, PreprocessMode.Full);
+    }
+}

src/Zilean.Database/Services/Lucene/ImdbLuceneMatchingService.cs

@@ -218,6 +218,14 @@ private async Task<LuceneSession> IndexImdbDocumentsInMemory()
                         )
                     )
                 ) AS "Title",
+                Lower(
+                unaccent(
+                    regexp_replace(
+                        regexp_replace(trim("OriginalTitle"), '\s+', ' ', 'g'),
+                        '[^\w\s]', '', 'g'
+                        )
+                    )
+                ) AS "OriginalTitle",
                 "Adult",
                 "Category",
                 "Year"
@@ -232,7 +240,6 @@ private async Task<LuceneSession> IndexImdbDocumentsInMemory()
             var doc = new Document
             {
                 new StringField(LuceneIndexEntry.ImdbId, imdb.ImdbId, Field.Store.YES),
-                new StringField(LuceneIndexEntry.Title, imdb.Title, Field.Store.YES),
                 new StringField(LuceneIndexEntry.Category, GetCategory(imdb.Category).ToLowerInvariant(), Field.Store.YES),
                 new Int32Field(LuceneIndexEntry.Year, imdb.Year, new FieldType
                 {
@@ -244,6 +251,16 @@ private async Task<LuceneSession> IndexImdbDocumentsInMemory()
                 }),
             };
 
+            foreach (var title in ImdbTitleMatching.GetCandidateTitles(imdb))
+            {
+                doc.Add(new StringField(LuceneIndexEntry.Title, title, Field.Store.YES));
+            }
+
+            if (doc.GetFields(LuceneIndexEntry.Title).Length == 0)
+            {
+                continue;
+            }
+
             luceneSession.Writer.AddDocument(doc);
         }
 

src/Zilean.Scraper/Features/Imdb/ImdbFileProcessor.cs

@@ -21,14 +21,13 @@ public async Task Import(string fileName, CancellationToken cancellationToken)
             BadDataFound = null,
             MissingFieldFound = null,
             HasHeaderRecord = true,
-            ShouldSkipRecord = record => !_requiredCategories.Contains(record.Row.GetField(1))
         };
 
         using var reader = new StreamReader(fileName);
         using var csv = new CsvReader(reader, csvConfig);
 
-        // skip header...
         await csv.ReadAsync();
+        csv.ReadHeader();
 
         await ReadBasicEntries(csv, imdbFileService, cancellationToken);
 
@@ -41,15 +40,25 @@ private static async Task ReadBasicEntries(CsvReader csv, IImdbFileService imdbF
     {
         while (await csv.ReadAsync())
         {
+            var category = csv.GetField(1);
+
+            if (!_requiredCategories.Contains(category))
+            {
+                continue;
+            }
+
+            var primaryTitle = GetNullableField(csv.GetField(2));
+            var originalTitle = GetNullableField(csv.GetField(3));
             var isAdultSet = int.TryParse(csv.GetField(4), out var adult);
             var yearField = csv.GetField(5);
             var isYearValid = int.TryParse(yearField == @"\N" ? "0" : yearField, out var year);
 
             var movieData = new ImdbFile
             {
                 ImdbId = csv.GetField(0),
-                Category = csv.GetField(1),
-                Title = csv.GetField(2),
+                Category = category,
+                Title = primaryTitle ?? originalTitle,
+                OriginalTitle = originalTitle ?? primaryTitle,
                 Adult = isAdultSet && adult == 1,
                 Year = isYearValid ? year : 0
             };
@@ -62,4 +71,7 @@ private static async Task ReadBasicEntries(CsvReader csv, IImdbFileService imdbF
             imdbFileService.AddImdbFile(movieData);
         }
     }
+
+    private static string? GetNullableField(string? value) =>
+        string.IsNullOrWhiteSpace(value) || value == @"\N" ? null : value;
 }

src/Zilean.Shared/Features/Imdb/ImdbFile.cs

@@ -6,6 +6,7 @@ public class ImdbFile
     public string ImdbId { get; set; } = default!;
     public string? Category { get; set; }
     public string? Title { get; set; }
+    public string? OriginalTitle { get; set; }
     public bool Adult { get; set; }
     public int Year { get; set; }
 }

tests/Zilean.Tests/Tests/ImdbOriginalTitleTests.cs

@@ -0,0 +1,62 @@
+using Zilean.Database.Services;
+
+namespace Zilean.Tests.Tests;
+
+public class ImdbOriginalTitleTests
+{
+    [Fact]
+    public void calculate_score_uses_original_title_when_primary_title_does_not_match()
+    {
+        var torrent = new TorrentInfo
+        {
+            ParsedTitle = "Le Fabuleux Destin d'Amelie Poulain",
+            Year = 2001,
+            Category = "movie"
+        };
+
+        var imdb = new ImdbFile
+        {
+            ImdbId = "tt0211915",
+            Title = "Amelie",
+            OriginalTitle = "Le Fabuleux Destin d'Amelie Poulain",
+            Year = 2001,
+            Category = "movie"
+        };
+
+        var score = ImdbTitleMatching.CalculateScore(torrent, imdb);
+
+        score.Should().Be(200);
+    }
+
+    [Fact]
+    public async Task import_reads_original_title_from_imdb_basics()
+    {
+        var logger = Substitute.For<ILogger<ImdbFileProcessor>>();
+        var imdbFileService = Substitute.For<IImdbFileService>();
+        var processor = new ImdbFileProcessor(logger, imdbFileService);
+        var filePath = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid():N}.tsv");
+
+        try
+        {
+            await File.WriteAllTextAsync(
+                filePath,
+                """
+                tconst	titleType	primaryTitle	originalTitle	isAdult	startYear	endYear	runtimeMinutes	genres
+                tt0211915	movie	Amelie	Le Fabuleux Destin d'Amelie Poulain	0	2001	\N	122	Comedy,Romance
+                """);
+
+            await processor.Import(filePath, CancellationToken.None);
+
+            imdbFileService.Received(1).AddImdbFile(Arg.Is<ImdbFile>(file =>
+                file.ImdbId == "tt0211915" &&
+                file.Title == "Amelie" &&
+                file.OriginalTitle == "Le Fabuleux Destin d'Amelie Poulain" &&
+                file.Year == 2001 &&
+                file.Category == "movie"));
+        }
+        finally
+        {
+            File.Delete(filePath);
+        }
+    }
+}