feat: 拆分artwork表，避免数据库体积爆炸

2026-06-09 16:25:21 +08:00
parent e9eb69db5f
commit fb1bd066f5
7 changed files with 214 additions and 16 deletions
@@ -29,6 +29,7 @@

 - [ ] 扫描时如果报错，报错信息可能会填满整个窗口，导致Overlay无法关闭。
 - [ ] 读取时如果报错，没有任何警告，会静默报错。需要有一个界面右下方的toast，或是发送系统通知告知用户遇到了错误。
+- [ ] 指针化Artwork字段。如果artwork的md5
 - [ ] 本地化框架

 ## 技术栈
@@ -8,6 +8,7 @@
  </PropertyGroup>

  <ItemGroup>
+    <PackageReference Include="Blake3" Version="2.2.1" />
    <PackageReference Include="coverlet.collector" Version="6.0.4" />
    <PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="2.1.10" />
    <PackageReference Include="FluentAssertions" Version="8.9.0" />
@@ -40,6 +40,9 @@ public class DatabaseTests
            Assert.Contains("bpm", columns);
            Assert.Contains("description", columns);

+            Assert.Contains("artwork_fingerprint", columns);
+            Assert.DoesNotContain("artwork", columns); // 旧 BLOB 列不应存在
+
            // 3. 索引存在
            HashSet<string> indexes = conn.Query<string>(
                "SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='audio_files';")
@@ -49,6 +52,17 @@ public class DatabaseTests
            Assert.Contains("idx_path", indexes);
            Assert.Contains("idx_unique_id", indexes);
            Assert.Contains("idx_description", indexes);
+            Assert.Contains("idx_artwork_fingerprint", indexes);
+
+            // 4. artworks 表存在
+            int artworksTableCount = conn.ExecuteScalar<int>(
+                "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='artworks';");
+            Assert.Equal(1, artworksTableCount);
+
+            HashSet<string> artworksColumns = conn.Query<string>(
+                "SELECT name FROM pragma_table_info('artworks') ORDER BY cid;").ToHashSet();
+            Assert.Contains("fingerprint", artworksColumns);
+            Assert.Contains("data", artworksColumns);
        }
        finally
        {
@@ -303,4 +317,107 @@ public class DatabaseTests
    }

    #endregion
+
+    #region Artwork 测试
+
+    [Fact]
+    public void AddEntry_WithArtwork_ShouldStoreFingerprintAndRetrieveData()
+    {
+        Database.InitializeDatabase();
+        AudioFileMeta meta = CreateSampleMeta();
+        byte[] artworkData = [0x89, 0x50, 0x4E, 0x47]; // PNG 幻数
+        meta.Artwork = artworkData;
+
+        Assert.True(Database.AddEntry(meta));
+
+        try
+        {
+            AudioFileMeta? retrieved = Database.GetEntryById(meta.Id);
+            Assert.NotNull(retrieved);
+            Assert.NotNull(retrieved!.ArtworkFingerprint);
+            Assert.Equal(64, retrieved.ArtworkFingerprint!.Length);
+
+            byte[]? actualData = Database.GetArtworkData(retrieved.ArtworkFingerprint);
+            Assert.NotNull(actualData);
+            Assert.Equal(artworkData, actualData);
+        }
+        finally
+        {
+            DeleteTestEntries(meta.UniqueId);
+            Database.DeleteOrphanedArtworks();
+        }
+    }
+
+    [Fact]
+    public void AddEntries_WithDuplicateArtwork_ShouldDeduplicate()
+    {
+        Database.InitializeDatabase();
+        byte[] artworkData = [0xFF, 0xD8, 0xFF, 0xE0]; // JPEG 幻数
+
+        AudioFileMeta meta1 = CreateSampleMeta();
+        meta1.Artwork = artworkData;
+        AudioFileMeta meta2 = CreateSampleMeta();
+        meta2.Artwork = artworkData;
+
+        Database.AddEntries([meta1, meta2]);
+
+        try
+        {
+            Assert.NotNull(meta1.ArtworkFingerprint);
+            Assert.Equal(meta1.ArtworkFingerprint, meta2.ArtworkFingerprint);
+
+            // artworks 表中只应有一条记录
+            using SQLiteConnection conn = new($"Data Source={GetDefaultDbPath()};Version=3;");
+            conn.Open();
+            int artworkCount = conn.ExecuteScalar<int>(
+                "SELECT COUNT(*) FROM artworks;");
+            Assert.Equal(1, artworkCount);
+        }
+        finally
+        {
+            DeleteTestEntries(meta1.UniqueId, meta2.UniqueId);
+            Database.DeleteOrphanedArtworks();
+        }
+    }
+
+    [Fact]
+    public void DeleteOrphanedArtworks_ShouldRemoveUnreferencedRecords()
+    {
+        Database.InitializeDatabase();
+        byte[] artworkData = [0x47, 0x49, 0x46, 0x38]; // GIF 幻数
+
+        AudioFileMeta meta = CreateSampleMeta();
+        meta.Artwork = artworkData;
+        Database.AddEntry(meta);
+
+        try
+        {
+            string fingerprint = meta.ArtworkFingerprint!;
+            Assert.NotNull(Database.GetArtworkData(fingerprint));
+
+            // 删除音频文件后，artwork 成为孤儿
+            DeleteTestEntries(meta.UniqueId);
+
+            int deleted = Database.DeleteOrphanedArtworks();
+            Assert.True(deleted >= 1); // 至少删除本次测试产生的孤儿
+
+            // 确认本次测试的 artwork 已被清理
+            Assert.Null(Database.GetArtworkData(fingerprint));
+        }
+        finally
+        {
+            DeleteTestEntries(meta.UniqueId);
+            Database.DeleteOrphanedArtworks();
+        }
+    }
+
+    [Fact]
+    public void GetArtworkData_NonExistent_ShouldReturnNull()
+    {
+        Database.InitializeDatabase();
+        byte[]? result = Database.GetArtworkData("nonexistentfingerprint000000000000000000000000000000000000000000000000");
+        Assert.Null(result);
+    }
+
+    #endregion
 }
@@ -69,7 +69,11 @@ public class IntegratedTest
                Assert.Equal(meta.Type, retrieved.Type);

                if (meta.Artwork != null)
-                    Assert.NotNull(retrieved.Artwork);
+                {
+                    Assert.NotNull(retrieved.ArtworkFingerprint);
+                    byte[]? artworkData = Database.GetArtworkData(retrieved.ArtworkFingerprint!);
+                    Assert.NotNull(artworkData);
+                }
            }
        }
        finally
@@ -197,6 +197,10 @@ public class AudioFileMeta
    
    public string? MicPerspective { get; set; }
    
+    /// <summary>封面图片指纹（Blake3 十六进制），映射 artworks 表；实际图片数据通过 Database.GetArtworkData() 获取</summary>
+    public string? ArtworkFingerprint { get; set; }
+    
+    /// <summary>封面图片原始数据，仅用于导入管线暂存，不映射数据库列</summary>
    public byte[]? Artwork { get; set; }
    
    public byte[]? Waveform { get; set; }
@@ -7,6 +7,7 @@
    </PropertyGroup>

    <ItemGroup>
+      <PackageReference Include="Blake3" Version="2.2.1" />
      <PackageReference Include="Dapper" Version="2.1.72" />
      <PackageReference Include="Newtonsoft.Json" Version="13.0.4" />
      <PackageReference Include="System.Data.SQLite" Version="2.0.3" />
@@ -1,5 +1,6 @@
 using System.Data;
 using System.Data.SQLite;
+using Blake3;
 using Dapper;

 namespace OCES.Resonance.Core;
@@ -106,16 +107,22 @@ public static class Database
                               channel_layout TEXT,
                               bwf_umid BLOB,
                               disk_number INTEGER,
-                               track_number INTEGER,
-                               artwork BLOB,
-                               waveform BLOB
-                           );
+                                track_number INTEGER,
+                                artwork_fingerprint TEXT,
+                                waveform BLOB
+                            );

-                           CREATE INDEX if not exists idx_file_name on audio_files(filename);
-                           CREATE INDEX if not exists idx_md5 on audio_files(md5);
-                           CREATE INDEX if not exists idx_path on audio_files(path);
-                           CREATE INDEX if not exists idx_unique_id on audio_files(unique_id);
-                           CREATE INDEX if not exists idx_description on audio_files(description);
+                            CREATE INDEX if not exists idx_file_name on audio_files(filename);
+                            CREATE INDEX if not exists idx_md5 on audio_files(md5);
+                            CREATE INDEX if not exists idx_path on audio_files(path);
+                            CREATE INDEX if not exists idx_unique_id on audio_files(unique_id);
+                            CREATE INDEX if not exists idx_description on audio_files(description);
+                            CREATE INDEX if not exists idx_artwork_fingerprint on audio_files(artwork_fingerprint);
+
+                            CREATE TABLE IF NOT EXISTS artworks (
+                                fingerprint TEXT PRIMARY KEY,
+                                data BLOB NOT NULL
+                            );
                                   
                           """;
        
@@ -147,7 +154,7 @@ public static class Database
                    release_date, track_year, is_edited, is_split, location, [group],
                    markers, comments, notes, copyright, coding_history, microphone,
                    mic_perspective,
-                    fx_name, channel_layout, bwf_umid, disk_number, track_number, artwork, waveform,
+                    fx_name, channel_layout, bwf_umid, disk_number, track_number, artwork_fingerprint, waveform,
                    user1, user2, user3, user4, user5, user6, user7, user8
                ) VALUES (
                    @UniqueId, @ShortId, @Md5, @Path, @Filename, @Folder, @Directory,
@@ -161,11 +168,17 @@ public static class Database
                    @ReleaseDate, @TrackYear, @IsEdited, @IsSplit, @Location, @Group,
                    @Markers, @Comments, @Notes, @Copyright, @BwfCodingHistory, @Microphone,
                    @MicPerspective,
-                    @FxName, @ChannelLayout, @BwfUmid, @DiskNumber, @TrackNumber, @Artwork, @Waveform,
+                    @FxName, @ChannelLayout, @BwfUmid, @DiskNumber, @TrackNumber, @ArtworkFingerprint, @Waveform,
                    @User1, @User2, @User3, @User4, @User5, @User6, @User7, @User8
                );
            ";
            
+            if (meta.Artwork is { Length: > 0 })
+            {
+                meta.ArtworkFingerprint = ComputeArtworkFingerprint(meta.Artwork);
+                EnsureArtwork(connection, null, meta.ArtworkFingerprint, meta.Artwork);
+            }
+            
            int newId = connection.ExecuteScalar<int>(sql + "; SELECT last_insert_rowid();", meta);
            meta.Id = newId;
            return true;
@@ -203,7 +216,7 @@ public static class Database
                    release_date, track_year, is_edited, is_split, location, [group],
                    markers, comments, notes, copyright, coding_history, microphone,
                    mic_perspective,
-                    fx_name, channel_layout, bwf_umid, disk_number, track_number, artwork, waveform,
+                    fx_name, channel_layout, bwf_umid, disk_number, track_number, artwork_fingerprint, waveform,
                    user1, user2, user3, user4, user5, user6, user7, user8
                ) VALUES (
                    @UniqueId, @ShortId, @Md5, @Path, @Filename, @Folder, @Directory,
@@ -217,13 +230,18 @@ public static class Database
                    @ReleaseDate, @TrackYear, @IsEdited, @IsSplit, @Location, @Group,
                    @Markers, @Comments, @Notes, @Copyright, @BwfCodingHistory, @Microphone,
                    @MicPerspective,
-                    @FxName, @ChannelLayout, @BwfUmid, @DiskNumber, @TrackNumber, @Artwork, @Waveform,
+                    @FxName, @ChannelLayout, @BwfUmid, @DiskNumber, @TrackNumber, @ArtworkFingerprint, @Waveform,
                    @User1, @User2, @User3, @User4, @User5, @User6, @User7, @User8
                );
            ";
            
            foreach (AudioFileMeta entry in entries)
            {
+                if (entry.Artwork is { Length: > 0 })
+                {
+                    entry.ArtworkFingerprint = ComputeArtworkFingerprint(entry.Artwork);
+                    EnsureArtwork(connection, transaction, entry.ArtworkFingerprint, entry.Artwork);
+                }
                connection.Execute(sql, entry, transaction);
                count++;
            }
@@ -275,10 +293,10 @@ public static class Database
        markers, comments, notes, copyright, coding_history AS bwf_coding_history, microphone,
        mic_perspective,
        fx_name, channel_layout, disk_number, track_number,
-        artwork, waveform, bwf_umid,
+        artwork_fingerprint, waveform, bwf_umid,
        user1, user2, user3, user4, user5, user6, user7, user8";

-    // 列表查询公共列（排除 BLOB：artwork / waveform / bwf_umid），减少数据传输
+    // 列表查询公共列（排除 BLOB：waveform / bwf_umid），减少数据传输
    private const string ListSelectColumns = @"
        id, unique_id, short_id, md5, path, filename, folder, directory,
        duration, bit_depth, channels, sample_rate, type,
@@ -292,6 +310,7 @@ public static class Database
        markers, comments, notes, copyright, coding_history AS bwf_coding_history, microphone,
        mic_perspective,
        fx_name, channel_layout, disk_number, track_number,
+        artwork_fingerprint,
        user1, user2, user3, user4, user5, user6, user7, user8";

    // 排序白名单，防止 SQL 注入
@@ -547,4 +566,55 @@ public static class Database
    }

    #endregion
+
+    #region Artwork
+    
+    /// <summary>
+    /// 计算 artwork 数据的 Blake3 指纹
+    /// </summary>
+    private static string ComputeArtworkFingerprint(byte[] data)
+    {
+        return Hasher.Hash(data).ToString().ToLowerInvariant();
+    }
+    
+    /// <summary>
+    /// 确保 artwork 数据已存入 artworks 表（指纹不存在时插入）
+    /// </summary>
+    private static void EnsureArtwork(IDbConnection connection, IDbTransaction? transaction, string fingerprint, byte[] data)
+    {
+        const string sql = "INSERT OR IGNORE INTO artworks (fingerprint, data) VALUES (@Fingerprint, @Data);";
+        connection.Execute(sql, new { Fingerprint = fingerprint, Data = data }, transaction);
+    }
+    
+    /// <summary>
+    /// 根据指纹获取 artwork 原始数据
+    /// </summary>
+    public static byte[]? GetArtworkData(string fingerprint)
+    {
+        using IDbConnection connection = GetConnection();
+        connection.Open();
+        const string sql = "SELECT data FROM artworks WHERE fingerprint = @Fingerprint;";
+        return connection.ExecuteScalar<byte[]>(sql, new { Fingerprint = fingerprint });
+    }
+    
+    /// <summary>
+    /// 清理 artworks 表中无 audio_files 引用的孤儿记录
+    /// </summary>
+    /// <returns>删除的记录数</returns>
+    public static int DeleteOrphanedArtworks()
+    {
+        using IDbConnection connection = GetConnection();
+        connection.Open();
+        const string sql = """
+                               DELETE FROM artworks
+                               WHERE fingerprint NOT IN (
+                                   SELECT DISTINCT artwork_fingerprint
+                                   FROM audio_files
+                                   WHERE artwork_fingerprint IS NOT NULL
+                               );
+                           """;
+        return connection.Execute(sql);
+    }
+    
+    #endregion
 }