feat: 拆分artwork表,避免数据库体积爆炸

This commit is contained in:
2026-06-09 16:25:21 +08:00
parent e9eb69db5f
commit fb1bd066f5
7 changed files with 214 additions and 16 deletions
+1
View File
@@ -29,6 +29,7 @@
- [ ] 扫描时如果报错,报错信息可能会填满整个窗口,导致Overlay无法关闭。 - [ ] 扫描时如果报错,报错信息可能会填满整个窗口,导致Overlay无法关闭。
- [ ] 读取时如果报错,没有任何警告,会静默报错。需要有一个界面右下方的toast,或是发送系统通知告知用户遇到了错误。 - [ ] 读取时如果报错,没有任何警告,会静默报错。需要有一个界面右下方的toast,或是发送系统通知告知用户遇到了错误。
- [ ] 指针化Artwork字段。如果artwork的md5
- [ ] 本地化框架 - [ ] 本地化框架
## 技术栈 ## 技术栈
+1
View File
@@ -8,6 +8,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Blake3" Version="2.2.1" />
<PackageReference Include="coverlet.collector" Version="6.0.4" /> <PackageReference Include="coverlet.collector" Version="6.0.4" />
<PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="2.1.10" /> <PackageReference Include="SQLitePCLRaw.bundle_e_sqlite3" Version="2.1.10" />
<PackageReference Include="FluentAssertions" Version="8.9.0" /> <PackageReference Include="FluentAssertions" Version="8.9.0" />
+117
View File
@@ -40,6 +40,9 @@ public class DatabaseTests
Assert.Contains("bpm", columns); Assert.Contains("bpm", columns);
Assert.Contains("description", columns); Assert.Contains("description", columns);
Assert.Contains("artwork_fingerprint", columns);
Assert.DoesNotContain("artwork", columns); // 旧 BLOB 列不应存在
// 3. 索引存在 // 3. 索引存在
HashSet<string> indexes = conn.Query<string>( HashSet<string> indexes = conn.Query<string>(
"SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='audio_files';") "SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='audio_files';")
@@ -49,6 +52,17 @@ public class DatabaseTests
Assert.Contains("idx_path", indexes); Assert.Contains("idx_path", indexes);
Assert.Contains("idx_unique_id", indexes); Assert.Contains("idx_unique_id", indexes);
Assert.Contains("idx_description", indexes); Assert.Contains("idx_description", indexes);
Assert.Contains("idx_artwork_fingerprint", indexes);
// 4. artworks 表存在
int artworksTableCount = conn.ExecuteScalar<int>(
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='artworks';");
Assert.Equal(1, artworksTableCount);
HashSet<string> artworksColumns = conn.Query<string>(
"SELECT name FROM pragma_table_info('artworks') ORDER BY cid;").ToHashSet();
Assert.Contains("fingerprint", artworksColumns);
Assert.Contains("data", artworksColumns);
} }
finally finally
{ {
@@ -303,4 +317,107 @@ public class DatabaseTests
} }
#endregion #endregion
#region Artwork
[Fact]
public void AddEntry_WithArtwork_ShouldStoreFingerprintAndRetrieveData()
{
Database.InitializeDatabase();
AudioFileMeta meta = CreateSampleMeta();
byte[] artworkData = [0x89, 0x50, 0x4E, 0x47]; // PNG 幻数
meta.Artwork = artworkData;
Assert.True(Database.AddEntry(meta));
try
{
AudioFileMeta? retrieved = Database.GetEntryById(meta.Id);
Assert.NotNull(retrieved);
Assert.NotNull(retrieved!.ArtworkFingerprint);
Assert.Equal(64, retrieved.ArtworkFingerprint!.Length);
byte[]? actualData = Database.GetArtworkData(retrieved.ArtworkFingerprint);
Assert.NotNull(actualData);
Assert.Equal(artworkData, actualData);
}
finally
{
DeleteTestEntries(meta.UniqueId);
Database.DeleteOrphanedArtworks();
}
}
[Fact]
public void AddEntries_WithDuplicateArtwork_ShouldDeduplicate()
{
Database.InitializeDatabase();
byte[] artworkData = [0xFF, 0xD8, 0xFF, 0xE0]; // JPEG 幻数
AudioFileMeta meta1 = CreateSampleMeta();
meta1.Artwork = artworkData;
AudioFileMeta meta2 = CreateSampleMeta();
meta2.Artwork = artworkData;
Database.AddEntries([meta1, meta2]);
try
{
Assert.NotNull(meta1.ArtworkFingerprint);
Assert.Equal(meta1.ArtworkFingerprint, meta2.ArtworkFingerprint);
// artworks 表中只应有一条记录
using SQLiteConnection conn = new($"Data Source={GetDefaultDbPath()};Version=3;");
conn.Open();
int artworkCount = conn.ExecuteScalar<int>(
"SELECT COUNT(*) FROM artworks;");
Assert.Equal(1, artworkCount);
}
finally
{
DeleteTestEntries(meta1.UniqueId, meta2.UniqueId);
Database.DeleteOrphanedArtworks();
}
}
[Fact]
public void DeleteOrphanedArtworks_ShouldRemoveUnreferencedRecords()
{
Database.InitializeDatabase();
byte[] artworkData = [0x47, 0x49, 0x46, 0x38]; // GIF 幻数
AudioFileMeta meta = CreateSampleMeta();
meta.Artwork = artworkData;
Database.AddEntry(meta);
try
{
string fingerprint = meta.ArtworkFingerprint!;
Assert.NotNull(Database.GetArtworkData(fingerprint));
// 删除音频文件后,artwork 成为孤儿
DeleteTestEntries(meta.UniqueId);
int deleted = Database.DeleteOrphanedArtworks();
Assert.True(deleted >= 1); // 至少删除本次测试产生的孤儿
// 确认本次测试的 artwork 已被清理
Assert.Null(Database.GetArtworkData(fingerprint));
}
finally
{
DeleteTestEntries(meta.UniqueId);
Database.DeleteOrphanedArtworks();
}
}
[Fact]
public void GetArtworkData_NonExistent_ShouldReturnNull()
{
Database.InitializeDatabase();
byte[]? result = Database.GetArtworkData("nonexistentfingerprint000000000000000000000000000000000000000000000000");
Assert.Null(result);
}
#endregion
} }
+5 -1
View File
@@ -69,7 +69,11 @@ public class IntegratedTest
Assert.Equal(meta.Type, retrieved.Type); Assert.Equal(meta.Type, retrieved.Type);
if (meta.Artwork != null) if (meta.Artwork != null)
Assert.NotNull(retrieved.Artwork); {
Assert.NotNull(retrieved.ArtworkFingerprint);
byte[]? artworkData = Database.GetArtworkData(retrieved.ArtworkFingerprint!);
Assert.NotNull(artworkData);
}
} }
} }
finally finally
+4
View File
@@ -197,6 +197,10 @@ public class AudioFileMeta
public string? MicPerspective { get; set; } public string? MicPerspective { get; set; }
/// <summary>封面图片指纹(Blake3 十六进制),映射 artworks 表;实际图片数据通过 Database.GetArtworkData() 获取</summary>
public string? ArtworkFingerprint { get; set; }
/// <summary>封面图片原始数据,仅用于导入管线暂存,不映射数据库列</summary>
public byte[]? Artwork { get; set; } public byte[]? Artwork { get; set; }
public byte[]? Waveform { get; set; } public byte[]? Waveform { get; set; }
+1
View File
@@ -7,6 +7,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Blake3" Version="2.2.1" />
<PackageReference Include="Dapper" Version="2.1.72" /> <PackageReference Include="Dapper" Version="2.1.72" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.4" /> <PackageReference Include="Newtonsoft.Json" Version="13.0.4" />
<PackageReference Include="System.Data.SQLite" Version="2.0.3" /> <PackageReference Include="System.Data.SQLite" Version="2.0.3" />
+77 -7
View File
@@ -1,5 +1,6 @@
using System.Data; using System.Data;
using System.Data.SQLite; using System.Data.SQLite;
using Blake3;
using Dapper; using Dapper;
namespace OCES.Resonance.Core; namespace OCES.Resonance.Core;
@@ -107,7 +108,7 @@ public static class Database
bwf_umid BLOB, bwf_umid BLOB,
disk_number INTEGER, disk_number INTEGER,
track_number INTEGER, track_number INTEGER,
artwork BLOB, artwork_fingerprint TEXT,
waveform BLOB waveform BLOB
); );
@@ -116,6 +117,12 @@ public static class Database
CREATE INDEX if not exists idx_path on audio_files(path); CREATE INDEX if not exists idx_path on audio_files(path);
CREATE INDEX if not exists idx_unique_id on audio_files(unique_id); CREATE INDEX if not exists idx_unique_id on audio_files(unique_id);
CREATE INDEX if not exists idx_description on audio_files(description); CREATE INDEX if not exists idx_description on audio_files(description);
CREATE INDEX if not exists idx_artwork_fingerprint on audio_files(artwork_fingerprint);
CREATE TABLE IF NOT EXISTS artworks (
fingerprint TEXT PRIMARY KEY,
data BLOB NOT NULL
);
"""; """;
@@ -147,7 +154,7 @@ public static class Database
release_date, track_year, is_edited, is_split, location, [group], release_date, track_year, is_edited, is_split, location, [group],
markers, comments, notes, copyright, coding_history, microphone, markers, comments, notes, copyright, coding_history, microphone,
mic_perspective, mic_perspective,
fx_name, channel_layout, bwf_umid, disk_number, track_number, artwork, waveform, fx_name, channel_layout, bwf_umid, disk_number, track_number, artwork_fingerprint, waveform,
user1, user2, user3, user4, user5, user6, user7, user8 user1, user2, user3, user4, user5, user6, user7, user8
) VALUES ( ) VALUES (
@UniqueId, @ShortId, @Md5, @Path, @Filename, @Folder, @Directory, @UniqueId, @ShortId, @Md5, @Path, @Filename, @Folder, @Directory,
@@ -161,11 +168,17 @@ public static class Database
@ReleaseDate, @TrackYear, @IsEdited, @IsSplit, @Location, @Group, @ReleaseDate, @TrackYear, @IsEdited, @IsSplit, @Location, @Group,
@Markers, @Comments, @Notes, @Copyright, @BwfCodingHistory, @Microphone, @Markers, @Comments, @Notes, @Copyright, @BwfCodingHistory, @Microphone,
@MicPerspective, @MicPerspective,
@FxName, @ChannelLayout, @BwfUmid, @DiskNumber, @TrackNumber, @Artwork, @Waveform, @FxName, @ChannelLayout, @BwfUmid, @DiskNumber, @TrackNumber, @ArtworkFingerprint, @Waveform,
@User1, @User2, @User3, @User4, @User5, @User6, @User7, @User8 @User1, @User2, @User3, @User4, @User5, @User6, @User7, @User8
); );
"; ";
if (meta.Artwork is { Length: > 0 })
{
meta.ArtworkFingerprint = ComputeArtworkFingerprint(meta.Artwork);
EnsureArtwork(connection, null, meta.ArtworkFingerprint, meta.Artwork);
}
int newId = connection.ExecuteScalar<int>(sql + "; SELECT last_insert_rowid();", meta); int newId = connection.ExecuteScalar<int>(sql + "; SELECT last_insert_rowid();", meta);
meta.Id = newId; meta.Id = newId;
return true; return true;
@@ -203,7 +216,7 @@ public static class Database
release_date, track_year, is_edited, is_split, location, [group], release_date, track_year, is_edited, is_split, location, [group],
markers, comments, notes, copyright, coding_history, microphone, markers, comments, notes, copyright, coding_history, microphone,
mic_perspective, mic_perspective,
fx_name, channel_layout, bwf_umid, disk_number, track_number, artwork, waveform, fx_name, channel_layout, bwf_umid, disk_number, track_number, artwork_fingerprint, waveform,
user1, user2, user3, user4, user5, user6, user7, user8 user1, user2, user3, user4, user5, user6, user7, user8
) VALUES ( ) VALUES (
@UniqueId, @ShortId, @Md5, @Path, @Filename, @Folder, @Directory, @UniqueId, @ShortId, @Md5, @Path, @Filename, @Folder, @Directory,
@@ -217,13 +230,18 @@ public static class Database
@ReleaseDate, @TrackYear, @IsEdited, @IsSplit, @Location, @Group, @ReleaseDate, @TrackYear, @IsEdited, @IsSplit, @Location, @Group,
@Markers, @Comments, @Notes, @Copyright, @BwfCodingHistory, @Microphone, @Markers, @Comments, @Notes, @Copyright, @BwfCodingHistory, @Microphone,
@MicPerspective, @MicPerspective,
@FxName, @ChannelLayout, @BwfUmid, @DiskNumber, @TrackNumber, @Artwork, @Waveform, @FxName, @ChannelLayout, @BwfUmid, @DiskNumber, @TrackNumber, @ArtworkFingerprint, @Waveform,
@User1, @User2, @User3, @User4, @User5, @User6, @User7, @User8 @User1, @User2, @User3, @User4, @User5, @User6, @User7, @User8
); );
"; ";
foreach (AudioFileMeta entry in entries) foreach (AudioFileMeta entry in entries)
{ {
if (entry.Artwork is { Length: > 0 })
{
entry.ArtworkFingerprint = ComputeArtworkFingerprint(entry.Artwork);
EnsureArtwork(connection, transaction, entry.ArtworkFingerprint, entry.Artwork);
}
connection.Execute(sql, entry, transaction); connection.Execute(sql, entry, transaction);
count++; count++;
} }
@@ -275,10 +293,10 @@ public static class Database
markers, comments, notes, copyright, coding_history AS bwf_coding_history, microphone, markers, comments, notes, copyright, coding_history AS bwf_coding_history, microphone,
mic_perspective, mic_perspective,
fx_name, channel_layout, disk_number, track_number, fx_name, channel_layout, disk_number, track_number,
artwork, waveform, bwf_umid, artwork_fingerprint, waveform, bwf_umid,
user1, user2, user3, user4, user5, user6, user7, user8"; user1, user2, user3, user4, user5, user6, user7, user8";
// 列表查询公共列(排除 BLOBartwork / waveform / bwf_umid),减少数据传输 // 列表查询公共列(排除 BLOBwaveform / bwf_umid),减少数据传输
private const string ListSelectColumns = @" private const string ListSelectColumns = @"
id, unique_id, short_id, md5, path, filename, folder, directory, id, unique_id, short_id, md5, path, filename, folder, directory,
duration, bit_depth, channels, sample_rate, type, duration, bit_depth, channels, sample_rate, type,
@@ -292,6 +310,7 @@ public static class Database
markers, comments, notes, copyright, coding_history AS bwf_coding_history, microphone, markers, comments, notes, copyright, coding_history AS bwf_coding_history, microphone,
mic_perspective, mic_perspective,
fx_name, channel_layout, disk_number, track_number, fx_name, channel_layout, disk_number, track_number,
artwork_fingerprint,
user1, user2, user3, user4, user5, user6, user7, user8"; user1, user2, user3, user4, user5, user6, user7, user8";
// 排序白名单,防止 SQL 注入 // 排序白名单,防止 SQL 注入
@@ -547,4 +566,55 @@ public static class Database
} }
#endregion #endregion
#region Artwork
/// <summary>
/// 计算 artwork 数据的 Blake3 指纹
/// </summary>
private static string ComputeArtworkFingerprint(byte[] data)
{
return Hasher.Hash(data).ToString().ToLowerInvariant();
}
/// <summary>
/// 确保 artwork 数据已存入 artworks 表(指纹不存在时插入)
/// </summary>
private static void EnsureArtwork(IDbConnection connection, IDbTransaction? transaction, string fingerprint, byte[] data)
{
const string sql = "INSERT OR IGNORE INTO artworks (fingerprint, data) VALUES (@Fingerprint, @Data);";
connection.Execute(sql, new { Fingerprint = fingerprint, Data = data }, transaction);
}
/// <summary>
/// 根据指纹获取 artwork 原始数据
/// </summary>
public static byte[]? GetArtworkData(string fingerprint)
{
using IDbConnection connection = GetConnection();
connection.Open();
const string sql = "SELECT data FROM artworks WHERE fingerprint = @Fingerprint;";
return connection.ExecuteScalar<byte[]>(sql, new { Fingerprint = fingerprint });
}
/// <summary>
/// 清理 artworks 表中无 audio_files 引用的孤儿记录
/// </summary>
/// <returns>删除的记录数</returns>
public static int DeleteOrphanedArtworks()
{
using IDbConnection connection = GetConnection();
connection.Open();
const string sql = """
DELETE FROM artworks
WHERE fingerprint NOT IN (
SELECT DISTINCT artwork_fingerprint
FROM audio_files
WHERE artwork_fingerprint IS NOT NULL
);
""";
return connection.Execute(sql);
}
#endregion
} }