lashman/driftwood
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Add GitHub metadata enrichment for catalog apps

Browse Source 
Enrich catalog apps with GitHub API data (stars, version, downloads,
release date) via two strategies: background drip for repo-level info
and on-demand fetch when opening a detail page.

- Add github_enrichment module with API calls, asset filtering, and
  architecture auto-detection for AppImage downloads
- DB migrations v14/v15 for GitHub metadata and release asset columns
- Extract github_owner/repo from feed links during catalog sync
- Display colored stat cards (stars, version, downloads, released) on
  detail pages with on-demand enrichment
- Show stars and version on browse tiles and featured carousel cards
- Replace install button with SplitButton dropdown when multiple arch
  assets available, preferring detected architecture
- Disable install button until enrichment completes to prevent stale
  AppImageHub URL downloads
- Keep enrichment banner visible on catalog page until truly complete,
  showing paused state when rate-limited
- Add GitHub token and auto-enrich toggle to preferences
This commit is contained in:
lashman
2026-02-28 16:49:13 +02:00
parent 92c51dc39e
commit f89aafca6a
15 changed files with 3027 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

320
src/core/database.rs
View File

@@ -98,6 +98,16 @@ pub struct CatalogApp {
pub icon_url: Option<String>,
pub homepage: Option<String>,
pub license: Option<String>,
pub screenshots: Option<String>,
pub github_owner: Option<String>,
pub github_repo: Option<String>,
pub github_stars: Option<i64>,
pub github_downloads: Option<i64>,
pub latest_version: Option<String>,
pub release_date: Option<String>,
pub github_enriched_at: Option<String>,
pub github_download_url: Option<String>,
pub github_release_assets: Option<String>,
}
#[derive(Debug, Clone)]
@@ -400,6 +410,22 @@ impl Database {
self.migrate_to_v11()?;
}
if current_version < 12 {
self.migrate_to_v12()?;
}
if current_version < 13 {
self.migrate_to_v13()?;
}
if current_version < 14 {
self.migrate_to_v14()?;
}
if current_version < 15 {
self.migrate_to_v15()?;
}
// Ensure all expected columns exist (repairs DBs where a migration
// was updated after it had already run on this database)
self.ensure_columns()?;
@@ -838,6 +864,72 @@ impl Database {
Ok(())
}
fn migrate_to_v12(&self) -> SqlResult<()> {
let new_columns = [
"screenshots TEXT",
"license TEXT",
];
for col in &new_columns {
let sql = format!("ALTER TABLE catalog_apps ADD COLUMN {}", col);
self.conn.execute(&sql, []).ok();
}
self.conn.execute(
"UPDATE schema_version SET version = ?1",
params![12],
)?;
Ok(())
}
fn migrate_to_v13(&self) -> SqlResult<()> {
// Remove duplicate catalog_apps entries, keeping the row with the highest id
// (most recent insert) per (source_id, name) pair
self.conn.execute_batch(
"DELETE FROM catalog_apps WHERE id NOT IN (
SELECT MAX(id) FROM catalog_apps GROUP BY source_id, name
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_catalog_apps_source_name
ON catalog_apps(source_id, name);
UPDATE schema_version SET version = 13;"
)?;
Ok(())
}
fn migrate_to_v14(&self) -> SqlResult<()> {
let new_columns = [
"github_owner TEXT",
"github_repo TEXT",
"github_stars INTEGER",
"github_downloads INTEGER",
"release_date TEXT",
"github_enriched_at TEXT",
];
for col in &new_columns {
let sql = format!("ALTER TABLE catalog_apps ADD COLUMN {}", col);
self.conn.execute(&sql, []).ok();
}
self.conn.execute(
"UPDATE schema_version SET version = ?1",
params![14],
)?;
Ok(())
}
fn migrate_to_v15(&self) -> SqlResult<()> {
let new_columns = [
"github_download_url TEXT",
"github_release_assets TEXT",
];
for col in &new_columns {
let sql = format!("ALTER TABLE catalog_apps ADD COLUMN {}", col);
self.conn.execute(&sql, []).ok();
}
self.conn.execute(
"UPDATE schema_version SET version = ?1",
params![15],
)?;
Ok(())
}
pub fn upsert_appimage(
&self,
path: &str,
@@ -2069,7 +2161,8 @@ impl Database {
limit: i32,
) -> SqlResult<Vec<CatalogApp>> {
let mut sql = String::from(
"SELECT id, name, description, categories, download_url, icon_url, homepage, architecture
"SELECT id, name, description, categories, download_url, icon_url, homepage, license, screenshots,
github_owner, github_repo, github_stars, github_downloads, latest_version, release_date, github_enriched_at, github_download_url, github_release_assets
FROM catalog_apps WHERE 1=1"
);
let mut params_list: Vec<Box<dyn rusqlite::types::ToSql>> = Vec::new();
@@ -2101,6 +2194,16 @@ impl Database {
icon_url: row.get(5)?,
homepage: row.get(6)?,
license: row.get(7)?,
screenshots: row.get(8)?,
github_owner: row.get(9)?,
github_repo: row.get(10)?,
github_stars: row.get(11)?,
github_downloads: row.get(12)?,
latest_version: row.get(13)?,
release_date: row.get(14)?,
github_enriched_at: row.get(15)?,
github_download_url: row.get(16)?,
github_release_assets: row.get(17)?,
})
})?;
@@ -2113,7 +2216,8 @@ impl Database {
pub fn get_catalog_app(&self, id: i64) -> SqlResult<Option<CatalogApp>> {
let result = self.conn.query_row(
"SELECT id, name, description, categories, download_url, icon_url, homepage, architecture
"SELECT id, name, description, categories, download_url, icon_url, homepage, license, screenshots,
github_owner, github_repo, github_stars, github_downloads, latest_version, release_date, github_enriched_at, github_download_url, github_release_assets
FROM catalog_apps WHERE id = ?1",
params![id],
|row| {
@@ -2126,6 +2230,16 @@ impl Database {
icon_url: row.get(5)?,
homepage: row.get(6)?,
license: row.get(7)?,
screenshots: row.get(8)?,
github_owner: row.get(9)?,
github_repo: row.get(10)?,
github_stars: row.get(11)?,
github_downloads: row.get(12)?,
latest_version: row.get(13)?,
release_date: row.get(14)?,
github_enriched_at: row.get(15)?,
github_download_url: row.get(16)?,
github_release_assets: row.get(17)?,
})
},
);
@@ -2136,6 +2250,65 @@ impl Database {
}
}
/// Get featured catalog apps. Apps with GitHub stars sort first (by stars desc),
/// then unenriched apps get a deterministic shuffle that rotates every 15 minutes.
pub fn get_featured_catalog_apps(&self, limit: i32) -> SqlResult<Vec<CatalogApp>> {
// Time seed rotates every 15 minutes (900 seconds)
let time_seed = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs() / 900;
let mut stmt = self.conn.prepare(
"SELECT id, name, description, categories, download_url, icon_url, homepage, license, screenshots,
github_owner, github_repo, github_stars, github_downloads, latest_version, release_date, github_enriched_at, github_download_url, github_release_assets
FROM catalog_apps
WHERE icon_url IS NOT NULL AND icon_url != ''
AND description IS NOT NULL AND description != ''
AND screenshots IS NOT NULL AND screenshots != ''"
)?;
let rows = stmt.query_map([], |row| {
Ok(CatalogApp {
id: row.get(0)?,
name: row.get(1)?,
description: row.get(2)?,
categories: row.get(3)?,
download_url: row.get(4)?,
icon_url: row.get(5)?,
homepage: row.get(6)?,
license: row.get(7)?,
screenshots: row.get(8)?,
github_owner: row.get(9)?,
github_repo: row.get(10)?,
github_stars: row.get(11)?,
github_downloads: row.get(12)?,
latest_version: row.get(13)?,
release_date: row.get(14)?,
github_enriched_at: row.get(15)?,
github_download_url: row.get(16)?,
github_release_assets: row.get(17)?,
})
})?;
let mut apps: Vec<CatalogApp> = rows.collect::<SqlResult<Vec<_>>>()?;
// Enriched apps (with stars) sort first by stars descending,
// unenriched apps get the deterministic shuffle after them
apps.sort_by(|a, b| {
match (a.github_stars, b.github_stars) {
(Some(sa), Some(sb)) => sb.cmp(&sa),
(Some(_), None) => std::cmp::Ordering::Less,
(None, Some(_)) => std::cmp::Ordering::Greater,
(None, None) => {
let ha = (a.id as u64 ^ time_seed).wrapping_mul(0x517cc1b727220a95);
let hb = (b.id as u64 ^ time_seed).wrapping_mul(0x517cc1b727220a95);
ha.cmp(&hb)
}
}
});
apps.truncate(limit as usize);
Ok(apps)
}
pub fn get_catalog_categories(&self) -> SqlResult<Vec<(String, u32)>> {
let mut stmt = self.conn.prepare(
"SELECT categories FROM catalog_apps WHERE categories IS NOT NULL AND categories != ''"
@@ -2172,12 +2345,26 @@ impl Database {
homepage: Option<&str>,
file_size: Option<i64>,
architecture: Option<&str>,
screenshots: Option<&str>,
license: Option<&str>,
) -> SqlResult<()> {
self.conn.execute(
"INSERT OR REPLACE INTO catalog_apps
(source_id, name, description, categories, latest_version, download_url, icon_url, homepage, file_size, architecture, cached_at)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, datetime('now'))",
params![source_id, name, description, categories, latest_version, download_url, icon_url, homepage, file_size, architecture],
"INSERT INTO catalog_apps
(source_id, name, description, categories, latest_version, download_url, icon_url, homepage, file_size, architecture, screenshots, license, cached_at)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, datetime('now'))
ON CONFLICT(source_id, name) DO UPDATE SET
description = COALESCE(excluded.description, description),
categories = COALESCE(excluded.categories, categories),
latest_version = COALESCE(excluded.latest_version, latest_version),
download_url = excluded.download_url,
icon_url = COALESCE(excluded.icon_url, icon_url),
homepage = COALESCE(excluded.homepage, homepage),
file_size = COALESCE(excluded.file_size, file_size),
architecture = COALESCE(excluded.architecture, architecture),
screenshots = COALESCE(excluded.screenshots, screenshots),
license = COALESCE(excluded.license, license),
cached_at = datetime('now')",
params![source_id, name, description, categories, latest_version, download_url, icon_url, homepage, file_size, architecture, screenshots, license],
)?;
Ok(())
}
@@ -2232,6 +2419,125 @@ impl Database {
})?;
rows.collect()
}
pub fn get_catalog_app_by_source_and_name(&self, source_id: i64, name: &str) -> SqlResult<Option<i64>> {
let result = self.conn.query_row(
"SELECT id FROM catalog_apps WHERE source_id = ?1 AND name = ?2",
params![source_id, name],
|row| row.get(0),
);
match result {
Ok(id) => Ok(Some(id)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
Err(e) => Err(e),
}
}
// --- GitHub enrichment methods ---
pub fn update_catalog_app_github_repo(
&self,
app_id: i64,
owner: &str,
repo: &str,
) -> SqlResult<()> {
self.conn.execute(
"UPDATE catalog_apps SET github_owner = ?2, github_repo = ?3 WHERE id = ?1",
params![app_id, owner, repo],
)?;
Ok(())
}
pub fn update_catalog_app_github_metadata(
&self,
app_id: i64,
stars: i64,
pushed_at: Option<&str>,
) -> SqlResult<()> {
self.conn.execute(
"UPDATE catalog_apps SET github_stars = ?2, github_enriched_at = datetime('now') WHERE id = ?1",
params![app_id, stars],
)?;
// Store pushed_at in release_date if no release info yet
if let Some(pushed) = pushed_at {
self.conn.execute(
"UPDATE catalog_apps SET release_date = COALESCE(release_date, ?2) WHERE id = ?1",
params![app_id, pushed],
)?;
}
Ok(())
}
pub fn update_catalog_app_release_info(
&self,
app_id: i64,
version: Option<&str>,
date: Option<&str>,
downloads: Option<i64>,
github_download_url: Option<&str>,
github_release_assets: Option<&str>,
) -> SqlResult<()> {
self.conn.execute(
"UPDATE catalog_apps SET
latest_version = COALESCE(?2, latest_version),
release_date = COALESCE(?3, release_date),
github_downloads = COALESCE(?4, github_downloads),
github_download_url = COALESCE(?5, github_download_url),
github_release_assets = COALESCE(?6, github_release_assets),
github_enriched_at = datetime('now')
WHERE id = ?1",
params![app_id, version, date, downloads, github_download_url, github_release_assets],
)?;
Ok(())
}
pub fn get_unenriched_catalog_apps(&self, limit: i32) -> SqlResult<Vec<CatalogApp>> {
let mut stmt = self.conn.prepare(
"SELECT id, name, description, categories, download_url, icon_url, homepage, license, screenshots,
github_owner, github_repo, github_stars, github_downloads, latest_version, release_date, github_enriched_at, github_download_url, github_release_assets
FROM catalog_apps
WHERE github_owner IS NOT NULL AND github_enriched_at IS NULL
ORDER BY id
LIMIT ?1"
)?;
let rows = stmt.query_map(params![limit], |row| {
Ok(CatalogApp {
id: row.get(0)?,
name: row.get(1)?,
description: row.get(2)?,
categories: row.get(3)?,
download_url: row.get(4)?,
icon_url: row.get(5)?,
homepage: row.get(6)?,
license: row.get(7)?,
screenshots: row.get(8)?,
github_owner: row.get(9)?,
github_repo: row.get(10)?,
github_stars: row.get(11)?,
github_downloads: row.get(12)?,
latest_version: row.get(13)?,
release_date: row.get(14)?,
github_enriched_at: row.get(15)?,
github_download_url: row.get(16)?,
github_release_assets: row.get(17)?,
})
})?;
rows.collect()
}
pub fn catalog_enrichment_progress(&self) -> SqlResult<(i64, i64)> {
let enriched: i64 = self.conn.query_row(
"SELECT COUNT(*) FROM catalog_apps WHERE github_owner IS NOT NULL AND github_enriched_at IS NOT NULL",
[],
|row| row.get(0),
)?;
let total_with_github: i64 = self.conn.query_row(
"SELECT COUNT(*) FROM catalog_apps WHERE github_owner IS NOT NULL",
[],
|row| row.get(0),
)?;
Ok((enriched, total_with_github))
}
}
#[cfg(test)]
@@ -2402,7 +2708,7 @@ mod tests {
[],
|row| row.get(0),
).unwrap();
assert_eq!(version, 11);
assert_eq!(version, 15);
// All tables that should exist after the full v1-v7 migration chain
let expected_tables = [