lashman/driftwood
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Add GitHub metadata enrichment for catalog apps

Browse Source 
Enrich catalog apps with GitHub API data (stars, version, downloads,
release date) via two strategies: background drip for repo-level info
and on-demand fetch when opening a detail page.

- Add github_enrichment module with API calls, asset filtering, and
  architecture auto-detection for AppImage downloads
- DB migrations v14/v15 for GitHub metadata and release asset columns
- Extract github_owner/repo from feed links during catalog sync
- Display colored stat cards (stars, version, downloads, released) on
  detail pages with on-demand enrichment
- Show stars and version on browse tiles and featured carousel cards
- Replace install button with SplitButton dropdown when multiple arch
  assets available, preferring detected architecture
- Disable install button until enrichment completes to prevent stale
  AppImageHub URL downloads
- Keep enrichment banner visible on catalog page until truly complete,
  showing paused state when rate-limited
- Add GitHub token and auto-enrich toggle to preferences
This commit is contained in:
lashman
2026-02-28 16:49:13 +02:00
parent 92c51dc39e
commit f89aafca6a
15 changed files with 3027 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

394
src/core/github_enrichment.rs Normal file
View File

@@ -0,0 +1,394 @@
use super::database::Database;
// --- API response structs ---
#[derive(Debug, serde::Deserialize)]
pub struct GitHubRepoInfo {
pub stargazers_count: i64,
pub pushed_at: Option<String>,
pub description: Option<String>,
}
#[derive(Debug, serde::Deserialize)]
pub struct GitHubReleaseInfo {
pub tag_name: String,
pub published_at: Option<String>,
pub assets: Vec<GitHubReleaseAsset>,
}
#[derive(Debug, serde::Deserialize)]
pub struct GitHubReleaseAsset {
pub name: String,
pub browser_download_url: String,
pub download_count: i64,
pub size: i64,
}
// --- URL parsing ---
/// Extract (owner, repo) from a GitHub URL.
/// Tries download_url first (most reliable for GitHub releases), then homepage.
pub fn extract_github_repo(homepage: Option<&str>, download_url: &str) -> Option<(String, String)> {
// Try download URL first - most AppImageHub entries point to GitHub releases
if let Some(pair) = parse_github_url(download_url) {
return Some(pair);
}
// Fallback to homepage
if let Some(hp) = homepage {
if let Some(pair) = parse_github_url(hp) {
return Some(pair);
}
}
None
}
/// Parse `github.com/{owner}/{repo}` from a URL, stripping .git suffix if present.
fn parse_github_url(url: &str) -> Option<(String, String)> {
let stripped = url.trim_start_matches("https://")
.trim_start_matches("http://");
if !stripped.starts_with("github.com/") {
return None;
}
let path = stripped.strip_prefix("github.com/")?;
let parts: Vec<&str> = path.splitn(3, '/').collect();
if parts.len() < 2 {
return None;
}
let owner = parts[0];
let repo = parts[1]
.trim_end_matches(".git")
.split('?').next().unwrap_or(parts[1]);
if owner.is_empty() || repo.is_empty() {
return None;
}
Some((owner.to_string(), repo.to_string()))
}
// --- API calls ---
fn github_get(url: &str, token: &str) -> Result<(String, u32), String> {
let mut req = ureq::get(url)
.header("Accept", "application/vnd.github+json")
.header("User-Agent", "Driftwood-AppImage-Manager");
if !token.is_empty() {
req = req.header("Authorization", &format!("Bearer {}", token));
}
let mut response = req.call()
.map_err(|e| format!("GitHub API error: {}", e))?;
// Parse rate limit header
let remaining: u32 = response.headers()
.get("x-ratelimit-remaining")
.and_then(|v| v.to_str().ok())
.and_then(|v| v.parse().ok())
.unwrap_or(0);
let body = response.body_mut().read_to_string()
.map_err(|e| format!("Read error: {}", e))?;
Ok((body, remaining))
}
pub fn fetch_repo_info(owner: &str, repo: &str, token: &str) -> Result<(GitHubRepoInfo, u32), String> {
let url = format!("https://api.github.com/repos/{}/{}", owner, repo);
let (body, remaining) = github_get(&url, token)?;
let info: GitHubRepoInfo = serde_json::from_str(&body)
.map_err(|e| format!("Parse error: {}", e))?;
Ok((info, remaining))
}
pub fn fetch_release_info(owner: &str, repo: &str, token: &str) -> Result<(GitHubReleaseInfo, u32), String> {
let url = format!("https://api.github.com/repos/{}/{}/releases/latest", owner, repo);
let (body, remaining) = github_get(&url, token)?;
let info: GitHubReleaseInfo = serde_json::from_str(&body)
.map_err(|e| format!("Parse error: {}", e))?;
Ok((info, remaining))
}
// --- AppImage asset filtering ---
/// A simplified release asset for storage (JSON-serializable).
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct AppImageAsset {
pub name: String,
pub url: String,
pub size: i64,
}
/// Filter release assets to only AppImage files.
pub fn filter_appimage_assets(assets: &[GitHubReleaseAsset]) -> Vec<AppImageAsset> {
assets.iter()
.filter(|a| {
let lower = a.name.to_lowercase();
lower.ends_with(".appimage") || lower.ends_with(".appimage.zsync")
})
.filter(|a| !a.name.to_lowercase().ends_with(".zsync"))
.map(|a| AppImageAsset {
name: a.name.clone(),
url: a.browser_download_url.clone(),
size: a.size,
})
.collect()
}
/// Detect the current system architecture string as used in AppImage filenames.
pub fn detect_arch() -> &'static str {
#[cfg(target_arch = "x86_64")]
{ "x86_64" }
#[cfg(target_arch = "aarch64")]
{ "aarch64" }
#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
{ std::env::consts::ARCH }
}
/// Pick the best AppImage asset for the current architecture.
/// Returns the matching asset, or the first one if no arch match.
pub fn pick_best_asset(assets: &[AppImageAsset]) -> Option<&AppImageAsset> {
if assets.is_empty() {
return None;
}
let arch = detect_arch();
// Prefer exact arch match in filename
let arch_match = assets.iter().find(|a| {
let lower = a.name.to_lowercase();
lower.contains(&arch.to_lowercase())
});
arch_match.or(assets.first())
}
// --- Enrichment logic ---
/// Enrich a catalog app with repo-level info (stars, pushed_at).
pub fn enrich_app_repo_info(
db: &Database,
app_id: i64,
owner: &str,
repo: &str,
token: &str,
) -> Result<u32, String> {
let (info, remaining) = fetch_repo_info(owner, repo, token)?;
db.update_catalog_app_github_metadata(app_id, info.stargazers_count, info.pushed_at.as_deref())
.map_err(|e| format!("DB error: {}", e))?;
Ok(remaining)
}
/// Enrich a catalog app with release info (version, date, downloads, assets).
pub fn enrich_app_release_info(
db: &Database,
app_id: i64,
owner: &str,
repo: &str,
token: &str,
) -> Result<u32, String> {
let (info, remaining) = fetch_release_info(owner, repo, token)?;
// Clean version string (strip leading "v")
let version = info.tag_name.strip_prefix('v')
.unwrap_or(&info.tag_name)
.to_string();
// Sum download counts across all assets
let total_downloads: i64 = info.assets.iter().map(|a| a.download_count).sum();
// Extract AppImage assets and pick the best download URL
let appimage_assets = filter_appimage_assets(&info.assets);
let best_url = pick_best_asset(&appimage_assets).map(|a| a.url.as_str());
let assets_json = if appimage_assets.is_empty() {
None
} else {
serde_json::to_string(&appimage_assets).ok()
};
db.update_catalog_app_release_info(
app_id,
Some(&version),
info.published_at.as_deref(),
if total_downloads > 0 { Some(total_downloads) } else { None },
best_url,
assets_json.as_deref(),
).map_err(|e| format!("DB error: {}", e))?;
Ok(remaining)
}
/// Background enrichment: process a batch of unenriched apps.
/// Returns (count_enriched, should_continue).
pub fn background_enrich_batch(
db: &Database,
token: &str,
batch_size: i32,
on_progress: &dyn Fn(i64, i64),
) -> Result<(u32, bool), String> {
let apps = db.get_unenriched_catalog_apps(batch_size)
.map_err(|e| format!("DB error: {}", e))?;
if apps.is_empty() {
return Ok((0, false));
}
let mut enriched = 0u32;
for app in &apps {
let owner = match app.github_owner.as_deref() {
Some(o) => o,
None => continue,
};
let repo = match app.github_repo.as_deref() {
Some(r) => r,
None => continue,
};
match enrich_app_repo_info(db, app.id, owner, repo, token) {
Ok(remaining) => {
enriched += 1;
// Report progress
if let Ok((done, total)) = db.catalog_enrichment_progress() {
on_progress(done, total);
}
// Stop if rate limit is getting low
if remaining < 5 {
log::info!("GitHub rate limit low ({}), pausing enrichment", remaining);
return Ok((enriched, false));
}
}
Err(e) => {
log::warn!("Failed to enrich {}/{}: {}", owner, repo, e);
// Mark as enriched anyway so we don't retry forever
db.update_catalog_app_github_metadata(app.id, 0, None).ok();
}
}
// Sleep between calls to be respectful
std::thread::sleep(std::time::Duration::from_secs(1));
}
Ok((enriched, enriched > 0))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_extract_github_repo_from_download() {
let result = extract_github_repo(
None,
"https://github.com/nickvdp/deno-spreadsheets/releases/download/v0.3.0/app.AppImage",
);
assert_eq!(result, Some(("nickvdp".to_string(), "deno-spreadsheets".to_string())));
}
#[test]
fn test_extract_github_repo_from_homepage() {
let result = extract_github_repo(
Some("https://github.com/nickvdp/deno-spreadsheets"),
"https://example.com/download.AppImage",
);
assert_eq!(result, Some(("nickvdp".to_string(), "deno-spreadsheets".to_string())));
}
#[test]
fn test_extract_github_repo_with_git_suffix() {
let result = extract_github_repo(
Some("https://github.com/user/repo.git"),
"https://example.com/download.AppImage",
);
assert_eq!(result, Some(("user".to_string(), "repo".to_string())));
}
#[test]
fn test_extract_github_repo_non_github() {
let result = extract_github_repo(
Some("https://gitlab.com/user/repo"),
"https://sourceforge.net/download.AppImage",
);
assert_eq!(result, None);
}
#[test]
fn test_parse_github_url_empty() {
assert_eq!(parse_github_url(""), None);
assert_eq!(parse_github_url("https://github.com/"), None);
assert_eq!(parse_github_url("https://github.com/user"), None);
}
#[test]
fn test_filter_appimage_assets() {
let assets = vec![
GitHubReleaseAsset {
name: "app-x86_64.AppImage".to_string(),
browser_download_url: "https://github.com/u/r/releases/download/v1/app-x86_64.AppImage".to_string(),
download_count: 100,
size: 50_000_000,
},
GitHubReleaseAsset {
name: "app-aarch64.AppImage".to_string(),
browser_download_url: "https://github.com/u/r/releases/download/v1/app-aarch64.AppImage".to_string(),
download_count: 20,
size: 48_000_000,
},
GitHubReleaseAsset {
name: "app-x86_64.AppImage.zsync".to_string(),
browser_download_url: "https://github.com/u/r/releases/download/v1/app-x86_64.AppImage.zsync".to_string(),
download_count: 5,
size: 1000,
},
GitHubReleaseAsset {
name: "source.tar.gz".to_string(),
browser_download_url: "https://github.com/u/r/releases/download/v1/source.tar.gz".to_string(),
download_count: 10,
size: 2_000_000,
},
];
let filtered = filter_appimage_assets(&assets);
assert_eq!(filtered.len(), 2);
assert_eq!(filtered[0].name, "app-x86_64.AppImage");
assert_eq!(filtered[1].name, "app-aarch64.AppImage");
}
#[test]
fn test_pick_best_asset_prefers_arch() {
let assets = vec![
AppImageAsset {
name: "app-aarch64.AppImage".to_string(),
url: "https://example.com/aarch64".to_string(),
size: 48_000_000,
},
AppImageAsset {
name: "app-x86_64.AppImage".to_string(),
url: "https://example.com/x86_64".to_string(),
size: 50_000_000,
},
];
let best = pick_best_asset(&assets).unwrap();
// On x86_64 systems this should pick x86_64, on aarch64 it picks aarch64
let arch = detect_arch();
assert!(best.name.contains(arch));
}
#[test]
fn test_pick_best_asset_empty() {
let assets: Vec<AppImageAsset> = vec![];
assert!(pick_best_asset(&assets).is_none());
}
#[test]
fn test_pick_best_asset_single() {
let assets = vec![
AppImageAsset {
name: "app.AppImage".to_string(),
url: "https://example.com/app".to_string(),
size: 50_000_000,
},
];
let best = pick_best_asset(&assets).unwrap();
assert_eq!(best.name, "app.AppImage");
}
}