lashman/driftwood
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Add AppImageHub.com OCS API as primary catalog source

Browse Source
This commit is contained in:
lashman
2026-02-28 20:33:40 +02:00
parent 848f4e7de7
commit 960eab965d
16 changed files with 2394 additions and 417 deletions
Download Patch File Download Diff File Expand all files Collapse all files

671
src/core/catalog.rs
View File

@@ -20,6 +20,7 @@ pub struct CatalogSource {
#[derive(Debug, Clone, PartialEq)]
pub enum CatalogType {
AppImageHub,
OcsAppImageHub,
GitHubSearch,
Custom,
}
@@ -28,6 +29,7 @@ impl CatalogType {
pub fn as_str(&self) -> &str {
match self {
Self::AppImageHub => "appimage-hub",
Self::OcsAppImageHub => "ocs-appimagehub",
Self::GitHubSearch => "github-search",
Self::Custom => "custom",
}
@@ -36,6 +38,7 @@ impl CatalogType {
pub fn from_str(s: &str) -> Self {
match s {
"appimage-hub" => Self::AppImageHub,
"ocs-appimagehub" => Self::OcsAppImageHub,
"github-search" => Self::GitHubSearch,
_ => Self::Custom,
}
@@ -63,10 +66,138 @@ pub struct CatalogApp {
/// Default AppImageHub registry URL.
const APPIMAGEHUB_API_URL: &str = "https://appimage.github.io/feed.json";
/// OCS API base URL for AppImageHub.com.
const OCS_API_URL: &str = "https://api.appimagehub.com/ocs/v1/content/data";
const OCS_PAGE_SIZE: u32 = 100;
// --- OCS API response types ---
/// Deserialize a JSON value that may be a number, a numeric string, or an empty string.
/// The OCS API is loosely typed and sometimes returns "" instead of null for numeric fields.
fn deserialize_lenient_i64<'de, D>(deserializer: D) -> Result<Option<i64>, D::Error>
where
D: serde::Deserializer<'de>,
{
use serde::Deserialize;
let v = serde_json::Value::deserialize(deserializer)?;
match &v {
serde_json::Value::Number(n) => Ok(n.as_i64()),
serde_json::Value::String(s) if s.is_empty() => Ok(None),
serde_json::Value::String(s) => Ok(s.parse().ok()),
serde_json::Value::Null => Ok(None),
_ => Ok(None),
}
}
fn deserialize_lenient_i32<'de, D>(deserializer: D) -> Result<Option<i32>, D::Error>
where
D: serde::Deserializer<'de>,
{
use serde::Deserialize;
let v = serde_json::Value::deserialize(deserializer)?;
match &v {
serde_json::Value::Number(n) => Ok(n.as_i64().map(|n| n as i32)),
serde_json::Value::String(s) if s.is_empty() => Ok(None),
serde_json::Value::String(s) => Ok(s.parse().ok()),
serde_json::Value::Null => Ok(None),
_ => Ok(None),
}
}
#[derive(Debug, serde::Deserialize)]
struct OcsResponse {
data: Vec<OcsItem>,
}
#[derive(Debug, serde::Deserialize)]
struct OcsItem {
#[serde(deserialize_with = "deserialize_lenient_i64", default)]
id: Option<i64>,
#[serde(default)]
name: String,
description: Option<String>,
summary: Option<String>,
downloads: Option<String>,
#[serde(deserialize_with = "deserialize_lenient_i32", default)]
score: Option<i32>,
typename: Option<String>,
personid: Option<String>,
version: Option<String>,
tags: Option<String>,
changed: Option<String>,
created: Option<String>,
previewpic1: Option<String>,
previewpic2: Option<String>,
previewpic3: Option<String>,
previewpic4: Option<String>,
previewpic5: Option<String>,
previewpic6: Option<String>,
smallpreviewpic1: Option<String>,
downloadlink1: Option<String>,
downloadname1: Option<String>,
#[serde(deserialize_with = "deserialize_lenient_i64", default)]
downloadsize1: Option<i64>,
download_package_arch1: Option<String>,
download_package_type1: Option<String>,
downloadmd5sum1: Option<String>,
detailpage: Option<String>,
#[serde(deserialize_with = "deserialize_lenient_i64", default)]
comments: Option<i64>,
}
/// Extra OCS-specific metadata not in the in-memory CatalogApp.
struct OcsExtra {
ocs_id: i64,
ocs_downloads: Option<i64>,
ocs_score: Option<i64>,
ocs_typename: Option<String>,
ocs_personid: Option<String>,
ocs_description: Option<String>,
ocs_summary: Option<String>,
ocs_version: Option<String>,
ocs_tags: Option<String>,
ocs_changed: Option<String>,
ocs_preview_url: Option<String>,
ocs_detailpage: Option<String>,
ocs_created: Option<String>,
ocs_downloadname: Option<String>,
ocs_downloadsize: Option<i64>,
ocs_arch: Option<String>,
ocs_md5sum: Option<String>,
ocs_comments: Option<i64>,
}
/// OCS download resolution response.
#[derive(Debug, serde::Deserialize)]
struct OcsDownloadResponse {
data: Vec<OcsDownloadItem>,
}
#[derive(Debug, serde::Deserialize)]
struct OcsDownloadItem {
#[serde(rename = "downloadlink")]
download_link: Option<String>,
}
/// A downloadable file from an OCS content item.
/// Each OCS item can have multiple download files (different versions).
#[derive(Debug, Clone)]
pub struct OcsDownloadFile {
pub slot: u32,
pub ocs_id: i64,
pub filename: String,
pub version: String,
pub size_kb: Option<i64>,
pub arch: Option<String>,
pub pkg_type: Option<String>,
}
/// Sync a catalog source - fetch the index and store entries in the database.
/// Progress updates sent during catalog sync.
#[derive(Debug, Clone)]
pub enum SyncProgress {
/// Starting sync for a named source.
SourceStarted { name: String, source_index: u32, source_count: u32 },
/// Fetching the feed from the remote source.
FetchingFeed,
/// Feed fetched, total number of apps found.
@@ -75,8 +206,10 @@ pub enum SyncProgress {
CachingIcon { current: u32, total: u32, app_name: String },
/// Saving apps to the database.
SavingApps { current: u32, total: u32 },
/// Sync complete.
/// Single source sync complete.
Done { total: u32 },
/// All sources finished syncing.
AllDone,
}
pub fn sync_catalog(db: &Database, source: &CatalogSource) -> Result<u32, CatalogError> {
@@ -90,6 +223,13 @@ pub fn sync_catalog_with_progress(
) -> Result<u32, CatalogError> {
on_progress(SyncProgress::FetchingFeed);
match source.source_type {
CatalogType::OcsAppImageHub => {
return sync_ocs_catalog(db, source, on_progress);
}
_ => {}
}
let apps = match source.source_type {
CatalogType::AppImageHub => fetch_appimage_hub()?,
CatalogType::Custom => fetch_custom_catalog(&source.url)?,
@@ -97,6 +237,7 @@ pub fn sync_catalog_with_progress(
log::warn!("GitHub catalog search not yet implemented");
Vec::new()
}
CatalogType::OcsAppImageHub => unreachable!(),
};
let total = apps.len() as u32;
@@ -107,9 +248,18 @@ pub fn sync_catalog_with_progress(
log::info!("Cached {} catalog icons", icon_count);
let source_id = source.id.ok_or(CatalogError::NoSourceId)?;
// Build set of OCS app names for dedup (skip apps already in OCS source)
let ocs_names = get_ocs_source_names(db);
let mut count = 0u32;
for app in &apps {
// Deduplicate: skip if this app name exists in the OCS source
if ocs_names.contains(&app.name.to_lowercase()) {
continue;
}
count += 1;
on_progress(SyncProgress::SavingApps { current: count, total });
@@ -153,20 +303,54 @@ pub fn sync_catalog_with_progress(
}
/// Download an AppImage from the catalog to a local directory.
/// If `ocs_id` is provided, resolves a fresh download URL from the OCS API
/// (since OCS download links use JWT tokens that expire).
pub fn install_from_catalog(app: &CatalogApp, install_dir: &Path) -> Result<PathBuf, CatalogError> {
install_from_catalog_with_ocs(app, install_dir, None, 1)
}
/// Download an AppImage, optionally resolving a fresh OCS download URL.
/// `ocs_id` - the OCS content ID; `ocs_slot` - the file slot number (1-based, default 1).
pub fn install_from_catalog_with_ocs(
app: &CatalogApp,
install_dir: &Path,
ocs_id: Option<i64>,
ocs_slot: u32,
) -> Result<PathBuf, CatalogError> {
fs::create_dir_all(install_dir).map_err(|e| CatalogError::Io(e.to_string()))?;
// Derive filename from URL
let filename = app.download_url
// For OCS apps, resolve a fresh download URL (JWT links expire)
let download_url = if let Some(id) = ocs_id {
match resolve_ocs_download(id, ocs_slot) {
Ok(url) => url,
Err(e) => {
log::warn!("Failed to resolve OCS download, falling back to stored URL: {}", e);
app.download_url.clone()
}
}
} else {
app.download_url.clone()
};
// Derive filename from URL (strip query params)
let url_path = download_url.split('?').next().unwrap_or(&download_url);
let filename = url_path
.rsplit('/')
.next()
.unwrap_or("downloaded.AppImage");
let dest = install_dir.join(filename);
// If filename doesn't look like an AppImage, use the app name
let filename = if filename.contains(".AppImage") || filename.ends_with(".appimage") {
filename.to_string()
} else {
format!("{}.AppImage", sanitize_filename(&app.name))
};
log::info!("Downloading {} to {}", app.download_url, dest.display());
let dest = install_dir.join(&filename);
let response = ureq::get(&app.download_url)
log::info!("Downloading {} to {}", download_url, dest.display());
let response = ureq::get(&download_url)
.call()
.map_err(|e| CatalogError::Network(e.to_string()))?;
@@ -195,6 +379,419 @@ pub fn install_from_catalog(app: &CatalogApp, install_dir: &Path) -> Result<Path
Ok(dest)
}
/// Get all app names from the OCS source for deduplication.
fn get_ocs_source_names(db: &Database) -> std::collections::HashSet<String> {
let sources = db.get_catalog_sources().unwrap_or_default();
for src in &sources {
if src.source_type == "ocs-appimagehub" {
return db.get_catalog_app_names_for_source(src.id).unwrap_or_default();
}
}
std::collections::HashSet::new()
}
/// Sync the OCS AppImageHub catalog (primary source).
fn sync_ocs_catalog(
db: &Database,
source: &CatalogSource,
on_progress: &dyn Fn(SyncProgress),
) -> Result<u32, CatalogError> {
let source_id = source.id.ok_or(CatalogError::NoSourceId)?;
let items = fetch_ocs_catalog(on_progress)?;
let total = items.len() as u32;
on_progress(SyncProgress::FeedFetched { total });
let mut count = 0u32;
for (app, extra) in &items {
count += 1;
if count % 50 == 0 || count == total {
on_progress(SyncProgress::SavingApps { current: count, total });
}
let screenshots_str = if app.screenshots.is_empty() {
None
} else {
Some(app.screenshots.join(";"))
};
db.insert_ocs_catalog_app(
source_id,
&app.name,
extra.ocs_id,
extra.ocs_description.as_deref(),
extra.ocs_summary.as_deref(),
Some(app.categories.join(";")).as_deref().filter(|s| !s.is_empty()),
extra.ocs_version.as_deref(),
&app.download_url,
app.icon_url.as_deref(),
app.homepage.as_deref(),
screenshots_str.as_deref(),
extra.ocs_downloads,
extra.ocs_score,
extra.ocs_typename.as_deref(),
extra.ocs_personid.as_deref(),
extra.ocs_tags.as_deref(),
extra.ocs_changed.as_deref(),
extra.ocs_preview_url.as_deref(),
app.license.as_deref(),
extra.ocs_detailpage.as_deref(),
extra.ocs_created.as_deref(),
extra.ocs_downloadname.as_deref(),
extra.ocs_downloadsize,
extra.ocs_arch.as_deref(),
extra.ocs_md5sum.as_deref(),
extra.ocs_comments,
).ok();
// Try to extract GitHub owner/repo from github_link (extracted from description),
// homepage, or download URL
let github_result = app.github_link.as_deref()
.and_then(|gl| github_enrichment::extract_github_repo(Some(gl), ""))
.or_else(|| github_enrichment::extract_github_repo(
app.homepage.as_deref(),
&app.download_url,
));
if let Some((owner, repo)) = github_result {
if let Ok(Some(db_app)) = db.get_catalog_app_by_source_and_name(source_id, &app.name) {
db.update_catalog_app_github_repo(db_app, &owner, &repo).ok();
}
}
}
// Clear stale screenshot cache so new screenshot indices map correctly
clear_screenshot_cache();
// Clear and re-cache icons for OCS apps (URLs may have changed)
let ocs_apps: Vec<CatalogApp> = items.into_iter().map(|(a, _)| a).collect();
clear_ocs_icon_cache(&ocs_apps);
let icon_count = cache_catalog_icons_with_progress(&ocs_apps, on_progress);
log::info!("Cached {} OCS catalog icons", icon_count);
db.update_catalog_source_sync(source_id, count as i32).ok();
on_progress(SyncProgress::Done { total: count });
Ok(count)
}
/// Fetch all apps from the OCS API, paginating through all pages.
fn fetch_ocs_catalog(
_on_progress: &dyn Fn(SyncProgress),
) -> Result<Vec<(CatalogApp, OcsExtra)>, CatalogError> {
let mut all_items = Vec::new();
let mut page = 0u32;
loop {
let url = format!(
"{}?format=json&pagesize={}&page={}",
OCS_API_URL, OCS_PAGE_SIZE, page
);
let response = ureq::get(&url)
.call()
.map_err(|e| CatalogError::Network(format!("OCS fetch page {} failed: {}", page, e)))?;
let body = response.into_body().read_to_string()
.map_err(|e| CatalogError::Network(e.to_string()))?;
let ocs_resp: OcsResponse = serde_json::from_str(&body)
.map_err(|e| CatalogError::Parse(format!("OCS JSON parse failed on page {}: {}", page, e)))?;
if ocs_resp.data.is_empty() {
break;
}
let page_count = ocs_resp.data.len();
for item in ocs_resp.data {
// Skip items with empty name, no id, or no download link
if item.name.trim().is_empty() {
continue;
}
let ocs_id = match item.id {
Some(id) => id,
None => continue,
};
let download_url = match item.downloadlink1 {
Some(ref dl) if !dl.is_empty() => dl.clone(),
_ => continue,
};
// Skip non-AppImage downloads (e.g. .dmg, .exe, .rpm, .zip)
let pkg_type = item.download_package_type1.as_deref().unwrap_or("");
if !pkg_type.is_empty() && pkg_type != "appimage" {
continue;
}
// Also check filename extension as fallback
if let Some(ref dname) = item.downloadname1 {
let lower = dname.to_lowercase();
if lower.ends_with(".dmg") || lower.ends_with(".exe")
|| lower.ends_with(".rpm") || lower.ends_with(".deb")
|| lower.ends_with(".zip") || lower.ends_with(".msi")
|| lower.ends_with(".pkg") || lower.ends_with(".7z")
{
continue;
}
}
let ocs_downloads = item.downloads
.as_deref()
.and_then(|s| s.parse::<i64>().ok());
// previewpic1 is always the app icon/logo on appimagehub.com.
// Actual screenshots start from previewpic2 onward.
let screenshots: Vec<String> = [
item.previewpic2.as_deref(),
item.previewpic3.as_deref(),
item.previewpic4.as_deref(),
item.previewpic5.as_deref(),
item.previewpic6.as_deref(),
]
.iter()
.filter_map(|p| p.filter(|s| !s.is_empty()).map(|s| s.to_string()))
.collect();
let categories = map_ocs_category(item.typename.as_deref().unwrap_or(""));
// Extract GitHub link from HTML description if present
let github_link = item.description.as_deref()
.and_then(extract_github_link_from_html);
// Use detailpage as homepage since OCS API has no homepage field
let homepage = item.detailpage.clone();
// Use smallpreviewpic1 as icon, fall back to previewpic1 (which is the icon/logo).
// Shrink from 770x540 to 100x100 since we only display at 48px.
let icon_url = item.smallpreviewpic1.clone()
.filter(|s| !s.is_empty())
.or_else(|| item.previewpic1.clone().filter(|s| !s.is_empty()))
.map(|url| shrink_ocs_image_url(&url, "100x100"));
let catalog_app = CatalogApp {
name: item.name.clone(),
description: item.summary.clone().or(item.description.clone()),
categories,
latest_version: item.version.clone(),
download_url,
icon_url,
homepage,
file_size: item.downloadsize1.map(|s| (s * 1024) as u64), // API gives KB, we store bytes
architecture: item.download_package_arch1.clone(),
screenshots,
license: None,
github_link,
};
let extra = OcsExtra {
ocs_id,
ocs_downloads,
ocs_score: item.score.map(|s| s as i64),
ocs_typename: item.typename,
ocs_personid: item.personid,
ocs_description: item.description,
ocs_summary: item.summary,
ocs_version: item.version,
ocs_tags: item.tags,
ocs_changed: item.changed,
ocs_preview_url: item.previewpic1,
ocs_detailpage: item.detailpage,
ocs_created: item.created,
ocs_downloadname: item.downloadname1,
ocs_downloadsize: item.downloadsize1,
ocs_arch: item.download_package_arch1,
ocs_md5sum: item.downloadmd5sum1,
ocs_comments: item.comments,
};
all_items.push((catalog_app, extra));
}
log::info!("OCS page {}: {} items (total so far: {})", page, page_count, all_items.len());
page += 1;
}
log::info!("OCS catalog fetch complete: {} apps total", all_items.len());
Ok(all_items)
}
/// Extract a GitHub repository URL from OCS HTML description.
/// Many OCS app descriptions contain links to their GitHub repo.
fn extract_github_link_from_html(html: &str) -> Option<String> {
// Find "github.com/" in the text (case-insensitive search)
let lower = html.to_lowercase();
let marker = "github.com/";
let idx = lower.find(marker)?;
// Walk backwards from idx to find the URL scheme (https:// or http://)
let before = &html[..idx];
let scheme_start = before.rfind("https://").or_else(|| before.rfind("http://"))?;
// Make sure the scheme is close to the github.com part (no intervening garbage)
if idx - scheme_start > 20 {
return None;
}
// From github.com/, extract owner/repo
let after_marker = &html[idx + marker.len()..];
// Take characters until we hit whitespace, quotes, <, >, or end
let end = after_marker.find(|c: char| c.is_whitespace() || c == '"' || c == '\'' || c == '<' || c == '>' || c == ')').unwrap_or(after_marker.len());
let path = &after_marker[..end];
// Split by / to get owner/repo (ignore further path components)
let parts: Vec<&str> = path.splitn(3, '/').collect();
if parts.len() >= 2 && !parts[0].is_empty() && !parts[1].is_empty() {
Some(format!("https://github.com/{}/{}", parts[0], parts[1]))
} else {
None
}
}
/// Map OCS typename to FreeDesktop categories.
fn map_ocs_category(typename: &str) -> Vec<String> {
let s = typename.to_lowercase();
if s.contains("game") {
vec!["Game".into()]
} else if s.contains("audio") || s.contains("music") {
vec!["Audio".into()]
} else if s.contains("video") || s.contains("multimedia") {
vec!["Video".into()]
} else if s.contains("graphic") || s.contains("photo") {
vec!["Graphics".into()]
} else if s.contains("office") || s.contains("document") {
vec!["Office".into()]
} else if s.contains("development") || s.contains("programming") {
vec!["Development".into()]
} else if s.contains("education") || s.contains("science") {
vec!["Education".into()]
} else if s.contains("network") || s.contains("internet") || s.contains("chat") || s.contains("browser") {
vec!["Network".into()]
} else if s.contains("system") || s.contains("tool") || s.contains("util") {
vec!["System".into()]
} else if typename.is_empty() {
Vec::new()
} else {
vec![typename.to_string()]
}
}
/// Resolve a fresh download URL for an OCS app at install time.
/// OCS download links are JWT-authenticated and expire, so we fetch a fresh one.
/// `slot` is the 1-based download file slot (default 1 for the primary file).
pub fn resolve_ocs_download(ocs_id: i64, slot: u32) -> Result<String, CatalogError> {
let url = format!(
"https://api.appimagehub.com/ocs/v1/content/download/{}/{}?format=json",
ocs_id, slot
);
let response = ureq::get(&url)
.call()
.map_err(|e| CatalogError::Network(format!("OCS download resolve failed: {}", e)))?;
let body = response.into_body().read_to_string()
.map_err(|e| CatalogError::Network(e.to_string()))?;
let dl_resp: OcsDownloadResponse = serde_json::from_str(&body)
.map_err(|e| CatalogError::Parse(format!("OCS download JSON parse failed: {}", e)))?;
dl_resp.data.first()
.and_then(|item| item.download_link.clone())
.filter(|link| !link.is_empty())
.ok_or_else(|| CatalogError::Network("No download link in OCS response".into()))
}
/// Fetch all available download files for an OCS content item.
/// Returns only AppImage files, sorted with newest version first.
pub fn fetch_ocs_download_files(ocs_id: i64) -> Result<Vec<OcsDownloadFile>, CatalogError> {
let url = format!(
"https://api.appimagehub.com/ocs/v1/content/data/{}?format=json",
ocs_id
);
let response = ureq::get(&url)
.call()
.map_err(|e| CatalogError::Network(format!("OCS files fetch failed: {}", e)))?;
let body = response.into_body().read_to_string()
.map_err(|e| CatalogError::Network(e.to_string()))?;
let json: serde_json::Value = serde_json::from_str(&body)
.map_err(|e| CatalogError::Parse(format!("OCS files JSON parse failed: {}", e)))?;
let data = json.get("data")
.and_then(|d| d.as_array())
.ok_or_else(|| CatalogError::Parse("No data array in OCS response".into()))?;
let item = data.first()
.ok_or_else(|| CatalogError::Parse("Empty data array in OCS response".into()))?;
let mut files = Vec::new();
for slot in 1..=20u32 {
let link_key = format!("downloadlink{}", slot);
let name_key = format!("downloadname{}", slot);
// Stop if no download link for this slot
let link = match item.get(&link_key).and_then(|v| v.as_str()) {
Some(l) if !l.is_empty() => l,
_ => break,
};
let filename = item.get(&name_key)
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
// Skip non-AppImage files
let pkg_type_key = format!("download_package_type{}", slot);
let pkg_type = item.get(&pkg_type_key)
.and_then(|v| v.as_str())
.unwrap_or("");
if !pkg_type.is_empty() && pkg_type != "appimage" {
continue;
}
let lower_name = filename.to_lowercase();
if lower_name.ends_with(".dmg") || lower_name.ends_with(".exe")
|| lower_name.ends_with(".rpm") || lower_name.ends_with(".deb")
|| lower_name.ends_with(".zip") || lower_name.ends_with(".msi")
|| lower_name.ends_with(".pkg") || lower_name.ends_with(".7z")
{
continue;
}
let version_key = format!("download_version{}", slot);
let version = item.get(&version_key)
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
let size_key = format!("downloadsize{}", slot);
let size_kb = item.get(&size_key)
.and_then(|v| v.as_str().and_then(|s| s.parse().ok()).or_else(|| v.as_i64()));
let arch_key = format!("download_package_arch{}", slot);
let arch = item.get(&arch_key)
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
.map(|s| s.to_string());
// Ignore the actual download link (JWT, may be expired) - we resolve fresh at install time
let _ = link;
files.push(OcsDownloadFile {
slot,
ocs_id,
filename,
version,
size_kb,
arch,
pkg_type: if pkg_type.is_empty() { None } else { Some(pkg_type.to_string()) },
});
}
// Sort: newest version first (by slot descending since newer versions are typically added last)
files.reverse();
Ok(files)
}
/// Fetch the AppImageHub feed and parse it into CatalogApp entries.
fn fetch_appimage_hub() -> Result<Vec<CatalogApp>, CatalogError> {
let response = ureq::get(APPIMAGEHUB_API_URL)
@@ -273,8 +870,18 @@ fn fetch_custom_catalog(url: &str) -> Result<Vec<CatalogApp>, CatalogError> {
}).collect())
}
/// Ensure the default AppImageHub source exists in the database.
/// Ensure the default catalog sources exist in the database.
/// OCS AppImageHub.com is the primary source (richer metadata), and
/// appimage.github.io is the secondary source for apps not in OCS.
pub fn ensure_default_sources(db: &Database) {
// Primary: OCS AppImageHub.com (insert first so it syncs first)
db.upsert_catalog_source(
"AppImageHub.com",
OCS_API_URL,
"ocs-appimagehub",
).ok();
// Secondary: appimage.github.io feed
db.upsert_catalog_source(
"AppImageHub",
APPIMAGEHUB_API_URL,
@@ -319,6 +926,45 @@ pub fn screenshot_cache_dir() -> PathBuf {
dir
}
/// Shrink an OCS CDN image URL to a smaller cache size.
/// OCS URLs look like https://images.pling.com/cache/770x540-4/img/...
/// We can replace the size portion to get smaller images.
fn shrink_ocs_image_url(url: &str, size: &str) -> String {
if let Some(start) = url.find("/cache/") {
let after_cache = start + "/cache/".len();
if let Some(end) = url[after_cache..].find('/') {
let mut result = String::with_capacity(url.len());
result.push_str(&url[..after_cache]);
result.push_str(size);
result.push_str(&url[after_cache + end..]);
return result;
}
}
url.to_string()
}
/// Clear the screenshot cache directory to force re-download of screenshots.
/// Called during catalog sync to avoid stale cached images.
fn clear_screenshot_cache() {
let cache_dir = screenshot_cache_dir();
if cache_dir.exists() {
fs::remove_dir_all(&cache_dir).ok();
fs::create_dir_all(&cache_dir).ok();
}
}
/// Clear icon cache entries for OCS apps to force re-download with updated URLs.
fn clear_ocs_icon_cache(apps: &[CatalogApp]) {
let cache_dir = icon_cache_dir();
for app in apps {
let sanitized = sanitize_filename(&app.name);
let path = cache_dir.join(format!("{}.png", sanitized));
if path.exists() {
fs::remove_file(&path).ok();
}
}
}
/// Resolve an asset path to a full URL (handles relative paths from AppImageHub).
fn resolve_asset_url(path: &str) -> String {
if path.starts_with("http://") || path.starts_with("https://") {
@@ -492,6 +1138,7 @@ mod tests {
#[test]
fn test_catalog_type_roundtrip() {
assert_eq!(CatalogType::from_str("appimage-hub"), CatalogType::AppImageHub);
assert_eq!(CatalogType::from_str("ocs-appimagehub"), CatalogType::OcsAppImageHub);
assert_eq!(CatalogType::from_str("github-search"), CatalogType::GitHubSearch);
assert_eq!(CatalogType::from_str("custom"), CatalogType::Custom);
assert_eq!(CatalogType::from_str("unknown"), CatalogType::Custom);
@@ -500,6 +1147,7 @@ mod tests {
#[test]
fn test_catalog_type_as_str() {
assert_eq!(CatalogType::AppImageHub.as_str(), "appimage-hub");
assert_eq!(CatalogType::OcsAppImageHub.as_str(), "ocs-appimagehub");
assert_eq!(CatalogType::GitHubSearch.as_str(), "github-search");
assert_eq!(CatalogType::Custom.as_str(), "custom");
}
@@ -517,9 +1165,12 @@ mod tests {
let db = crate::core::database::Database::open_in_memory().unwrap();
ensure_default_sources(&db);
let sources = get_sources(&db);
assert_eq!(sources.len(), 1);
assert_eq!(sources[0].name, "AppImageHub");
assert_eq!(sources[0].source_type, CatalogType::AppImageHub);
assert_eq!(sources.len(), 2);
let ocs = sources.iter().find(|s| s.source_type == CatalogType::OcsAppImageHub);
assert!(ocs.is_some(), "OCS source should exist");
assert_eq!(ocs.unwrap().name, "AppImageHub.com");
let hub = sources.iter().find(|s| s.source_type == CatalogType::AppImageHub);
assert!(hub.is_some(), "AppImageHub source should exist");
}
#[test]

446
src/core/database.rs
View File

@@ -88,6 +88,33 @@ pub struct SystemModification {
pub previous_value: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum CatalogSortOrder {
NameAsc,
NameDesc,
StarsDesc,
StarsAsc,
DownloadsDesc,
DownloadsAsc,
ReleaseDateDesc,
ReleaseDateAsc,
}
impl CatalogSortOrder {
pub fn sql_clause(&self) -> &'static str {
match self {
Self::NameAsc => "ORDER BY name COLLATE NOCASE ASC",
Self::NameDesc => "ORDER BY name COLLATE NOCASE DESC",
Self::StarsDesc => "ORDER BY COALESCE(github_stars, 0) DESC, name COLLATE NOCASE ASC",
Self::StarsAsc => "ORDER BY CASE WHEN github_stars IS NULL THEN 1 ELSE 0 END, github_stars ASC, name COLLATE NOCASE ASC",
Self::DownloadsDesc => "ORDER BY (COALESCE(ocs_downloads, 0) + COALESCE(github_downloads, 0)) DESC, name COLLATE NOCASE ASC",
Self::DownloadsAsc => "ORDER BY CASE WHEN ocs_downloads IS NULL AND github_downloads IS NULL THEN 1 ELSE 0 END, (COALESCE(ocs_downloads, 0) + COALESCE(github_downloads, 0)) ASC, name COLLATE NOCASE ASC",
Self::ReleaseDateDesc => "ORDER BY COALESCE(release_date, '0000') DESC, name COLLATE NOCASE ASC",
Self::ReleaseDateAsc => "ORDER BY CASE WHEN release_date IS NULL THEN 1 ELSE 0 END, release_date ASC, name COLLATE NOCASE ASC",
}
}
}
#[derive(Debug, Clone)]
pub struct CatalogApp {
pub id: i64,
@@ -108,6 +135,27 @@ pub struct CatalogApp {
pub github_enriched_at: Option<String>,
pub github_download_url: Option<String>,
pub github_release_assets: Option<String>,
pub github_description: Option<String>,
pub github_readme: Option<String>,
// OCS (appimagehub.com) metadata
pub ocs_id: Option<i64>,
pub ocs_downloads: Option<i64>,
pub ocs_score: Option<i64>,
pub ocs_typename: Option<String>,
pub ocs_personid: Option<String>,
pub ocs_description: Option<String>,
pub ocs_summary: Option<String>,
pub ocs_version: Option<String>,
pub ocs_tags: Option<String>,
pub ocs_changed: Option<String>,
pub ocs_preview_url: Option<String>,
pub ocs_detailpage: Option<String>,
pub ocs_created: Option<String>,
pub ocs_downloadname: Option<String>,
pub ocs_downloadsize: Option<i64>,
pub ocs_arch: Option<String>,
pub ocs_md5sum: Option<String>,
pub ocs_comments: Option<i64>,
}
#[derive(Debug, Clone)]
@@ -426,6 +474,18 @@ impl Database {
self.migrate_to_v15()?;
}
if current_version < 16 {
self.migrate_to_v16()?;
}
if current_version < 17 {
self.migrate_to_v17()?;
}
if current_version < 18 {
self.migrate_to_v18()?;
}
// Ensure all expected columns exist (repairs DBs where a migration
// was updated after it had already run on this database)
self.ensure_columns()?;
@@ -930,6 +990,68 @@ impl Database {
Ok(())
}
fn migrate_to_v16(&self) -> SqlResult<()> {
let new_columns = [
"github_description TEXT",
"github_readme TEXT",
];
for col in &new_columns {
let sql = format!("ALTER TABLE catalog_apps ADD COLUMN {}", col);
self.conn.execute(&sql, []).ok();
}
self.conn.execute(
"UPDATE schema_version SET version = ?1",
params![16],
)?;
Ok(())
}
fn migrate_to_v17(&self) -> SqlResult<()> {
let new_columns = [
"ocs_id INTEGER",
"ocs_downloads INTEGER",
"ocs_score INTEGER",
"ocs_typename TEXT",
"ocs_personid TEXT",
"ocs_description TEXT",
"ocs_summary TEXT",
"ocs_version TEXT",
"ocs_tags TEXT",
"ocs_changed TEXT",
"ocs_preview_url TEXT",
];
for col in &new_columns {
let sql = format!("ALTER TABLE catalog_apps ADD COLUMN {}", col);
self.conn.execute(&sql, []).ok();
}
self.conn.execute(
"UPDATE schema_version SET version = ?1",
params![17],
)?;
Ok(())
}
fn migrate_to_v18(&self) -> SqlResult<()> {
let new_columns = [
"ocs_detailpage TEXT",
"ocs_created TEXT",
"ocs_downloadname TEXT",
"ocs_downloadsize INTEGER",
"ocs_arch TEXT",
"ocs_md5sum TEXT",
"ocs_comments INTEGER",
];
for col in &new_columns {
let sql = format!("ALTER TABLE catalog_apps ADD COLUMN {}", col);
self.conn.execute(&sql, []).ok();
}
self.conn.execute(
"UPDATE schema_version SET version = ?1",
params![18],
)?;
Ok(())
}
pub fn upsert_appimage(
&self,
path: &str,
@@ -1095,6 +1217,57 @@ impl Database {
previous_version_path, source_url, autostart, startup_wm_class,
verification_status, first_run_prompted, system_wide, is_portable, mount_point";
const CATALOG_APP_COLUMNS: &str =
"id, name, description, categories, download_url, icon_url, homepage, license, screenshots,
github_owner, github_repo, github_stars, github_downloads, latest_version, release_date,
github_enriched_at, github_download_url, github_release_assets, github_description, github_readme,
ocs_id, ocs_downloads, ocs_score, ocs_typename, ocs_personid, ocs_description, ocs_summary,
ocs_version, ocs_tags, ocs_changed, ocs_preview_url,
ocs_detailpage, ocs_created, ocs_downloadname, ocs_downloadsize, ocs_arch, ocs_md5sum, ocs_comments";
fn catalog_app_from_row(row: &rusqlite::Row) -> rusqlite::Result<CatalogApp> {
Ok(CatalogApp {
id: row.get(0)?,
name: row.get(1)?,
description: row.get(2)?,
categories: row.get(3)?,
download_url: row.get(4)?,
icon_url: row.get(5)?,
homepage: row.get(6)?,
license: row.get(7)?,
screenshots: row.get(8)?,
github_owner: row.get(9)?,
github_repo: row.get(10)?,
github_stars: row.get(11)?,
github_downloads: row.get(12)?,
latest_version: row.get(13)?,
release_date: row.get(14)?,
github_enriched_at: row.get(15)?,
github_download_url: row.get(16)?,
github_release_assets: row.get(17)?,
github_description: row.get(18)?,
github_readme: row.get(19)?,
ocs_id: row.get(20).unwrap_or(None),
ocs_downloads: row.get(21).unwrap_or(None),
ocs_score: row.get(22).unwrap_or(None),
ocs_typename: row.get(23).unwrap_or(None),
ocs_personid: row.get(24).unwrap_or(None),
ocs_description: row.get(25).unwrap_or(None),
ocs_summary: row.get(26).unwrap_or(None),
ocs_version: row.get(27).unwrap_or(None),
ocs_tags: row.get(28).unwrap_or(None),
ocs_changed: row.get(29).unwrap_or(None),
ocs_preview_url: row.get(30).unwrap_or(None),
ocs_detailpage: row.get(31).unwrap_or(None),
ocs_created: row.get(32).unwrap_or(None),
ocs_downloadname: row.get(33).unwrap_or(None),
ocs_downloadsize: row.get(34).unwrap_or(None),
ocs_arch: row.get(35).unwrap_or(None),
ocs_md5sum: row.get(36).unwrap_or(None),
ocs_comments: row.get(37).unwrap_or(None),
})
}
fn row_to_record(row: &rusqlite::Row) -> rusqlite::Result<AppImageRecord> {
Ok(AppImageRecord {
id: row.get(0)?,
@@ -2159,16 +2332,16 @@ impl Database {
query: &str,
category: Option<&str>,
limit: i32,
sort: CatalogSortOrder,
) -> SqlResult<Vec<CatalogApp>> {
let mut sql = String::from(
"SELECT id, name, description, categories, download_url, icon_url, homepage, license, screenshots,
github_owner, github_repo, github_stars, github_downloads, latest_version, release_date, github_enriched_at, github_download_url, github_release_assets
FROM catalog_apps WHERE 1=1"
let mut sql = format!(
"SELECT {} FROM catalog_apps WHERE 1=1",
Self::CATALOG_APP_COLUMNS
);
let mut params_list: Vec<Box<dyn rusqlite::types::ToSql>> = Vec::new();
if !query.is_empty() {
sql.push_str(" AND (name LIKE ?1 OR description LIKE ?1)");
sql.push_str(" AND (name LIKE ?1 OR description LIKE ?1 OR ocs_summary LIKE ?1 OR ocs_description LIKE ?1)");
params_list.push(Box::new(format!("%{}%", query)));
}
@@ -2178,34 +2351,13 @@ impl Database {
params_list.push(Box::new(format!("%{}%", cat)));
}
sql.push_str(&format!(" ORDER BY name LIMIT {}", limit));
sql.push_str(&format!(" {} LIMIT {}", sort.sql_clause(), limit));
let params_refs: Vec<&dyn rusqlite::types::ToSql> =
params_list.iter().map(|p| p.as_ref()).collect();
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt.query_map(params_refs.as_slice(), |row| {
Ok(CatalogApp {
id: row.get(0)?,
name: row.get(1)?,
description: row.get(2)?,
categories: row.get(3)?,
download_url: row.get(4)?,
icon_url: row.get(5)?,
homepage: row.get(6)?,
license: row.get(7)?,
screenshots: row.get(8)?,
github_owner: row.get(9)?,
github_repo: row.get(10)?,
github_stars: row.get(11)?,
github_downloads: row.get(12)?,
latest_version: row.get(13)?,
release_date: row.get(14)?,
github_enriched_at: row.get(15)?,
github_download_url: row.get(16)?,
github_release_assets: row.get(17)?,
})
})?;
let rows = stmt.query_map(params_refs.as_slice(), Self::catalog_app_from_row)?;
let mut results = Vec::new();
for row in rows {
@@ -2215,34 +2367,11 @@ impl Database {
}
pub fn get_catalog_app(&self, id: i64) -> SqlResult<Option<CatalogApp>> {
let result = self.conn.query_row(
"SELECT id, name, description, categories, download_url, icon_url, homepage, license, screenshots,
github_owner, github_repo, github_stars, github_downloads, latest_version, release_date, github_enriched_at, github_download_url, github_release_assets
FROM catalog_apps WHERE id = ?1",
params![id],
|row| {
Ok(CatalogApp {
id: row.get(0)?,
name: row.get(1)?,
description: row.get(2)?,
categories: row.get(3)?,
download_url: row.get(4)?,
icon_url: row.get(5)?,
homepage: row.get(6)?,
license: row.get(7)?,
screenshots: row.get(8)?,
github_owner: row.get(9)?,
github_repo: row.get(10)?,
github_stars: row.get(11)?,
github_downloads: row.get(12)?,
latest_version: row.get(13)?,
release_date: row.get(14)?,
github_enriched_at: row.get(15)?,
github_download_url: row.get(16)?,
github_release_assets: row.get(17)?,
})
},
let sql = format!(
"SELECT {} FROM catalog_apps WHERE id = ?1",
Self::CATALOG_APP_COLUMNS
);
let result = self.conn.query_row(&sql, params![id], Self::catalog_app_from_row);
match result {
Ok(app) => Ok(Some(app)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
@@ -2250,8 +2379,9 @@ impl Database {
}
}
/// Get featured catalog apps. Apps with GitHub stars sort first (by stars desc),
/// then unenriched apps get a deterministic shuffle that rotates every 15 minutes.
/// Get featured catalog apps. Apps with enrichment data (stars or OCS downloads)
/// sort first by combined popularity, then unenriched apps get a deterministic
/// shuffle that rotates every 15 minutes.
pub fn get_featured_catalog_apps(&self, limit: i32) -> SqlResult<Vec<CatalogApp>> {
// Time seed rotates every 15 minutes (900 seconds)
let time_seed = std::time::SystemTime::now()
@@ -2259,46 +2389,31 @@ impl Database {
.unwrap_or_default()
.as_secs() / 900;
let mut stmt = self.conn.prepare(
"SELECT id, name, description, categories, download_url, icon_url, homepage, license, screenshots,
github_owner, github_repo, github_stars, github_downloads, latest_version, release_date, github_enriched_at, github_download_url, github_release_assets
FROM catalog_apps
let sql = format!(
"SELECT {} FROM catalog_apps
WHERE icon_url IS NOT NULL AND icon_url != ''
AND description IS NOT NULL AND description != ''
AND screenshots IS NOT NULL AND screenshots != ''"
)?;
let rows = stmt.query_map([], |row| {
Ok(CatalogApp {
id: row.get(0)?,
name: row.get(1)?,
description: row.get(2)?,
categories: row.get(3)?,
download_url: row.get(4)?,
icon_url: row.get(5)?,
homepage: row.get(6)?,
license: row.get(7)?,
screenshots: row.get(8)?,
github_owner: row.get(9)?,
github_repo: row.get(10)?,
github_stars: row.get(11)?,
github_downloads: row.get(12)?,
latest_version: row.get(13)?,
release_date: row.get(14)?,
github_enriched_at: row.get(15)?,
github_download_url: row.get(16)?,
github_release_assets: row.get(17)?,
})
})?;
AND (description IS NOT NULL AND description != ''
OR ocs_summary IS NOT NULL AND ocs_summary != '')
AND (screenshots IS NOT NULL AND screenshots != ''
OR ocs_preview_url IS NOT NULL AND ocs_preview_url != '')",
Self::CATALOG_APP_COLUMNS
);
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt.query_map([], Self::catalog_app_from_row)?;
let mut apps: Vec<CatalogApp> = rows.collect::<SqlResult<Vec<_>>>()?;
// Enriched apps (with stars) sort first by stars descending,
// unenriched apps get the deterministic shuffle after them
// Sort by combined popularity: OCS downloads + GitHub stars.
// Apps with any enrichment sort first, then deterministic shuffle.
apps.sort_by(|a, b| {
match (a.github_stars, b.github_stars) {
(Some(sa), Some(sb)) => sb.cmp(&sa),
(Some(_), None) => std::cmp::Ordering::Less,
(None, Some(_)) => std::cmp::Ordering::Greater,
(None, None) => {
let a_pop = a.ocs_downloads.unwrap_or(0) + a.github_stars.unwrap_or(0);
let b_pop = b.ocs_downloads.unwrap_or(0) + b.github_stars.unwrap_or(0);
let a_enriched = a_pop > 0;
let b_enriched = b_pop > 0;
match (a_enriched, b_enriched) {
(true, false) => std::cmp::Ordering::Less,
(false, true) => std::cmp::Ordering::Greater,
(true, true) => b_pop.cmp(&a_pop),
(false, false) => {
let ha = (a.id as u64 ^ time_seed).wrapping_mul(0x517cc1b727220a95);
let hb = (b.id as u64 ^ time_seed).wrapping_mul(0x517cc1b727220a95);
ha.cmp(&hb)
@@ -2369,6 +2484,97 @@ impl Database {
Ok(())
}
pub fn insert_ocs_catalog_app(
&self,
source_id: i64,
name: &str,
ocs_id: i64,
description: Option<&str>,
summary: Option<&str>,
categories: Option<&str>,
version: Option<&str>,
download_url: &str,
icon_url: Option<&str>,
homepage: Option<&str>,
screenshots: Option<&str>,
ocs_downloads: Option<i64>,
ocs_score: Option<i64>,
ocs_typename: Option<&str>,
ocs_personid: Option<&str>,
ocs_tags: Option<&str>,
ocs_changed: Option<&str>,
ocs_preview_url: Option<&str>,
license: Option<&str>,
ocs_detailpage: Option<&str>,
ocs_created: Option<&str>,
ocs_downloadname: Option<&str>,
ocs_downloadsize: Option<i64>,
ocs_arch: Option<&str>,
ocs_md5sum: Option<&str>,
ocs_comments: Option<i64>,
) -> SqlResult<()> {
self.conn.execute(
"INSERT INTO catalog_apps
(source_id, name, description, categories, latest_version, download_url, icon_url, homepage,
screenshots, license, cached_at,
ocs_id, ocs_downloads, ocs_score, ocs_typename, ocs_personid, ocs_description, ocs_summary,
ocs_version, ocs_tags, ocs_changed, ocs_preview_url,
ocs_detailpage, ocs_created, ocs_downloadname, ocs_downloadsize, ocs_arch, ocs_md5sum, ocs_comments)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, datetime('now'),
?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21,
?22, ?23, ?24, ?25, ?26, ?27, ?28)
ON CONFLICT(source_id, name) DO UPDATE SET
description = excluded.description,
categories = excluded.categories,
latest_version = excluded.latest_version,
download_url = excluded.download_url,
icon_url = excluded.icon_url,
homepage = excluded.homepage,
screenshots = excluded.screenshots,
license = excluded.license,
cached_at = datetime('now'),
ocs_id = excluded.ocs_id,
ocs_downloads = excluded.ocs_downloads,
ocs_score = excluded.ocs_score,
ocs_typename = excluded.ocs_typename,
ocs_personid = excluded.ocs_personid,
ocs_description = excluded.ocs_description,
ocs_summary = excluded.ocs_summary,
ocs_version = excluded.ocs_version,
ocs_tags = excluded.ocs_tags,
ocs_changed = excluded.ocs_changed,
ocs_preview_url = excluded.ocs_preview_url,
ocs_detailpage = excluded.ocs_detailpage,
ocs_created = excluded.ocs_created,
ocs_downloadname = excluded.ocs_downloadname,
ocs_downloadsize = excluded.ocs_downloadsize,
ocs_arch = excluded.ocs_arch,
ocs_md5sum = excluded.ocs_md5sum,
ocs_comments = excluded.ocs_comments",
params![
source_id, name, summary, categories, version, download_url, icon_url, homepage,
screenshots, license,
ocs_id, ocs_downloads, ocs_score, ocs_typename, ocs_personid, description, summary,
version, ocs_tags, ocs_changed, ocs_preview_url,
ocs_detailpage, ocs_created, ocs_downloadname, ocs_downloadsize, ocs_arch, ocs_md5sum, ocs_comments
],
)?;
Ok(())
}
/// Get all app names for a given source (used for deduplication).
pub fn get_catalog_app_names_for_source(&self, source_id: i64) -> SqlResult<std::collections::HashSet<String>> {
let mut stmt = self.conn.prepare(
"SELECT LOWER(name) FROM catalog_apps WHERE source_id = ?1"
)?;
let rows = stmt.query_map(params![source_id], |row| row.get::<_, String>(0))?;
let mut names = std::collections::HashSet::new();
for row in rows {
names.insert(row?);
}
Ok(names)
}
pub fn search_catalog_apps(&self, query: &str) -> SqlResult<Vec<CatalogAppRecord>> {
let pattern = format!("%{}%", query);
let mut stmt = self.conn.prepare(
@@ -2453,10 +2659,11 @@ impl Database {
app_id: i64,
stars: i64,
pushed_at: Option<&str>,
description: Option<&str>,
) -> SqlResult<()> {
self.conn.execute(
"UPDATE catalog_apps SET github_stars = ?2, github_enriched_at = datetime('now') WHERE id = ?1",
params![app_id, stars],
"UPDATE catalog_apps SET github_stars = ?2, github_description = COALESCE(?3, github_description), github_enriched_at = datetime('now') WHERE id = ?1",
params![app_id, stars, description],
)?;
// Store pushed_at in release_date if no release info yet
if let Some(pushed) = pushed_at {
@@ -2492,36 +2699,15 @@ impl Database {
}
pub fn get_unenriched_catalog_apps(&self, limit: i32) -> SqlResult<Vec<CatalogApp>> {
let mut stmt = self.conn.prepare(
"SELECT id, name, description, categories, download_url, icon_url, homepage, license, screenshots,
github_owner, github_repo, github_stars, github_downloads, latest_version, release_date, github_enriched_at, github_download_url, github_release_assets
FROM catalog_apps
let sql = format!(
"SELECT {} FROM catalog_apps
WHERE github_owner IS NOT NULL AND github_enriched_at IS NULL
ORDER BY id
LIMIT ?1"
)?;
let rows = stmt.query_map(params![limit], |row| {
Ok(CatalogApp {
id: row.get(0)?,
name: row.get(1)?,
description: row.get(2)?,
categories: row.get(3)?,
download_url: row.get(4)?,
icon_url: row.get(5)?,
homepage: row.get(6)?,
license: row.get(7)?,
screenshots: row.get(8)?,
github_owner: row.get(9)?,
github_repo: row.get(10)?,
github_stars: row.get(11)?,
github_downloads: row.get(12)?,
latest_version: row.get(13)?,
release_date: row.get(14)?,
github_enriched_at: row.get(15)?,
github_download_url: row.get(16)?,
github_release_assets: row.get(17)?,
})
})?;
LIMIT ?1",
Self::CATALOG_APP_COLUMNS
);
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt.query_map(params![limit], Self::catalog_app_from_row)?;
rows.collect()
}
@@ -2538,6 +2724,18 @@ impl Database {
)?;
Ok((enriched, total_with_github))
}
pub fn update_catalog_app_readme(
&self,
app_id: i64,
readme: &str,
) -> SqlResult<()> {
self.conn.execute(
"UPDATE catalog_apps SET github_readme = ?2 WHERE id = ?1",
params![app_id, readme],
)?;
Ok(())
}
}
#[cfg(test)]
@@ -2708,7 +2906,7 @@ mod tests {
[],
|row| row.get(0),
).unwrap();
assert_eq!(version, 15);
assert_eq!(version, 18);
// All tables that should exist after the full v1-v7 migration chain
let expected_tables = [

71
src/core/github_enrichment.rs
View File

@@ -110,6 +110,54 @@ pub fn fetch_release_info(owner: &str, repo: &str, token: &str) -> Result<(GitHu
Ok((info, remaining))
}
#[derive(Debug, serde::Deserialize)]
struct GitHubReadmeResponse {
content: String,
#[serde(default)]
encoding: String,
}
/// Fetch the README content for a repo (decoded from base64).
pub fn fetch_readme(owner: &str, repo: &str, token: &str) -> Result<(String, u32), String> {
let url = format!("https://api.github.com/repos/{}/{}/readme", owner, repo);
let (body, remaining) = github_get(&url, token)?;
let resp: GitHubReadmeResponse = serde_json::from_str(&body)
.map_err(|e| format!("Parse error: {}", e))?;
if resp.encoding != "base64" {
return Err(format!("Unexpected encoding: {}", resp.encoding));
}
// GitHub returns base64 with newlines; strip them before decoding
let clean = resp.content.replace('\n', "");
let decoded = base64_decode(&clean)
.map_err(|e| format!("Base64 decode error: {}", e))?;
let text = String::from_utf8(decoded)
.map_err(|e| format!("UTF-8 error: {}", e))?;
Ok((text, remaining))
}
/// Simple base64 decoder (standard alphabet, no padding required).
fn base64_decode(input: &str) -> Result<Vec<u8>, String> {
const TABLE: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
let mut output = Vec::with_capacity(input.len() * 3 / 4);
let mut buf = 0u32;
let mut bits = 0u32;
for &b in input.as_bytes() {
if b == b'=' { break; }
let val = TABLE.iter().position(|&c| c == b)
.ok_or_else(|| format!("Invalid base64 char: {}", b as char))? as u32;
buf = (buf << 6) | val;
bits += 6;
if bits >= 8 {
bits -= 8;
output.push((buf >> bits) as u8);
buf &= (1 << bits) - 1;
}
}
Ok(output)
}
// --- AppImage asset filtering ---
/// A simplified release asset for storage (JSON-serializable).
@@ -163,7 +211,7 @@ pub fn pick_best_asset(assets: &[AppImageAsset]) -> Option<&AppImageAsset> {
// --- Enrichment logic ---
/// Enrich a catalog app with repo-level info (stars, pushed_at).
/// Enrich a catalog app with repo-level info (stars, pushed_at, description).
pub fn enrich_app_repo_info(
db: &Database,
app_id: i64,
@@ -172,8 +220,9 @@ pub fn enrich_app_repo_info(
token: &str,
) -> Result<u32, String> {
let (info, remaining) = fetch_repo_info(owner, repo, token)?;
db.update_catalog_app_github_metadata(app_id, info.stargazers_count, info.pushed_at.as_deref())
.map_err(|e| format!("DB error: {}", e))?;
db.update_catalog_app_github_metadata(
app_id, info.stargazers_count, info.pushed_at.as_deref(), info.description.as_deref(),
).map_err(|e| format!("DB error: {}", e))?;
Ok(remaining)
}
@@ -216,6 +265,20 @@ pub fn enrich_app_release_info(
Ok(remaining)
}
/// Fetch and store the README for a catalog app.
pub fn enrich_app_readme(
db: &Database,
app_id: i64,
owner: &str,
repo: &str,
token: &str,
) -> Result<u32, String> {
let (readme, remaining) = fetch_readme(owner, repo, token)?;
db.update_catalog_app_readme(app_id, &readme)
.map_err(|e| format!("DB error: {}", e))?;
Ok(remaining)
}
/// Background enrichment: process a batch of unenriched apps.
/// Returns (count_enriched, should_continue).
pub fn background_enrich_batch(
@@ -261,7 +324,7 @@ pub fn background_enrich_batch(
Err(e) => {
log::warn!("Failed to enrich {}/{}: {}", owner, repo, e);
// Mark as enriched anyway so we don't retry forever
db.update_catalog_app_github_metadata(app.id, 0, None).ok();
db.update_catalog_app_github_metadata(app.id, 0, None, None).ok();
}
}