lashman/driftwood
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Add AppImageHub.com OCS API as primary catalog source

Browse Source 
Integrate the Pling/OCS REST API (appimagehub.com) as the primary catalog
source with richer metadata than the existing appimage.github.io feed.

Backend:
- Add OCS API fetch with pagination, lenient JSON deserializers for loosely
  typed numeric fields, and non-AppImage file filtering (.dmg, .exe, etc.)
- Database migration v17 adds OCS-specific columns (ocs_id, downloads, score,
  typename, personid, description, summary, version, tags, etc.)
- Deduplicate secondary source apps against OCS entries
- Shrink OCS CDN icon URLs from 770x540 to 100x100 for faster loading
- Clear stale screenshot and icon caches on sync
- Extract GitHub repo links from OCS HTML descriptions
- Add fetch_ocs_download_files() to get all version files for an app
- Resolve fresh JWT download URLs per slot at install time

Detail page:
- Fetch OCS download files on page open and populate install SplitButton
  with version dropdown (newest first, filtered for AppImage only)
- Show OCS metadata: downloads, score, author, typename, tags, comments,
  created/updated dates, architecture, filename, file size, MD5
- Prefer ocs_description (full HTML with features/changelog) over short
  summary for the About section
- Add html_to_description() to preserve formatting (lists, paragraphs)
- Remove redundant Download link from Links section
- Escape ampersands in Pango markup subtitles (categories, typename, tags)

Catalog view:
- OCS source syncs first as primary, appimage.github.io as secondary
- Featured apps consider OCS download counts alongside GitHub stars

UI:
- Add pulldown-cmark for GitHub README markdown rendering in detail pages
- Add build_markdown_view() widget for rendered markdown content
This commit is contained in:
lashman
2026-02-28 20:33:40 +02:00
parent f89aafca6a
commit 4b939f044a
16 changed files with 2394 additions and 417 deletions
Download Patch File Download Diff File Expand all files Collapse all files

446
src/core/database.rs
View File

@@ -88,6 +88,33 @@ pub struct SystemModification {
pub previous_value: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum CatalogSortOrder {
NameAsc,
NameDesc,
StarsDesc,
StarsAsc,
DownloadsDesc,
DownloadsAsc,
ReleaseDateDesc,
ReleaseDateAsc,
}
impl CatalogSortOrder {
pub fn sql_clause(&self) -> &'static str {
match self {
Self::NameAsc => "ORDER BY name COLLATE NOCASE ASC",
Self::NameDesc => "ORDER BY name COLLATE NOCASE DESC",
Self::StarsDesc => "ORDER BY COALESCE(github_stars, 0) DESC, name COLLATE NOCASE ASC",
Self::StarsAsc => "ORDER BY CASE WHEN github_stars IS NULL THEN 1 ELSE 0 END, github_stars ASC, name COLLATE NOCASE ASC",
Self::DownloadsDesc => "ORDER BY (COALESCE(ocs_downloads, 0) + COALESCE(github_downloads, 0)) DESC, name COLLATE NOCASE ASC",
Self::DownloadsAsc => "ORDER BY CASE WHEN ocs_downloads IS NULL AND github_downloads IS NULL THEN 1 ELSE 0 END, (COALESCE(ocs_downloads, 0) + COALESCE(github_downloads, 0)) ASC, name COLLATE NOCASE ASC",
Self::ReleaseDateDesc => "ORDER BY COALESCE(release_date, '0000') DESC, name COLLATE NOCASE ASC",
Self::ReleaseDateAsc => "ORDER BY CASE WHEN release_date IS NULL THEN 1 ELSE 0 END, release_date ASC, name COLLATE NOCASE ASC",
}
}
}
#[derive(Debug, Clone)]
pub struct CatalogApp {
pub id: i64,
@@ -108,6 +135,27 @@ pub struct CatalogApp {
pub github_enriched_at: Option<String>,
pub github_download_url: Option<String>,
pub github_release_assets: Option<String>,
pub github_description: Option<String>,
pub github_readme: Option<String>,
// OCS (appimagehub.com) metadata
pub ocs_id: Option<i64>,
pub ocs_downloads: Option<i64>,
pub ocs_score: Option<i64>,
pub ocs_typename: Option<String>,
pub ocs_personid: Option<String>,
pub ocs_description: Option<String>,
pub ocs_summary: Option<String>,
pub ocs_version: Option<String>,
pub ocs_tags: Option<String>,
pub ocs_changed: Option<String>,
pub ocs_preview_url: Option<String>,
pub ocs_detailpage: Option<String>,
pub ocs_created: Option<String>,
pub ocs_downloadname: Option<String>,
pub ocs_downloadsize: Option<i64>,
pub ocs_arch: Option<String>,
pub ocs_md5sum: Option<String>,
pub ocs_comments: Option<i64>,
}
#[derive(Debug, Clone)]
@@ -426,6 +474,18 @@ impl Database {
self.migrate_to_v15()?;
}
if current_version < 16 {
self.migrate_to_v16()?;
}
if current_version < 17 {
self.migrate_to_v17()?;
}
if current_version < 18 {
self.migrate_to_v18()?;
}
// Ensure all expected columns exist (repairs DBs where a migration
// was updated after it had already run on this database)
self.ensure_columns()?;
@@ -930,6 +990,68 @@ impl Database {
Ok(())
}
fn migrate_to_v16(&self) -> SqlResult<()> {
let new_columns = [
"github_description TEXT",
"github_readme TEXT",
];
for col in &new_columns {
let sql = format!("ALTER TABLE catalog_apps ADD COLUMN {}", col);
self.conn.execute(&sql, []).ok();
}
self.conn.execute(
"UPDATE schema_version SET version = ?1",
params![16],
)?;
Ok(())
}
fn migrate_to_v17(&self) -> SqlResult<()> {
let new_columns = [
"ocs_id INTEGER",
"ocs_downloads INTEGER",
"ocs_score INTEGER",
"ocs_typename TEXT",
"ocs_personid TEXT",
"ocs_description TEXT",
"ocs_summary TEXT",
"ocs_version TEXT",
"ocs_tags TEXT",
"ocs_changed TEXT",
"ocs_preview_url TEXT",
];
for col in &new_columns {
let sql = format!("ALTER TABLE catalog_apps ADD COLUMN {}", col);
self.conn.execute(&sql, []).ok();
}
self.conn.execute(
"UPDATE schema_version SET version = ?1",
params![17],
)?;
Ok(())
}
fn migrate_to_v18(&self) -> SqlResult<()> {
let new_columns = [
"ocs_detailpage TEXT",
"ocs_created TEXT",
"ocs_downloadname TEXT",
"ocs_downloadsize INTEGER",
"ocs_arch TEXT",
"ocs_md5sum TEXT",
"ocs_comments INTEGER",
];
for col in &new_columns {
let sql = format!("ALTER TABLE catalog_apps ADD COLUMN {}", col);
self.conn.execute(&sql, []).ok();
}
self.conn.execute(
"UPDATE schema_version SET version = ?1",
params![18],
)?;
Ok(())
}
pub fn upsert_appimage(
&self,
path: &str,
@@ -1095,6 +1217,57 @@ impl Database {
previous_version_path, source_url, autostart, startup_wm_class,
verification_status, first_run_prompted, system_wide, is_portable, mount_point";
const CATALOG_APP_COLUMNS: &str =
"id, name, description, categories, download_url, icon_url, homepage, license, screenshots,
github_owner, github_repo, github_stars, github_downloads, latest_version, release_date,
github_enriched_at, github_download_url, github_release_assets, github_description, github_readme,
ocs_id, ocs_downloads, ocs_score, ocs_typename, ocs_personid, ocs_description, ocs_summary,
ocs_version, ocs_tags, ocs_changed, ocs_preview_url,
ocs_detailpage, ocs_created, ocs_downloadname, ocs_downloadsize, ocs_arch, ocs_md5sum, ocs_comments";
fn catalog_app_from_row(row: &rusqlite::Row) -> rusqlite::Result<CatalogApp> {
Ok(CatalogApp {
id: row.get(0)?,
name: row.get(1)?,
description: row.get(2)?,
categories: row.get(3)?,
download_url: row.get(4)?,
icon_url: row.get(5)?,
homepage: row.get(6)?,
license: row.get(7)?,
screenshots: row.get(8)?,
github_owner: row.get(9)?,
github_repo: row.get(10)?,
github_stars: row.get(11)?,
github_downloads: row.get(12)?,
latest_version: row.get(13)?,
release_date: row.get(14)?,
github_enriched_at: row.get(15)?,
github_download_url: row.get(16)?,
github_release_assets: row.get(17)?,
github_description: row.get(18)?,
github_readme: row.get(19)?,
ocs_id: row.get(20).unwrap_or(None),
ocs_downloads: row.get(21).unwrap_or(None),
ocs_score: row.get(22).unwrap_or(None),
ocs_typename: row.get(23).unwrap_or(None),
ocs_personid: row.get(24).unwrap_or(None),
ocs_description: row.get(25).unwrap_or(None),
ocs_summary: row.get(26).unwrap_or(None),
ocs_version: row.get(27).unwrap_or(None),
ocs_tags: row.get(28).unwrap_or(None),
ocs_changed: row.get(29).unwrap_or(None),
ocs_preview_url: row.get(30).unwrap_or(None),
ocs_detailpage: row.get(31).unwrap_or(None),
ocs_created: row.get(32).unwrap_or(None),
ocs_downloadname: row.get(33).unwrap_or(None),
ocs_downloadsize: row.get(34).unwrap_or(None),
ocs_arch: row.get(35).unwrap_or(None),
ocs_md5sum: row.get(36).unwrap_or(None),
ocs_comments: row.get(37).unwrap_or(None),
})
}
fn row_to_record(row: &rusqlite::Row) -> rusqlite::Result<AppImageRecord> {
Ok(AppImageRecord {
id: row.get(0)?,
@@ -2159,16 +2332,16 @@ impl Database {
query: &str,
category: Option<&str>,
limit: i32,
sort: CatalogSortOrder,
) -> SqlResult<Vec<CatalogApp>> {
let mut sql = String::from(
"SELECT id, name, description, categories, download_url, icon_url, homepage, license, screenshots,
github_owner, github_repo, github_stars, github_downloads, latest_version, release_date, github_enriched_at, github_download_url, github_release_assets
FROM catalog_apps WHERE 1=1"
let mut sql = format!(
"SELECT {} FROM catalog_apps WHERE 1=1",
Self::CATALOG_APP_COLUMNS
);
let mut params_list: Vec<Box<dyn rusqlite::types::ToSql>> = Vec::new();
if !query.is_empty() {
sql.push_str(" AND (name LIKE ?1 OR description LIKE ?1)");
sql.push_str(" AND (name LIKE ?1 OR description LIKE ?1 OR ocs_summary LIKE ?1 OR ocs_description LIKE ?1)");
params_list.push(Box::new(format!("%{}%", query)));
}
@@ -2178,34 +2351,13 @@ impl Database {
params_list.push(Box::new(format!("%{}%", cat)));
}
sql.push_str(&format!(" ORDER BY name LIMIT {}", limit));
sql.push_str(&format!(" {} LIMIT {}", sort.sql_clause(), limit));
let params_refs: Vec<&dyn rusqlite::types::ToSql> =
params_list.iter().map(|p| p.as_ref()).collect();
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt.query_map(params_refs.as_slice(), |row| {
Ok(CatalogApp {
id: row.get(0)?,
name: row.get(1)?,
description: row.get(2)?,
categories: row.get(3)?,
download_url: row.get(4)?,
icon_url: row.get(5)?,
homepage: row.get(6)?,
license: row.get(7)?,
screenshots: row.get(8)?,
github_owner: row.get(9)?,
github_repo: row.get(10)?,
github_stars: row.get(11)?,
github_downloads: row.get(12)?,
latest_version: row.get(13)?,
release_date: row.get(14)?,
github_enriched_at: row.get(15)?,
github_download_url: row.get(16)?,
github_release_assets: row.get(17)?,
})
})?;
let rows = stmt.query_map(params_refs.as_slice(), Self::catalog_app_from_row)?;
let mut results = Vec::new();
for row in rows {
@@ -2215,34 +2367,11 @@ impl Database {
}
pub fn get_catalog_app(&self, id: i64) -> SqlResult<Option<CatalogApp>> {
let result = self.conn.query_row(
"SELECT id, name, description, categories, download_url, icon_url, homepage, license, screenshots,
github_owner, github_repo, github_stars, github_downloads, latest_version, release_date, github_enriched_at, github_download_url, github_release_assets
FROM catalog_apps WHERE id = ?1",
params![id],
|row| {
Ok(CatalogApp {
id: row.get(0)?,
name: row.get(1)?,
description: row.get(2)?,
categories: row.get(3)?,
download_url: row.get(4)?,
icon_url: row.get(5)?,
homepage: row.get(6)?,
license: row.get(7)?,
screenshots: row.get(8)?,
github_owner: row.get(9)?,
github_repo: row.get(10)?,
github_stars: row.get(11)?,
github_downloads: row.get(12)?,
latest_version: row.get(13)?,
release_date: row.get(14)?,
github_enriched_at: row.get(15)?,
github_download_url: row.get(16)?,
github_release_assets: row.get(17)?,
})
},
let sql = format!(
"SELECT {} FROM catalog_apps WHERE id = ?1",
Self::CATALOG_APP_COLUMNS
);
let result = self.conn.query_row(&sql, params![id], Self::catalog_app_from_row);
match result {
Ok(app) => Ok(Some(app)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
@@ -2250,8 +2379,9 @@ impl Database {
}
}
/// Get featured catalog apps. Apps with GitHub stars sort first (by stars desc),
/// then unenriched apps get a deterministic shuffle that rotates every 15 minutes.
/// Get featured catalog apps. Apps with enrichment data (stars or OCS downloads)
/// sort first by combined popularity, then unenriched apps get a deterministic
/// shuffle that rotates every 15 minutes.
pub fn get_featured_catalog_apps(&self, limit: i32) -> SqlResult<Vec<CatalogApp>> {
// Time seed rotates every 15 minutes (900 seconds)
let time_seed = std::time::SystemTime::now()
@@ -2259,46 +2389,31 @@ impl Database {
.unwrap_or_default()
.as_secs() / 900;
let mut stmt = self.conn.prepare(
"SELECT id, name, description, categories, download_url, icon_url, homepage, license, screenshots,
github_owner, github_repo, github_stars, github_downloads, latest_version, release_date, github_enriched_at, github_download_url, github_release_assets
FROM catalog_apps
let sql = format!(
"SELECT {} FROM catalog_apps
WHERE icon_url IS NOT NULL AND icon_url != ''
AND description IS NOT NULL AND description != ''
AND screenshots IS NOT NULL AND screenshots != ''"
)?;
let rows = stmt.query_map([], |row| {
Ok(CatalogApp {
id: row.get(0)?,
name: row.get(1)?,
description: row.get(2)?,
categories: row.get(3)?,
download_url: row.get(4)?,
icon_url: row.get(5)?,
homepage: row.get(6)?,
license: row.get(7)?,
screenshots: row.get(8)?,
github_owner: row.get(9)?,
github_repo: row.get(10)?,
github_stars: row.get(11)?,
github_downloads: row.get(12)?,
latest_version: row.get(13)?,
release_date: row.get(14)?,
github_enriched_at: row.get(15)?,
github_download_url: row.get(16)?,
github_release_assets: row.get(17)?,
})
})?;
AND (description IS NOT NULL AND description != ''
OR ocs_summary IS NOT NULL AND ocs_summary != '')
AND (screenshots IS NOT NULL AND screenshots != ''
OR ocs_preview_url IS NOT NULL AND ocs_preview_url != '')",
Self::CATALOG_APP_COLUMNS
);
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt.query_map([], Self::catalog_app_from_row)?;
let mut apps: Vec<CatalogApp> = rows.collect::<SqlResult<Vec<_>>>()?;
// Enriched apps (with stars) sort first by stars descending,
// unenriched apps get the deterministic shuffle after them
// Sort by combined popularity: OCS downloads + GitHub stars.
// Apps with any enrichment sort first, then deterministic shuffle.
apps.sort_by(|a, b| {
match (a.github_stars, b.github_stars) {
(Some(sa), Some(sb)) => sb.cmp(&sa),
(Some(_), None) => std::cmp::Ordering::Less,
(None, Some(_)) => std::cmp::Ordering::Greater,
(None, None) => {
let a_pop = a.ocs_downloads.unwrap_or(0) + a.github_stars.unwrap_or(0);
let b_pop = b.ocs_downloads.unwrap_or(0) + b.github_stars.unwrap_or(0);
let a_enriched = a_pop > 0;
let b_enriched = b_pop > 0;
match (a_enriched, b_enriched) {
(true, false) => std::cmp::Ordering::Less,
(false, true) => std::cmp::Ordering::Greater,
(true, true) => b_pop.cmp(&a_pop),
(false, false) => {
let ha = (a.id as u64 ^ time_seed).wrapping_mul(0x517cc1b727220a95);
let hb = (b.id as u64 ^ time_seed).wrapping_mul(0x517cc1b727220a95);
ha.cmp(&hb)
@@ -2369,6 +2484,97 @@ impl Database {
Ok(())
}
pub fn insert_ocs_catalog_app(
&self,
source_id: i64,
name: &str,
ocs_id: i64,
description: Option<&str>,
summary: Option<&str>,
categories: Option<&str>,
version: Option<&str>,
download_url: &str,
icon_url: Option<&str>,
homepage: Option<&str>,
screenshots: Option<&str>,
ocs_downloads: Option<i64>,
ocs_score: Option<i64>,
ocs_typename: Option<&str>,
ocs_personid: Option<&str>,
ocs_tags: Option<&str>,
ocs_changed: Option<&str>,
ocs_preview_url: Option<&str>,
license: Option<&str>,
ocs_detailpage: Option<&str>,
ocs_created: Option<&str>,
ocs_downloadname: Option<&str>,
ocs_downloadsize: Option<i64>,
ocs_arch: Option<&str>,
ocs_md5sum: Option<&str>,
ocs_comments: Option<i64>,
) -> SqlResult<()> {
self.conn.execute(
"INSERT INTO catalog_apps
(source_id, name, description, categories, latest_version, download_url, icon_url, homepage,
screenshots, license, cached_at,
ocs_id, ocs_downloads, ocs_score, ocs_typename, ocs_personid, ocs_description, ocs_summary,
ocs_version, ocs_tags, ocs_changed, ocs_preview_url,
ocs_detailpage, ocs_created, ocs_downloadname, ocs_downloadsize, ocs_arch, ocs_md5sum, ocs_comments)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, datetime('now'),
?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21,
?22, ?23, ?24, ?25, ?26, ?27, ?28)
ON CONFLICT(source_id, name) DO UPDATE SET
description = excluded.description,
categories = excluded.categories,
latest_version = excluded.latest_version,
download_url = excluded.download_url,
icon_url = excluded.icon_url,
homepage = excluded.homepage,
screenshots = excluded.screenshots,
license = excluded.license,
cached_at = datetime('now'),
ocs_id = excluded.ocs_id,
ocs_downloads = excluded.ocs_downloads,
ocs_score = excluded.ocs_score,
ocs_typename = excluded.ocs_typename,
ocs_personid = excluded.ocs_personid,
ocs_description = excluded.ocs_description,
ocs_summary = excluded.ocs_summary,
ocs_version = excluded.ocs_version,
ocs_tags = excluded.ocs_tags,
ocs_changed = excluded.ocs_changed,
ocs_preview_url = excluded.ocs_preview_url,
ocs_detailpage = excluded.ocs_detailpage,
ocs_created = excluded.ocs_created,
ocs_downloadname = excluded.ocs_downloadname,
ocs_downloadsize = excluded.ocs_downloadsize,
ocs_arch = excluded.ocs_arch,
ocs_md5sum = excluded.ocs_md5sum,
ocs_comments = excluded.ocs_comments",
params![
source_id, name, summary, categories, version, download_url, icon_url, homepage,
screenshots, license,
ocs_id, ocs_downloads, ocs_score, ocs_typename, ocs_personid, description, summary,
version, ocs_tags, ocs_changed, ocs_preview_url,
ocs_detailpage, ocs_created, ocs_downloadname, ocs_downloadsize, ocs_arch, ocs_md5sum, ocs_comments
],
)?;
Ok(())
}
/// Get all app names for a given source (used for deduplication).
pub fn get_catalog_app_names_for_source(&self, source_id: i64) -> SqlResult<std::collections::HashSet<String>> {
let mut stmt = self.conn.prepare(
"SELECT LOWER(name) FROM catalog_apps WHERE source_id = ?1"
)?;
let rows = stmt.query_map(params![source_id], |row| row.get::<_, String>(0))?;
let mut names = std::collections::HashSet::new();
for row in rows {
names.insert(row?);
}
Ok(names)
}
pub fn search_catalog_apps(&self, query: &str) -> SqlResult<Vec<CatalogAppRecord>> {
let pattern = format!("%{}%", query);
let mut stmt = self.conn.prepare(
@@ -2453,10 +2659,11 @@ impl Database {
app_id: i64,
stars: i64,
pushed_at: Option<&str>,
description: Option<&str>,
) -> SqlResult<()> {
self.conn.execute(
"UPDATE catalog_apps SET github_stars = ?2, github_enriched_at = datetime('now') WHERE id = ?1",
params![app_id, stars],
"UPDATE catalog_apps SET github_stars = ?2, github_description = COALESCE(?3, github_description), github_enriched_at = datetime('now') WHERE id = ?1",
params![app_id, stars, description],
)?;
// Store pushed_at in release_date if no release info yet
if let Some(pushed) = pushed_at {
@@ -2492,36 +2699,15 @@ impl Database {
}
pub fn get_unenriched_catalog_apps(&self, limit: i32) -> SqlResult<Vec<CatalogApp>> {
let mut stmt = self.conn.prepare(
"SELECT id, name, description, categories, download_url, icon_url, homepage, license, screenshots,
github_owner, github_repo, github_stars, github_downloads, latest_version, release_date, github_enriched_at, github_download_url, github_release_assets
FROM catalog_apps
let sql = format!(
"SELECT {} FROM catalog_apps
WHERE github_owner IS NOT NULL AND github_enriched_at IS NULL
ORDER BY id
LIMIT ?1"
)?;
let rows = stmt.query_map(params![limit], |row| {
Ok(CatalogApp {
id: row.get(0)?,
name: row.get(1)?,
description: row.get(2)?,
categories: row.get(3)?,
download_url: row.get(4)?,
icon_url: row.get(5)?,
homepage: row.get(6)?,
license: row.get(7)?,
screenshots: row.get(8)?,
github_owner: row.get(9)?,
github_repo: row.get(10)?,
github_stars: row.get(11)?,
github_downloads: row.get(12)?,
latest_version: row.get(13)?,
release_date: row.get(14)?,
github_enriched_at: row.get(15)?,
github_download_url: row.get(16)?,
github_release_assets: row.get(17)?,
})
})?;
LIMIT ?1",
Self::CATALOG_APP_COLUMNS
);
let mut stmt = self.conn.prepare(&sql)?;
let rows = stmt.query_map(params![limit], Self::catalog_app_from_row)?;
rows.collect()
}
@@ -2538,6 +2724,18 @@ impl Database {
)?;
Ok((enriched, total_with_github))
}
pub fn update_catalog_app_readme(
&self,
app_id: i64,
readme: &str,
) -> SqlResult<()> {
self.conn.execute(
"UPDATE catalog_apps SET github_readme = ?2 WHERE id = ?1",
params![app_id, readme],
)?;
Ok(())
}
}
#[cfg(test)]
@@ -2708,7 +2906,7 @@ mod tests {
[],
|row| row.get(0),
).unwrap();
assert_eq!(version, 15);
assert_eq!(version, 18);
// All tables that should exist after the full v1-v7 migration chain
let expected_tables = [