Add AppImageHub.com OCS API as primary catalog source
Integrate the Pling/OCS REST API (appimagehub.com) as the primary catalog source with richer metadata than the existing appimage.github.io feed. Backend: - Add OCS API fetch with pagination, lenient JSON deserializers for loosely typed numeric fields, and non-AppImage file filtering (.dmg, .exe, etc.) - Database migration v17 adds OCS-specific columns (ocs_id, downloads, score, typename, personid, description, summary, version, tags, etc.) - Deduplicate secondary source apps against OCS entries - Shrink OCS CDN icon URLs from 770x540 to 100x100 for faster loading - Clear stale screenshot and icon caches on sync - Extract GitHub repo links from OCS HTML descriptions - Add fetch_ocs_download_files() to get all version files for an app - Resolve fresh JWT download URLs per slot at install time Detail page: - Fetch OCS download files on page open and populate install SplitButton with version dropdown (newest first, filtered for AppImage only) - Show OCS metadata: downloads, score, author, typename, tags, comments, created/updated dates, architecture, filename, file size, MD5 - Prefer ocs_description (full HTML with features/changelog) over short summary for the About section - Add html_to_description() to preserve formatting (lists, paragraphs) - Remove redundant Download link from Links section - Escape ampersands in Pango markup subtitles (categories, typename, tags) Catalog view: - OCS source syncs first as primary, appimage.github.io as secondary - Featured apps consider OCS download counts alongside GitHub stars UI: - Add pulldown-cmark for GitHub README markdown rendering in detail pages - Add build_markdown_view() widget for rendered markdown content
This commit is contained in:
@@ -110,6 +110,54 @@ pub fn fetch_release_info(owner: &str, repo: &str, token: &str) -> Result<(GitHu
|
||||
Ok((info, remaining))
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
struct GitHubReadmeResponse {
|
||||
content: String,
|
||||
#[serde(default)]
|
||||
encoding: String,
|
||||
}
|
||||
|
||||
/// Fetch the README content for a repo (decoded from base64).
|
||||
pub fn fetch_readme(owner: &str, repo: &str, token: &str) -> Result<(String, u32), String> {
|
||||
let url = format!("https://api.github.com/repos/{}/{}/readme", owner, repo);
|
||||
let (body, remaining) = github_get(&url, token)?;
|
||||
let resp: GitHubReadmeResponse = serde_json::from_str(&body)
|
||||
.map_err(|e| format!("Parse error: {}", e))?;
|
||||
|
||||
if resp.encoding != "base64" {
|
||||
return Err(format!("Unexpected encoding: {}", resp.encoding));
|
||||
}
|
||||
|
||||
// GitHub returns base64 with newlines; strip them before decoding
|
||||
let clean = resp.content.replace('\n', "");
|
||||
let decoded = base64_decode(&clean)
|
||||
.map_err(|e| format!("Base64 decode error: {}", e))?;
|
||||
let text = String::from_utf8(decoded)
|
||||
.map_err(|e| format!("UTF-8 error: {}", e))?;
|
||||
Ok((text, remaining))
|
||||
}
|
||||
|
||||
/// Simple base64 decoder (standard alphabet, no padding required).
|
||||
fn base64_decode(input: &str) -> Result<Vec<u8>, String> {
|
||||
const TABLE: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
let mut output = Vec::with_capacity(input.len() * 3 / 4);
|
||||
let mut buf = 0u32;
|
||||
let mut bits = 0u32;
|
||||
for &b in input.as_bytes() {
|
||||
if b == b'=' { break; }
|
||||
let val = TABLE.iter().position(|&c| c == b)
|
||||
.ok_or_else(|| format!("Invalid base64 char: {}", b as char))? as u32;
|
||||
buf = (buf << 6) | val;
|
||||
bits += 6;
|
||||
if bits >= 8 {
|
||||
bits -= 8;
|
||||
output.push((buf >> bits) as u8);
|
||||
buf &= (1 << bits) - 1;
|
||||
}
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
// --- AppImage asset filtering ---
|
||||
|
||||
/// A simplified release asset for storage (JSON-serializable).
|
||||
@@ -163,7 +211,7 @@ pub fn pick_best_asset(assets: &[AppImageAsset]) -> Option<&AppImageAsset> {
|
||||
|
||||
// --- Enrichment logic ---
|
||||
|
||||
/// Enrich a catalog app with repo-level info (stars, pushed_at).
|
||||
/// Enrich a catalog app with repo-level info (stars, pushed_at, description).
|
||||
pub fn enrich_app_repo_info(
|
||||
db: &Database,
|
||||
app_id: i64,
|
||||
@@ -172,8 +220,9 @@ pub fn enrich_app_repo_info(
|
||||
token: &str,
|
||||
) -> Result<u32, String> {
|
||||
let (info, remaining) = fetch_repo_info(owner, repo, token)?;
|
||||
db.update_catalog_app_github_metadata(app_id, info.stargazers_count, info.pushed_at.as_deref())
|
||||
.map_err(|e| format!("DB error: {}", e))?;
|
||||
db.update_catalog_app_github_metadata(
|
||||
app_id, info.stargazers_count, info.pushed_at.as_deref(), info.description.as_deref(),
|
||||
).map_err(|e| format!("DB error: {}", e))?;
|
||||
Ok(remaining)
|
||||
}
|
||||
|
||||
@@ -216,6 +265,20 @@ pub fn enrich_app_release_info(
|
||||
Ok(remaining)
|
||||
}
|
||||
|
||||
/// Fetch and store the README for a catalog app.
|
||||
pub fn enrich_app_readme(
|
||||
db: &Database,
|
||||
app_id: i64,
|
||||
owner: &str,
|
||||
repo: &str,
|
||||
token: &str,
|
||||
) -> Result<u32, String> {
|
||||
let (readme, remaining) = fetch_readme(owner, repo, token)?;
|
||||
db.update_catalog_app_readme(app_id, &readme)
|
||||
.map_err(|e| format!("DB error: {}", e))?;
|
||||
Ok(remaining)
|
||||
}
|
||||
|
||||
/// Background enrichment: process a batch of unenriched apps.
|
||||
/// Returns (count_enriched, should_continue).
|
||||
pub fn background_enrich_batch(
|
||||
@@ -261,7 +324,7 @@ pub fn background_enrich_batch(
|
||||
Err(e) => {
|
||||
log::warn!("Failed to enrich {}/{}: {}", owner, repo, e);
|
||||
// Mark as enriched anyway so we don't retry forever
|
||||
db.update_catalog_app_github_metadata(app.id, 0, None).ok();
|
||||
db.update_catalog_app_github_metadata(app.id, 0, None, None).ok();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user