use std::collections::BTreeMap; use std::fmt::Write as _; use std::fs; use std::path::{Path, PathBuf}; use std::sync::Arc; use anyhow::{Context, Result}; use crate::highlight::Cache as GrammarCache; use crate::render::{self, RefKind, RefTip, Site, StaticMode}; pub const MANIFEST_FILE: &str = ".sorcery-blobs.json"; pub const STATE_FILE: &str = ".sorcery-state"; pub const STYLESHEET: &str = include_str!("style.css"); pub(crate) const OUTPUT_FORMAT_VERSION: u32 = 55; const MAX_STATIC_BRANCHES: usize = 8; const MAX_HIGHLIGHTED_FILES: usize = 10_000; #[derive(Clone)] pub struct Config { pub repo: PathBuf, pub out: PathBuf, pub instance_name: String, pub name: Option, pub clone_url: Option, } #[derive(serde::Deserialize, serde::Serialize)] struct BlobManifest { format: u32, instance_name: String, name: String, refs: BTreeMap, } #[derive(serde::Deserialize, serde::Serialize)] struct RefManifest { tip: String, files: BTreeMap, } struct PendingBlob { oid: String, mode: u16, touched: Option, } impl BlobManifest { fn load(out: &Path, instance_name: &str, name: &str) -> Option { let data = fs::read(out.join(MANIFEST_FILE)).ok()?; let manifest = serde_json::from_slice::(&data).ok()?; (manifest.format == OUTPUT_FORMAT_VERSION && manifest.instance_name == instance_name && manifest.name == name) .then_some(manifest) } } /// The complete input identity of generated output. Bump /// `OUTPUT_FORMAT_VERSION` whenever templates or styling change incompatibly. pub fn expected_state(config: &Config) -> Result { let repo = gix::open(&config.repo) .with_context(|| format!("opening git repo at {}", config.repo.display()))?; let name = config.name.clone().unwrap_or_else(|| repo_name(&repo)); let description = description(&repo); let tips = render::list_refs(&repo)?; Ok(state_for(config, &repo, &tips, &name, description.as_deref())) } pub fn full(config: &Config, previous: Option<&Path>, highlights: Arc) -> Result<()> { let repo = gix::open(&config.repo) .with_context(|| format!("opening git repo at {}", config.repo.display()))?; let name = config.name.clone().unwrap_or_else(|| repo_name(&repo)); let description = description(&repo); let mut tips = render::list_refs(&repo)?; let state = state_for(config, &repo, &tips, &name, description.as_deref()); plan_rendering(&repo, &mut tips)?; let base_url = format!("/{}/", render::encode_path(&name)); let site = Site { repo: &repo, instance_name: config.instance_name.clone(), name, base_url, description, clone_url: config.clone_url.clone(), }; fs::create_dir_all(&config.out)?; let previous_manifest = previous .and_then(|out| BlobManifest::load(out, &site.instance_name, &site.name)); let refs = rebuild_all( &site, &tips, &config.out, previous, previous_manifest.as_ref(), highlights.clone(), )?; render::render_refs_page(&site, &tips, &config.out)?; render::render_site_index(&site, &tips, &config.out, highlights)?; render::write_file(&config.out.join("gitinfo.json"), gitinfo(&repo, &tips)?)?; let manifest = BlobManifest { format: OUTPUT_FORMAT_VERSION, instance_name: site.instance_name.clone(), name: site.name.clone(), refs, }; render::write_file( &config.out.join(MANIFEST_FILE), serde_json::to_vec(&manifest)?, )?; render::write_file(&config.out.join(STATE_FILE), state) } fn plan_rendering(repo: &gix::Repository, tips: &mut [RefTip]) -> Result<()> { let branch_count = tips.iter().filter(|tip| tip.kind == RefKind::Branch).count(); let head = render::head_tip(repo, tips).map(|tip| tip.name.clone()); for tip in tips { if tip.kind != RefKind::Branch || (branch_count > MAX_STATIC_BRANCHES && Some(&tip.name) != head.as_ref()) { continue; } let commit = repo.find_object(tip.commit_id)?.try_into_commit()?; tip.static_mode = Some(if exceeds_file_limit(repo, &commit.tree()?, MAX_HIGHLIGHTED_FILES)? { StaticMode::Plain } else { StaticMode::Highlighted }); } Ok(()) } fn exceeds_file_limit( repo: &gix::Repository, tree: &gix::Tree<'_>, limit: usize, ) -> Result { fn walk( repo: &gix::Repository, tree: &gix::Tree<'_>, files: &mut usize, limit: usize, ) -> Result { for entry in tree.iter() { let entry = entry?; let mode = entry.mode(); if mode.is_tree() { let tree = repo.find_object(entry.oid())?.try_into_tree()?; if walk(repo, &tree, files, limit)? { return Ok(true); } } else if !mode.is_commit() { *files += 1; if *files > limit { return Ok(true); } } } Ok(false) } let mut files = 0; walk(repo, tree, &mut files, limit) } /// Discovery manifest for the in-browser git client: HEAD, refs (peeled to /// commits, branch-shadows-tag like the rest of the site), and pack names so /// the client can find `.idx` files without directory listings. fn gitinfo(repo: &gix::Repository, tips: &[RefTip]) -> Result { let head = render::head_tip(repo, tips).map(|t| t.name.clone()); let refs = tips .iter() .map(|t| { serde_json::json!({ "kind": match t.kind { RefKind::Branch => "branch", RefKind::Tag => "tag", }, "name": t.name, "oid": t.commit_id.to_string(), }) }) .collect::>(); Ok(serde_json::json!({ "head": head, "refs": refs, "packs": pack_names(repo), }) .to_string()) } /// Sorted `pack-*` stems in the repo's object store. fn pack_names(repo: &gix::Repository) -> Vec { let mut packs: Vec = fs::read_dir(repo.git_dir().join("objects/pack")) .into_iter() .flatten() .flatten() .filter_map(|entry| { let name = entry.file_name().to_string_lossy().into_owned(); name.strip_suffix(".pack") .filter(|stem| stem.starts_with("pack-")) .map(str::to_owned) }) .collect(); packs.sort(); packs } pub(crate) fn description(repo: &gix::Repository) -> Option { fs::read_to_string(repo.git_dir().join("description")) .ok() .map(|d| d.trim().to_string()) .filter(|d| !d.is_empty() && !d.starts_with("Unnamed repository")) } fn state_for( config: &Config, repo: &gix::Repository, tips: &[RefTip], name: &str, description: Option<&str>, ) -> String { let mut refs = tips.iter().collect::>(); refs.sort_by(|a, b| a.name.cmp(&b.name)); let mut state = format!( "format {OUTPUT_FORMAT_VERSION}\ninstance {:?}\nname {name:?}\nclone {:?}\ndescription {description:?}\nhead {:?}\n", config.instance_name, config.clone_url, repo.head_name().ok().flatten().map(|n| n.to_string()), ); for tip in refs { let kind = match tip.kind { RefKind::Branch => "branch", RefKind::Tag => "tag", }; let _ = writeln!(state, "{kind} {:?} {}", tip.name, tip.commit_id); } // a gc changes packs without changing refs; the git-dir client cares for pack in pack_names(repo) { let _ = writeln!(state, "pack {pack}"); } state } /// `/srv/git/foo.git` → `foo`, `/home/x/foo/.git` → `foo` fn repo_name(repo: &gix::Repository) -> String { let dir = repo.git_dir().canonicalize().unwrap_or_else(|_| repo.git_dir().into()); let dir = if dir.file_name() == Some(".git".as_ref()) { dir.parent().unwrap_or(&dir) } else { &dir }; dir.file_name() .map(|n| n.to_string_lossy().trim_end_matches(".git").to_string()) .filter(|n| !n.is_empty()) .unwrap_or_else(|| "repository".into()) } fn blob_versions( repo: &gix::Repository, tip: &gix::Commit, previous: Option<&RefManifest>, ) -> Result> { let mut files = BTreeMap::new(); collect_blobs(repo, tip.tree_id()?.detach(), &mut Vec::new(), &mut files)?; let mut unresolved = files.len(); let mut commit_id = tip.id; // Match the browser's first-parent path history. Once its previous tip is // reached, paths untouched by newer commits can inherit their old result. while unresolved > 0 { let commit = repo.find_object(commit_id)?.try_into_commit()?; let commit_id_string = commit_id.to_string(); if let Some(previous) = previous.filter(|previous| previous.tip == commit_id_string) { for (path, file) in &mut files { if file.touched.is_some() { continue; } if let Some(old) = previous.files.get(path) && old.oid == file.oid && old.mode == file.mode { file.touched = Some(old.touched.clone()); unresolved -= 1; } } if unresolved == 0 { break; } } let new_tree = commit.tree_id()?.detach(); let parent = commit.parent_ids().next().map(|id| id.detach()); let old_tree = match parent { Some(id) => Some( repo.find_object(id)? .try_into_commit()? .tree_id()? .detach(), ), None => None, }; mark_tree_changes( repo, old_tree, Some(new_tree), &commit_id_string, &mut files, &mut unresolved, )?; let Some(parent) = parent else { break; }; commit_id = parent; } files .into_iter() .map(|(path, file)| { Ok(( path, render::BlobVersion { oid: file.oid, mode: file.mode, touched: file.touched.context("file has no introducing commit")?, }, )) }) .collect() } fn collect_blobs( repo: &gix::Repository, tree_id: gix::ObjectId, path: &mut Vec, files: &mut BTreeMap, ) -> Result<()> { let tree = repo.find_object(tree_id)?.try_into_tree()?; for entry in tree.iter() { let entry = entry?; let mode = entry.mode(); path.push(entry.filename().to_string()); if mode.is_tree() { collect_blobs(repo, entry.oid().to_owned(), path, files)?; } else if !mode.is_commit() { files.insert( path.join("/"), PendingBlob { oid: entry.oid().to_string(), mode: mode.value(), touched: None, }, ); } path.pop(); } Ok(()) } fn mark_tree_changes( repo: &gix::Repository, old_tree: Option, new_tree: Option, commit_id: &str, files: &mut BTreeMap, unresolved: &mut usize, ) -> Result<()> { let old_tree = match old_tree { Some(id) => Some(repo.find_object(id)?.try_into_tree()?), None => None, }; let new_tree = match new_tree { Some(id) => Some(repo.find_object(id)?.try_into_tree()?), None => None, }; for change in repo.diff_tree_to_tree( old_tree.as_ref(), new_tree.as_ref(), Some(gix::diff::Options::default()), )? { use gix::object::tree::diff::ChangeDetached; let path = match change { ChangeDetached::Addition { location, .. } | ChangeDetached::Deletion { location, .. } | ChangeDetached::Modification { location, .. } | ChangeDetached::Rewrite { location, .. } => location.to_string(), }; if let Some(file) = files.get_mut(&path) && file.touched.is_none() { file.touched = Some(commit_id.to_owned()); *unresolved -= 1; } } Ok(()) } /// Rebuild all visible refs, replacing `ref/` wholesale (which also prunes /// refs deleted since the last run). fn rebuild_all( site: &Site, tips: &[RefTip], out: &Path, previous: Option<&Path>, previous_manifest: Option<&BlobManifest>, highlights: Arc, ) -> Result> { let staging = out.join(".new-ref"); if staging.exists() { fs::remove_dir_all(&staging)?; } fs::create_dir_all(&staging)?; let mut refs = BTreeMap::new(); for tip in tips.iter().filter(|tip| tip.static_mode.is_some()) { let commit = site.repo.find_object(tip.commit_id)?.try_into_commit()?; let previous_ref = previous.map(|out| out.join("ref").join(&tip.name)); let previous_version = previous_manifest.and_then(|manifest| manifest.refs.get(&tip.name)); let files = if tip.static_mode == Some(StaticMode::Highlighted) { Some(blob_versions(site.repo, &commit, previous_version)?) } else { None }; let reuse = files.as_ref().map(|versions| render::BlobReuse { versions, previous: previous_ref .as_deref() .zip(previous_version.map(|version| &version.files)), }); render::render_ref( site, tips, tip, &commit, &staging.join(&tip.name), reuse, highlights.clone(), ) .with_context(|| format!("rendering {}", tip.name))?; if let Some(files) = files { refs.insert( tip.name.clone(), RefManifest { tip: tip.commit_id.to_string(), files, }, ); } } swap_dir(&staging, &out.join("ref"))?; Ok(refs) } /// Move `new` into place at `dst`, displacing any existing tree with only a /// brief window where `dst` is missing (rename is atomic, recursive delete /// is not, so the old tree is renamed aside before removal, and restored if /// the install rename fails). pub(crate) fn swap_dir(new: &Path, dst: &Path) -> Result<()> { if let Some(parent) = dst.parent() { fs::create_dir_all(parent)?; } let trash = new.with_extension("old"); if trash.exists() { fs::remove_dir_all(&trash)?; } let displaced = dst.exists(); if displaced { fs::rename(dst, &trash)?; } if let Err(error) = fs::rename(new, dst) { if displaced { let _ = fs::rename(&trash, dst); } return Err(error).with_context(|| format!("installing {}", dst.display())); } if displaced { if let Err(error) = fs::remove_dir_all(&trash) { eprintln!("sorcery: removing {}: {error}", trash.display()); } } Ok(()) } #[cfg(test)] mod tests { use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; use anyhow::Result; use percent_encoding::percent_decode_str; use crate::testutil::{TempDir, commit, git, grammar_cache, init_repo, init_sha256_repo}; use super::{ BlobManifest, Config, MANIFEST_FILE, OUTPUT_FORMAT_VERSION, RefManifest, blob_versions, full, }; #[test] fn tracks_last_touch_across_incremental_rebuilds_and_reverts() -> Result<()> { let root = TempDir::new("last-touch"); let repo_path = root.join("repo"); init_repo(&repo_path)?; fs::create_dir(repo_path.join("src"))?; fs::write(repo_path.join("src/stable.txt"), "stable")?; fs::write(repo_path.join("changed.txt"), "one")?; let first = commit(&repo_path, "first")?; fs::write(repo_path.join("changed.txt"), "two")?; let second = commit(&repo_path, "second")?; fs::write(repo_path.join("changed.txt"), "one")?; let third = commit(&repo_path, "revert")?; let repo = gix::open(&repo_path)?; let commit_at = |hex: &str| -> Result> { Ok(repo .find_object(gix::ObjectId::from_hex(hex.as_bytes())?)? .try_into_commit()?) }; let first_files = blob_versions(&repo, &commit_at(&first)?, None)?; assert_eq!(first_files["src/stable.txt"].touched, first); assert_eq!(first_files["changed.txt"].touched, first); let first_manifest = RefManifest { tip: first.clone(), files: first_files.clone(), }; let second_files = blob_versions(&repo, &commit_at(&second)?, Some(&first_manifest))?; assert_eq!(second_files["src/stable.txt"].touched, first); assert_eq!(second_files["changed.txt"].touched, second); let third_files = blob_versions(&repo, &commit_at(&third)?, Some(&first_manifest))?; assert_eq!(third_files["changed.txt"].oid, first_files["changed.txt"].oid); assert_eq!(third_files["changed.txt"].touched, third); Ok(()) } #[test] fn renders_sha256_repositories() -> Result<()> { let root = TempDir::new("sha256"); let repo = root.join("repo"); init_sha256_repo(&repo)?; fs::write(repo.join("README.md"), "# SHA-256\n")?; let head = commit(&repo, "initial")?; assert_eq!(head.len(), 64); let out = root.join("out"); full( &Config { repo, out: out.clone(), instance_name: "test".into(), name: Some("sha256".into()), clone_url: None, }, None, grammar_cache(), )?; let info: serde_json::Value = serde_json::from_slice(&fs::read(out.join("gitinfo.json"))?)?; assert_eq!(info["refs"][0]["oid"].as_str(), Some(head.as_str())); assert!(out.join("ref/main/blob/README.md").is_file()); Ok(()) } #[test] fn loads_only_compatible_blob_manifests() -> Result<()> { let root = TempDir::new("blob-manifest"); let files = BTreeMap::from([( "odd\npath.rs".into(), crate::render::BlobVersion { oid: "abc123".into(), mode: 0o100644, touched: "def456".into(), }, )]); let manifest = BlobManifest { format: OUTPUT_FORMAT_VERSION, instance_name: "example".into(), name: "alice/repo".into(), refs: BTreeMap::from([( "main".into(), RefManifest { tip: "def456".into(), files, }, )]), }; fs::write(root.join(MANIFEST_FILE), serde_json::to_vec(&manifest)?)?; let loaded = BlobManifest::load(&root, "example", "alice/repo").unwrap(); assert_eq!(loaded.refs["main"].files["odd\npath.rs"].oid, "abc123"); assert_eq!(loaded.refs["main"].files["odd\npath.rs"].touched, "def456"); assert!(BlobManifest::load(&root, "other", "alice/repo").is_none()); assert!(BlobManifest::load(&root, "example", "other/repo").is_none()); let incompatible = BlobManifest { format: OUTPUT_FORMAT_VERSION + 1, ..manifest }; fs::write(root.join(MANIFEST_FILE), serde_json::to_vec(&incompatible)?)?; assert!(BlobManifest::load(&root, "example", "alice/repo").is_none()); fs::write(root.join(MANIFEST_FILE), "not json")?; assert!(BlobManifest::load(&root, "example", "alice/repo").is_none()); Ok(()) } fn html_files(dir: &Path, into: &mut Vec) -> Result<()> { for entry in fs::read_dir(dir)? { let path = entry?.path(); if path.is_dir() { html_files(&path, into)?; } else if fs::read(&path)?.starts_with(b"") { into.push(path); } } Ok(()) } /// The generator's links and its output paths are produced by separate /// code, so a mismatch in encoding or layout between them only shows up /// by following the links. #[test] fn every_internal_link_resolves() -> Result<()> { let root = TempDir::new("links"); let repo = root.join("repo"); init_repo(&repo)?; fs::create_dir_all(repo.join("dir with space/nested"))?; fs::write(repo.join("README"), "plain readme")?; fs::write(repo.join("dir with space/nested/f#1?.txt"), "one")?; fs::write(repo.join("dir with space/\u{e9}.txt"), "two")?; std::os::unix::fs::symlink("README", repo.join("link"))?; commit(&repo, "first")?; git(&repo, &["tag", "v1"])?; git(&repo, &["branch", "feature/x"])?; let out = root.join("out"); let config = Config { repo: repo.clone(), out: out.clone(), instance_name: "test".into(), name: Some("alice/my repo".into()), clone_url: None, }; full(&config, None, grammar_cache())?; let base = "/alice/my%20repo/"; let mut pages = Vec::new(); html_files(&out, &mut pages)?; assert!(!pages.is_empty()); let mut checked = 0; for page in pages { let html = fs::read_to_string(&page)?; let links = ["href=\"", "src=\""] .iter() .flat_map(|attr| html.split(attr).skip(1)) .map(|rest| rest.split('"').next().unwrap()); for link in links { assert!(link.starts_with(['/', '#']), "{}: relative link {link}", page.display()); let Some(rel) = link.strip_prefix(base) else { continue; }; let rel = rel.split('#').next().unwrap(); // Raw blobs are served by the daemon straight from the repo. if rel.starts_with("raw/") { continue; } let rel = percent_decode_str(rel).decode_utf8()?; let target = if rel.is_empty() || rel.ends_with('/') { out.join(&*rel).join("index.html") } else { out.join(&*rel) }; assert!( target.is_file(), "{}: {link} -> missing {}", page.display(), target.display(), ); checked += 1; } } assert!(checked > 0); Ok(()) } }