char/sorcery

git clone https://git.t4t.associates/char/sorcery

Charlotte Somexperiment: soft line wrapping on source code rendersf92b8f42026-09-04

use std::collections::BTreeMap;
use std::fmt::Write as _;
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::Arc;

use anyhow::{Context, Result};

use crate::highlight::Cache as GrammarCache;
use crate::render::{self, RefKind, RefTip, Site, StaticMode};

pub const MANIFEST_FILE: &str = ".sorcery-blobs.json";
pub const STATE_FILE: &str = ".sorcery-state";
pub const STYLESHEET: &str = include_str!("style.css");
pub(crate) const OUTPUT_FORMAT_VERSION: u32 = 55;
const MAX_STATIC_BRANCHES: usize = 8;
const MAX_HIGHLIGHTED_FILES: usize = 10_000;

#[derive(Clone)]
pub struct Config {
    pub repo: PathBuf,
    pub out: PathBuf,
    pub instance_name: String,
    pub name: Option<String>,
    pub clone_url: Option<String>,
}

#[derive(serde::Deserialize, serde::Serialize)]
struct BlobManifest {
    format: u32,
    instance_name: String,
    name: String,
    refs: BTreeMap<String, RefManifest>,
}

#[derive(serde::Deserialize, serde::Serialize)]
struct RefManifest {
    tip: String,
    files: BTreeMap<String, render::BlobVersion>,
}

struct PendingBlob {
    oid: String,
    mode: u16,
    touched: Option<String>,
}

impl BlobManifest {
    fn load(out: &Path, instance_name: &str, name: &str) -> Option<Self> {
        let data = fs::read(out.join(MANIFEST_FILE)).ok()?;
        let manifest = serde_json::from_slice::<Self>(&data).ok()?;
        (manifest.format == OUTPUT_FORMAT_VERSION
            && manifest.instance_name == instance_name
            && manifest.name == name)
            .then_some(manifest)
    }
}

/// The complete input identity of generated output. Bump
/// `OUTPUT_FORMAT_VERSION` whenever templates or styling change incompatibly.
pub fn expected_state(config: &Config) -> Result<String> {
    let repo = gix::open(&config.repo)
        .with_context(|| format!("opening git repo at {}", config.repo.display()))?;
    let name = config.name.clone().unwrap_or_else(|| repo_name(&repo));
    let description = description(&repo);
    let tips = render::list_refs(&repo)?;
    Ok(state_for(config, &repo, &tips, &name, description.as_deref()))
}

pub fn full(config: &Config, previous: Option<&Path>, highlights: Arc<GrammarCache>) -> Result<()> {
    let repo = gix::open(&config.repo)
        .with_context(|| format!("opening git repo at {}", config.repo.display()))?;
    let name = config.name.clone().unwrap_or_else(|| repo_name(&repo));
    let description = description(&repo);
    let mut tips = render::list_refs(&repo)?;
    let state = state_for(config, &repo, &tips, &name, description.as_deref());
    plan_rendering(&repo, &mut tips)?;
    let base_url = format!("/{}/", render::encode_path(&name));
    let site = Site {
        repo: &repo,
        instance_name: config.instance_name.clone(),
        name,
        base_url,
        description,
        clone_url: config.clone_url.clone(),
    };

    fs::create_dir_all(&config.out)?;
    let previous_manifest = previous
        .and_then(|out| BlobManifest::load(out, &site.instance_name, &site.name));
    let refs = rebuild_all(
        &site,
        &tips,
        &config.out,
        previous,
        previous_manifest.as_ref(),
        highlights.clone(),
    )?;
    render::render_refs_page(&site, &tips, &config.out)?;
    render::render_site_index(&site, &tips, &config.out, highlights)?;
    render::write_file(&config.out.join("gitinfo.json"), gitinfo(&repo, &tips)?)?;
    let manifest = BlobManifest {
        format: OUTPUT_FORMAT_VERSION,
        instance_name: site.instance_name.clone(),
        name: site.name.clone(),
        refs,
    };
    render::write_file(
        &config.out.join(MANIFEST_FILE),
        serde_json::to_vec(&manifest)?,
    )?;
    render::write_file(&config.out.join(STATE_FILE), state)
}

fn plan_rendering(repo: &gix::Repository, tips: &mut [RefTip]) -> Result<()> {
    let branch_count = tips.iter().filter(|tip| tip.kind == RefKind::Branch).count();
    let head = render::head_tip(repo, tips).map(|tip| tip.name.clone());
    for tip in tips {
        if tip.kind != RefKind::Branch
            || (branch_count > MAX_STATIC_BRANCHES && Some(&tip.name) != head.as_ref())
        {
            continue;
        }
        let commit = repo.find_object(tip.commit_id)?.try_into_commit()?;
        tip.static_mode = Some(if exceeds_file_limit(repo, &commit.tree()?, MAX_HIGHLIGHTED_FILES)? {
            StaticMode::Plain
        } else {
            StaticMode::Highlighted
        });
    }
    Ok(())
}

fn exceeds_file_limit(
    repo: &gix::Repository,
    tree: &gix::Tree<'_>,
    limit: usize,
) -> Result<bool> {
    fn walk(
        repo: &gix::Repository,
        tree: &gix::Tree<'_>,
        files: &mut usize,
        limit: usize,
    ) -> Result<bool> {
        for entry in tree.iter() {
            let entry = entry?;
            let mode = entry.mode();
            if mode.is_tree() {
                let tree = repo.find_object(entry.oid())?.try_into_tree()?;
                if walk(repo, &tree, files, limit)? {
                    return Ok(true);
                }
            } else if !mode.is_commit() {
                *files += 1;
                if *files > limit {
                    return Ok(true);
                }
            }
        }
        Ok(false)
    }

    let mut files = 0;
    walk(repo, tree, &mut files, limit)
}

/// Discovery manifest for the in-browser git client: HEAD, refs (peeled to
/// commits, branch-shadows-tag like the rest of the site), and pack names so
/// the client can find `.idx` files without directory listings.
fn gitinfo(repo: &gix::Repository, tips: &[RefTip]) -> Result<String> {
    let head = render::head_tip(repo, tips).map(|t| t.name.clone());
    let refs = tips
        .iter()
        .map(|t| {
            serde_json::json!({
                "kind": match t.kind {
                    RefKind::Branch => "branch",
                    RefKind::Tag => "tag",
                },
                "name": t.name,
                "oid": t.commit_id.to_string(),
            })
        })
        .collect::<Vec<_>>();
    Ok(serde_json::json!({
        "head": head,
        "refs": refs,
        "packs": pack_names(repo),
    })
    .to_string())
}

/// Sorted `pack-*` stems in the repo's object store.
fn pack_names(repo: &gix::Repository) -> Vec<String> {
    let mut packs: Vec<String> = fs::read_dir(repo.git_dir().join("objects/pack"))
        .into_iter()
        .flatten()
        .flatten()
        .filter_map(|entry| {
            let name = entry.file_name().to_string_lossy().into_owned();
            name.strip_suffix(".pack")
                .filter(|stem| stem.starts_with("pack-"))
                .map(str::to_owned)
        })
        .collect();
    packs.sort();
    packs
}

pub(crate) fn description(repo: &gix::Repository) -> Option<String> {
    fs::read_to_string(repo.git_dir().join("description"))
        .ok()
        .map(|d| d.trim().to_string())
        .filter(|d| !d.is_empty() && !d.starts_with("Unnamed repository"))
}

fn state_for(
    config: &Config,
    repo: &gix::Repository,
    tips: &[RefTip],
    name: &str,
    description: Option<&str>,
) -> String {
    let mut refs = tips.iter().collect::<Vec<_>>();
    refs.sort_by(|a, b| a.name.cmp(&b.name));
    let mut state = format!(
        "format {OUTPUT_FORMAT_VERSION}\ninstance {:?}\nname {name:?}\nclone {:?}\ndescription {description:?}\nhead {:?}\n",
        config.instance_name,
        config.clone_url,
        repo.head_name().ok().flatten().map(|n| n.to_string()),
    );
    for tip in refs {
        let kind = match tip.kind {
            RefKind::Branch => "branch",
            RefKind::Tag => "tag",
        };
        let _ = writeln!(state, "{kind} {:?} {}", tip.name, tip.commit_id);
    }
    // a gc changes packs without changing refs; the git-dir client cares
    for pack in pack_names(repo) {
        let _ = writeln!(state, "pack {pack}");
    }
    state
}

/// `/srv/git/foo.git` → `foo`, `/home/x/foo/.git` → `foo`
fn repo_name(repo: &gix::Repository) -> String {
    let dir = repo.git_dir().canonicalize().unwrap_or_else(|_| repo.git_dir().into());
    let dir = if dir.file_name() == Some(".git".as_ref()) {
        dir.parent().unwrap_or(&dir)
    } else {
        &dir
    };
    dir.file_name()
        .map(|n| n.to_string_lossy().trim_end_matches(".git").to_string())
        .filter(|n| !n.is_empty())
        .unwrap_or_else(|| "repository".into())
}

fn blob_versions(
    repo: &gix::Repository,
    tip: &gix::Commit,
    previous: Option<&RefManifest>,
) -> Result<BTreeMap<String, render::BlobVersion>> {
    let mut files = BTreeMap::new();
    collect_blobs(repo, tip.tree_id()?.detach(), &mut Vec::new(), &mut files)?;
    let mut unresolved = files.len();
    let mut commit_id = tip.id;

    // Match the browser's first-parent path history. Once its previous tip is
    // reached, paths untouched by newer commits can inherit their old result.
    while unresolved > 0 {
        let commit = repo.find_object(commit_id)?.try_into_commit()?;
        let commit_id_string = commit_id.to_string();
        if let Some(previous) = previous.filter(|previous| previous.tip == commit_id_string) {
            for (path, file) in &mut files {
                if file.touched.is_some() {
                    continue;
                }
                if let Some(old) = previous.files.get(path)
                    && old.oid == file.oid
                    && old.mode == file.mode
                {
                    file.touched = Some(old.touched.clone());
                    unresolved -= 1;
                }
            }
            if unresolved == 0 {
                break;
            }
        }

        let new_tree = commit.tree_id()?.detach();
        let parent = commit.parent_ids().next().map(|id| id.detach());
        let old_tree = match parent {
            Some(id) => Some(
                repo.find_object(id)?
                    .try_into_commit()?
                    .tree_id()?
                    .detach(),
            ),
            None => None,
        };
        mark_tree_changes(
            repo,
            old_tree,
            Some(new_tree),
            &commit_id_string,
            &mut files,
            &mut unresolved,
        )?;
        let Some(parent) = parent else {
            break;
        };
        commit_id = parent;
    }

    files
        .into_iter()
        .map(|(path, file)| {
            Ok((
                path,
                render::BlobVersion {
                    oid: file.oid,
                    mode: file.mode,
                    touched: file.touched.context("file has no introducing commit")?,
                },
            ))
        })
        .collect()
}

fn collect_blobs(
    repo: &gix::Repository,
    tree_id: gix::ObjectId,
    path: &mut Vec<String>,
    files: &mut BTreeMap<String, PendingBlob>,
) -> Result<()> {
    let tree = repo.find_object(tree_id)?.try_into_tree()?;
    for entry in tree.iter() {
        let entry = entry?;
        let mode = entry.mode();
        path.push(entry.filename().to_string());
        if mode.is_tree() {
            collect_blobs(repo, entry.oid().to_owned(), path, files)?;
        } else if !mode.is_commit() {
            files.insert(
                path.join("/"),
                PendingBlob {
                    oid: entry.oid().to_string(),
                    mode: mode.value(),
                    touched: None,
                },
            );
        }
        path.pop();
    }
    Ok(())
}

fn mark_tree_changes(
    repo: &gix::Repository,
    old_tree: Option<gix::ObjectId>,
    new_tree: Option<gix::ObjectId>,
    commit_id: &str,
    files: &mut BTreeMap<String, PendingBlob>,
    unresolved: &mut usize,
) -> Result<()> {
    let old_tree = match old_tree {
        Some(id) => Some(repo.find_object(id)?.try_into_tree()?),
        None => None,
    };
    let new_tree = match new_tree {
        Some(id) => Some(repo.find_object(id)?.try_into_tree()?),
        None => None,
    };
    for change in repo.diff_tree_to_tree(
        old_tree.as_ref(),
        new_tree.as_ref(),
        Some(gix::diff::Options::default()),
    )? {
        use gix::object::tree::diff::ChangeDetached;
        let path = match change {
            ChangeDetached::Addition { location, .. }
            | ChangeDetached::Deletion { location, .. }
            | ChangeDetached::Modification { location, .. }
            | ChangeDetached::Rewrite { location, .. } => location.to_string(),
        };
        if let Some(file) = files.get_mut(&path)
            && file.touched.is_none()
        {
            file.touched = Some(commit_id.to_owned());
            *unresolved -= 1;
        }
    }
    Ok(())
}

/// Rebuild all visible refs, replacing `ref/` wholesale (which also prunes
/// refs deleted since the last run).
fn rebuild_all(
    site: &Site,
    tips: &[RefTip],
    out: &Path,
    previous: Option<&Path>,
    previous_manifest: Option<&BlobManifest>,
    highlights: Arc<GrammarCache>,
) -> Result<BTreeMap<String, RefManifest>> {
    let staging = out.join(".new-ref");
    if staging.exists() {
        fs::remove_dir_all(&staging)?;
    }
    fs::create_dir_all(&staging)?;
    let mut refs = BTreeMap::new();
    for tip in tips.iter().filter(|tip| tip.static_mode.is_some()) {
        let commit = site.repo.find_object(tip.commit_id)?.try_into_commit()?;
        let previous_ref = previous.map(|out| out.join("ref").join(&tip.name));
        let previous_version = previous_manifest.and_then(|manifest| manifest.refs.get(&tip.name));
        let files = if tip.static_mode == Some(StaticMode::Highlighted) {
            Some(blob_versions(site.repo, &commit, previous_version)?)
        } else {
            None
        };
        let reuse = files.as_ref().map(|versions| render::BlobReuse {
            versions,
            previous: previous_ref
                .as_deref()
                .zip(previous_version.map(|version| &version.files)),
        });
        render::render_ref(
            site,
            tips,
            tip,
            &commit,
            &staging.join(&tip.name),
            reuse,
            highlights.clone(),
        )
        .with_context(|| format!("rendering {}", tip.name))?;
        if let Some(files) = files {
            refs.insert(
                tip.name.clone(),
                RefManifest {
                    tip: tip.commit_id.to_string(),
                    files,
                },
            );
        }
    }
    swap_dir(&staging, &out.join("ref"))?;
    Ok(refs)
}

/// Move `new` into place at `dst`, displacing any existing tree with only a
/// brief window where `dst` is missing (rename is atomic, recursive delete
/// is not, so the old tree is renamed aside before removal, and restored if
/// the install rename fails).
pub(crate) fn swap_dir(new: &Path, dst: &Path) -> Result<()> {
    if let Some(parent) = dst.parent() {
        fs::create_dir_all(parent)?;
    }
    let trash = new.with_extension("old");
    if trash.exists() {
        fs::remove_dir_all(&trash)?;
    }
    let displaced = dst.exists();
    if displaced {
        fs::rename(dst, &trash)?;
    }
    if let Err(error) = fs::rename(new, dst) {
        if displaced {
            let _ = fs::rename(&trash, dst);
        }
        return Err(error).with_context(|| format!("installing {}", dst.display()));
    }
    if displaced {
        if let Err(error) = fs::remove_dir_all(&trash) {
            eprintln!("sorcery: removing {}: {error}", trash.display());
        }
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use std::collections::BTreeMap;
    use std::fs;
    use std::path::{Path, PathBuf};

    use anyhow::Result;
    use percent_encoding::percent_decode_str;

    use crate::testutil::{TempDir, commit, git, grammar_cache, init_repo, init_sha256_repo};

    use super::{
        BlobManifest, Config, MANIFEST_FILE, OUTPUT_FORMAT_VERSION, RefManifest, blob_versions,
        full,
    };

    #[test]
    fn tracks_last_touch_across_incremental_rebuilds_and_reverts() -> Result<()> {
        let root = TempDir::new("last-touch");
        let repo_path = root.join("repo");
        init_repo(&repo_path)?;
        fs::create_dir(repo_path.join("src"))?;
        fs::write(repo_path.join("src/stable.txt"), "stable")?;
        fs::write(repo_path.join("changed.txt"), "one")?;
        let first = commit(&repo_path, "first")?;
        fs::write(repo_path.join("changed.txt"), "two")?;
        let second = commit(&repo_path, "second")?;
        fs::write(repo_path.join("changed.txt"), "one")?;
        let third = commit(&repo_path, "revert")?;

        let repo = gix::open(&repo_path)?;
        let commit_at = |hex: &str| -> Result<gix::Commit<'_>> {
            Ok(repo
                .find_object(gix::ObjectId::from_hex(hex.as_bytes())?)?
                .try_into_commit()?)
        };
        let first_files = blob_versions(&repo, &commit_at(&first)?, None)?;
        assert_eq!(first_files["src/stable.txt"].touched, first);
        assert_eq!(first_files["changed.txt"].touched, first);

        let first_manifest = RefManifest {
            tip: first.clone(),
            files: first_files.clone(),
        };
        let second_files = blob_versions(&repo, &commit_at(&second)?, Some(&first_manifest))?;
        assert_eq!(second_files["src/stable.txt"].touched, first);
        assert_eq!(second_files["changed.txt"].touched, second);

        let third_files = blob_versions(&repo, &commit_at(&third)?, Some(&first_manifest))?;
        assert_eq!(third_files["changed.txt"].oid, first_files["changed.txt"].oid);
        assert_eq!(third_files["changed.txt"].touched, third);
        Ok(())
    }

    #[test]
    fn renders_sha256_repositories() -> Result<()> {
        let root = TempDir::new("sha256");
        let repo = root.join("repo");
        init_sha256_repo(&repo)?;
        fs::write(repo.join("README.md"), "# SHA-256\n")?;
        let head = commit(&repo, "initial")?;
        assert_eq!(head.len(), 64);

        let out = root.join("out");
        full(
            &Config {
                repo,
                out: out.clone(),
                instance_name: "test".into(),
                name: Some("sha256".into()),
                clone_url: None,
            },
            None,
            grammar_cache(),
        )?;

        let info: serde_json::Value = serde_json::from_slice(&fs::read(out.join("gitinfo.json"))?)?;
        assert_eq!(info["refs"][0]["oid"].as_str(), Some(head.as_str()));
        assert!(out.join("ref/main/blob/README.md").is_file());
        Ok(())
    }

    #[test]
    fn loads_only_compatible_blob_manifests() -> Result<()> {
        let root = TempDir::new("blob-manifest");
        let files = BTreeMap::from([(
            "odd\npath.rs".into(),
            crate::render::BlobVersion {
                oid: "abc123".into(),
                mode: 0o100644,
                touched: "def456".into(),
            },
        )]);
        let manifest = BlobManifest {
            format: OUTPUT_FORMAT_VERSION,
            instance_name: "example".into(),
            name: "alice/repo".into(),
            refs: BTreeMap::from([(
                "main".into(),
                RefManifest {
                    tip: "def456".into(),
                    files,
                },
            )]),
        };
        fs::write(root.join(MANIFEST_FILE), serde_json::to_vec(&manifest)?)?;

        let loaded = BlobManifest::load(&root, "example", "alice/repo").unwrap();
        assert_eq!(loaded.refs["main"].files["odd\npath.rs"].oid, "abc123");
        assert_eq!(loaded.refs["main"].files["odd\npath.rs"].touched, "def456");
        assert!(BlobManifest::load(&root, "other", "alice/repo").is_none());
        assert!(BlobManifest::load(&root, "example", "other/repo").is_none());

        let incompatible = BlobManifest {
            format: OUTPUT_FORMAT_VERSION + 1,
            ..manifest
        };
        fs::write(root.join(MANIFEST_FILE), serde_json::to_vec(&incompatible)?)?;
        assert!(BlobManifest::load(&root, "example", "alice/repo").is_none());

        fs::write(root.join(MANIFEST_FILE), "not json")?;
        assert!(BlobManifest::load(&root, "example", "alice/repo").is_none());
        Ok(())
    }

    fn html_files(dir: &Path, into: &mut Vec<PathBuf>) -> Result<()> {
        for entry in fs::read_dir(dir)? {
            let path = entry?.path();
            if path.is_dir() {
                html_files(&path, into)?;
            } else if fs::read(&path)?.starts_with(b"<!doctype html>") {
                into.push(path);
            }
        }
        Ok(())
    }

    /// The generator's links and its output paths are produced by separate
    /// code, so a mismatch in encoding or layout between them only shows up
    /// by following the links.
    #[test]
    fn every_internal_link_resolves() -> Result<()> {
        let root = TempDir::new("links");
        let repo = root.join("repo");
        init_repo(&repo)?;
        fs::create_dir_all(repo.join("dir with space/nested"))?;
        fs::write(repo.join("README"), "plain readme")?;
        fs::write(repo.join("dir with space/nested/f#1?.txt"), "one")?;
        fs::write(repo.join("dir with space/\u{e9}.txt"), "two")?;
        std::os::unix::fs::symlink("README", repo.join("link"))?;
        commit(&repo, "first")?;
        git(&repo, &["tag", "v1"])?;
        git(&repo, &["branch", "feature/x"])?;

        let out = root.join("out");
        let config = Config {
            repo: repo.clone(),
            out: out.clone(),
            instance_name: "test".into(),
            name: Some("alice/my repo".into()),
            clone_url: None,
        };
        full(&config, None, grammar_cache())?;

        let base = "/alice/my%20repo/";
        let mut pages = Vec::new();
        html_files(&out, &mut pages)?;
        assert!(!pages.is_empty());
        let mut checked = 0;
        for page in pages {
            let html = fs::read_to_string(&page)?;
            let links = ["href=\"", "src=\""]
                .iter()
                .flat_map(|attr| html.split(attr).skip(1))
                .map(|rest| rest.split('"').next().unwrap());
            for link in links {
                assert!(link.starts_with(['/', '#']), "{}: relative link {link}", page.display());
                let Some(rel) = link.strip_prefix(base) else {
                    continue;
                };
                let rel = rel.split('#').next().unwrap();
                // Raw blobs are served by the daemon straight from the repo.
                if rel.starts_with("raw/") {
                    continue;
                }
                let rel = percent_decode_str(rel).decode_utf8()?;
                let target = if rel.is_empty() || rel.ends_with('/') {
                    out.join(&*rel).join("index.html")
                } else {
                    out.join(&*rel)
                };
                assert!(
                    target.is_file(),
                    "{}: {link} -> missing {}",
                    page.display(),
                    target.display(),
                );
                checked += 1;
            }
        }
        assert!(checked > 0);
        Ok(())
    }
}