workerlib/

lib.rs

//!
//! This is the workerlib crate
//!

mod arguments;
mod check_lfs_refs;
mod download;
pub mod fetch;
mod get_head;
mod importer;
pub mod local;
mod ls_remote;
mod push;
mod raw_files;
pub mod redact;
mod remote;

#[cfg(test)]
mod test_server;

pub mod execute;
pub mod git_config;
pub mod git_fsck;
pub mod git_gc;
pub mod lorry_specs;
pub mod workspace;

use crate::execute::{execute, Error as ExecutionError};
use git2::{Error as Libgit2Error, Repository};
use git_config::DEFAULT_GIT_BRANCH;
use glob::Pattern;
use lorry_specs::SingleLorry;
use remote::Ref;
use remote::Remote;
use serde::Deserialize;
use std::collections::BTreeMap;
use thiserror::Error;
use url::Url;

pub use arguments::Arguments;
pub use lorry_specs::extract_lorry_specs;
pub use lorry_specs::LorrySpec;
pub use push::RefStatus;

/// Extra header exposed on Git fetch uperations to upstream servers and
/// push operations on downstream.
pub const LORRY_VERSION_HEADER: &str = concat!("Lorry-Version: ", env!("CARGO_PKG_VERSION"));

/// Default branch if HEAD is missing or unresolvable from the target repository.
pub const DEFAULT_BRANCH_NAME: &str = "main";

/// Default ref is missing from the target repository.
pub const DEFAULT_REF_NAME: &str = "refs/heads/main";

#[derive(Debug, Clone, Default, Deserialize)]
pub struct WarningConfig {
    #[serde(default)]
    pub skip_non_matching_ref_specs: bool,
    #[serde(default)]
    pub skip_only_ref_tags: bool,
    #[serde(default)]
    pub skip_lfs_refs_skipped: bool,
}

/// Non-fatal error encountered when running a Lorry.
#[derive(Clone, Debug)]
pub enum Warning {
    /// Anytime a refspec is specified but it doesn't match any refs in the
    /// upstream repository.
    NonMatchingRefSpecs {
        /// Pattern from the configuration that did not match any refs
        pattern: String,
    },
    /// If only tags have been pulled from upstream, the repo will look broken
    /// so we display a warning.
    OnlyRefTags,
    /// If there are lfs refs in upstream but the setting for them is lfs: false
    /// the mirror would still succeed but we show a warning
    LFSRefsSkipped,
}
// TODO: Perhaps add a feature to the web ui to suppress warnings.

impl Warning {
    /// Return a nicely formatted name
    pub fn name(&self) -> String {
        match self {
            Warning::NonMatchingRefSpecs { pattern: _ } => String::from("NonMatchingRefSpec"),
            Warning::OnlyRefTags => String::from("OnlyRefTags"),
            Warning::LFSRefsSkipped => String::from("LFSRefsSkipped"),
        }
    }

    /// Return the body of the error message
    pub fn message(&self) -> String {
        match self {
            Warning::NonMatchingRefSpecs { pattern } => pattern.clone(),
            Warning::OnlyRefTags => String::from(
                "Only tags have been pulled from upstream, repository will appear empty",
            ),
            Warning::LFSRefsSkipped => String::from(
                "Upstream contains LFS refs, but lfs setting for the mirror has been set to false",
            ),
        }
    }
}

/// Results of a parsing refs from porcelain git push output
///
/// Example command:
/// ```txt
/// git push --porcelain origin 'refs/heads/*' 2>/dev/null`
/// ```
#[derive(Clone, Debug, Default)]
pub struct PushRefs(pub Vec<(String, RefStatus)>);

/// Contains the status of a single mirror operation
#[derive(Default)]
pub struct MirrorStatus {
    /// Refs we have attempted to push to upstream.
    pub push_refs: PushRefs,
    /// Warnings for display on Lorry's Web UI e.g. NoMatchingRefSpecs.
    pub warnings: Vec<Warning>,
}

/// Top-level error type containing all possible errors during a mirroring operation.
#[derive(Error, Debug)]
pub enum Error {
    /// The upstream was a git repo. There was an error pulling from the remote.
    #[error("Failed to fetch upstream: {0}")]
    Fetch(#[from] fetch::Error),

    /// Indicates that the job exceeded the time it was allocated in the
    /// Lorry configuration
    #[error("job exceeded the allocated timeout: {seconds}")]
    TimeoutExceeded {
        /// Display elapsed time
        seconds: u64,
    },

    /// An error occurred that originated in the workspace module.
    #[error("Workspace Error: {0}")]
    WorkspaceError(#[from] workspace::Error),

    /// An error occured when importing raw files into a working directory.
    #[error("LFS Importer Error: {0}")]
    LFSImporter(#[from] importer::Error),

    /// Generic command execution error not specific to any particular module.
    #[error("Command Execution Error: {0}")]
    Command(#[from] ExecutionError),

    /// Generic IO error
    #[error("IO Error")]
    Io {
        /// Display command
        command: String,
    },
    /// Tried to access a repository on the local filesystem as the downstream,
    /// but the given path is not a valid one.
    #[error(
        "Downstream is specified as a path on the local filesystem, but the path is malformed."
    )]
    InvalidPath(String),

    /// Problem parsing a downstream git url
    #[error("Failed to construct path to downstream git repo")]
    ParseError(#[from] url::ParseError),

    /// Generic Libgit2 failure not specific to any particular module.
    #[error("Internal git failure: {0}")]
    Libgit2Error(#[from] Libgit2Error),

    /// Indicates that 100% of the desired refs failed to push to the
    /// downstream.
    #[error("All ({n_attempted}) refspecs failed")]
    AllRefspecsFailed {
        /// Display refs
        refs: PushRefs,
        /// Number of attempted refs
        n_attempted: i64,
    },

    /// Indicates some but not all of the desired refspecs were not able to
    /// be pushed downstream for some reason.
    #[error("{n_failed} refspecs failed")]
    SomeRefspecsFailed {
        /// Refs pushed to upstream
        refs: PushRefs,
        /// Number of failed refs
        n_failed: i64,
    },

    /// Indicates a refspec was malformed
    #[error("Invalid Glob: {pattern} - {error}")]
    InvalidGlobPattern {
        /// Invalid pattern
        pattern: String,
        /// Reason for refspec malformity
        error: String,
    },

    /// An error related to raw-file management occurred.
    #[error("Raw File Related Error: {0}")]
    RawFiles(#[from] raw_files::Error),

    /// Indicates none of the refs in the mirror configuration matches those
    /// that were available in the repository.
    #[error("No Matching Refspecs")]
    NoMatchingRefspecs,

    /// Indicates that a push command failed and we could not understand the
    /// the reason why. This may be due to a downstream server error.
    #[error("Cannot parse push output: {0}")]
    CannotParsePushOutput(String),

    /// Indicates that the server requires Sha256sum values for all aw-files
    /// but the file did not have one specified.
    #[error("Sha256sums are missing: \n{0}")]
    Sha256sumsNotSpecified(String),

    /// Indicates the ignore pattern was not a valid regular expression
    #[error("Invalid ignore pattern: {0}")]
    InvalidIgnorePattern(String),

    /// Raw file mirrors are not supported
    #[error("Raw file mirror skipped: {0}")]
    RawFileNotSupported(String),

    /// An operation has failed on a remote
    #[error("Remote failed: {0}")]
    Remote(#[from] remote::Error),
}

impl Error {
    /// Return the underlying status code from the error if the error is the
    /// result of an command execution failure otherwise return nothing.
    #[allow(clippy::collapsible_match)]
    pub fn status(&self) -> Option<i32> {
        match self {
            Error::Fetch(e) => match e {
                fetch::Error::Command(e) => e.status(),
                _ => None,
            },
            Error::Command(e) => match e {
                ExecutionError::IO {
                    command: _,
                    source: _,
                } => None,
                ExecutionError::CommandError {
                    command: _,
                    status,
                    stderr: _,
                    stdout: _,
                } => Some(status.code().unwrap_or(-1)),
            },
            _ => None,
        }
    }
}

/// Fetch [PushRefs] from the result of a git push operation
///
/// If there are any failed refs, an error is recorded - otherwise return
/// the push refs from the result of parsing stdout.
fn get_refs(n_refs: usize, stdout: &str) -> Result<PushRefs, Error> {
    match crate::push::Push::parse_output(stdout) {
        Ok(results) => {
            let n_failed = results
                .0
                .iter()
                .filter(|r| matches!(r.1, RefStatus::Rejected))
                .count();
            if n_failed == n_refs {
                return Err(Error::AllRefspecsFailed {
                    refs: results.clone(),
                    n_attempted: n_refs as i64,
                });
            }
            if n_failed > 0 {
                Err(Error::SomeRefspecsFailed {
                    refs: results.clone(),
                    n_failed: n_failed as i64,
                })
            } else {
                Ok(results)
            }
        }
        Err(message) => {
            tracing::warn!("Failed to parse git push output:\n{}", message);
            Err(Error::CannotParsePushOutput(message))
        }
    }
}

/// Match refs ignoring refs/{heads,tags}/ unless they're explicitly specified
fn match_ref(pattern: &Pattern, input: &str) -> bool {
    let pattern_str = pattern.as_str();
    if pattern_str.starts_with("refs/heads/") || pattern_str.starts_with("refs/tags/") {
        pattern.matches(input)
    } else {
        pattern.matches(
            input
                .trim_start_matches("refs/heads/")
                .trim_start_matches("refs/tags/"),
        )
    }
}

/// Given all of the refs return a vec of those with some matches and a vec of
/// any patterns that didn't result in any matches at all.
///
/// # Example
/// ```yaml
///
/// ref-patterns:
/// - 'ma*'
/// - 'refs/tags/*'
/// ignore-patterns:
/// - 'refs/tags/*rc*
/// ```
///
/// -> Parses the ignore-patterns and ref-patterns into globs
/// * Matches refs against ignore patterns, removing all matching refs
/// * Matches valid against ref-patterns and returns a vector of matching refs.
fn parse_refs(
    refs: &[Ref],
    ref_patterns: Option<&[String]>,
    ignore_patterns: Option<&[String]>,
) -> Result<(Vec<Ref>, Vec<Ref>), Error> {
    let ignore_globs = ignore_patterns
        .as_ref()
        .map(|ignore_patterns| {
            ignore_patterns
                .iter()
                .try_fold(Vec::new(), |mut accm, pattern| {
                    let glob_pattern =
                        Pattern::new(pattern).map_err(|e| Error::InvalidGlobPattern {
                            pattern: pattern.clone(),
                            error: e.to_string(),
                        })?;
                    accm.push(glob_pattern);
                    Ok::<Vec<Pattern>, Error>(accm)
                })
        })
        .transpose()?;

    let refs: Vec<Ref> = refs
        .iter()
        .filter_map(|ref_name| {
            if ignore_globs.as_ref().is_some_and(|ignore_globs| {
                ignore_globs
                    .iter()
                    .any(|pattern| match_ref(pattern, &ref_name.0))
            }) {
                None
            } else {
                Some(ref_name.clone())
            }
        })
        .collect();

    if let Some(ref_specs) = ref_patterns {
        let mut patterns_with_matches: BTreeMap<String, bool> = BTreeMap::new();
        let globs: Vec<Pattern> = ref_specs.iter().try_fold(Vec::new(), |mut accm, pattern| {
            match Pattern::new(pattern) {
                Ok(regex) => {
                    patterns_with_matches.insert(pattern.clone(), false);
                    accm.push(regex);
                    Ok(accm)
                }
                Err(e) => Err(Error::InvalidGlobPattern {
                    pattern: pattern.clone(),
                    error: e.to_string(),
                }),
            }
        })?;

        Ok((
            refs.iter().fold(Vec::new(), |mut accm, ref_name| {
                if let Some(matching_glob) = globs.iter().find_map(|glob_pattern| {
                    if match_ref(glob_pattern, &ref_name.0) {
                        Some(glob_pattern.to_string())
                    } else {
                        None
                    }
                }) {
                    patterns_with_matches.insert(matching_glob, true);
                    accm.push(ref_name.clone());
                };
                accm
            }),
            patterns_with_matches
                .iter()
                .filter_map(|(pattern, had_match)| {
                    if !had_match {
                        Some(Ref(pattern.clone()))
                    } else {
                        None
                    }
                })
                .collect(),
        ))
    } else {
        Ok((refs.to_vec(), vec![]))
    }
}

/// Push a git based mirror to the downstream server
///
/// * If the downstream host is local, the default branch is set to match the
///   HEAD of the associated workspace.
///
/// * refs on `repository` are parsed against the Lorry spec - valid refs are
///   then pushed to the Lorry mirror server.
///
/// TODO: Add option to use libgit2 for push in addition to pull on git mirrors.
async fn push_to_mirror_server(
    lorry_spec: &SingleLorry,
    lorry_name: &str,
    url_to_push_to: &Url,
    active_repo: &workspace::Workspace,
    arguments: &arguments::Arguments,
) -> Result<MirrorStatus, Error> {
    tracing::debug!("Pushing {} to mirror at {:?}", lorry_name, &url_to_push_to);
    let mut warnings = vec![];

    // The workspace on disk that Lorry stores the result of fetch operation to
    let repository = Repository::open(active_repo.repository_path())?;
    // If we are pushing to a local downstream
    if url_to_push_to.scheme() == "file" {
        // This repo is being pushed to the local filesystem
        let file = url_to_push_to
            .to_file_path()
            .map_err(|_| Error::InvalidPath(url_to_push_to.to_string()))?;

        let local_repo = Repository::open(file)?;
        let head = repository.head()?;
        // Set remote HEAD on local mirror to match workspace.
        local_repo.set_head(
            head.name()
                .unwrap_or(&format!("refs/heads/{DEFAULT_GIT_BRANCH}")),
        )?;
    }
    let ref_names = repository
        .references()?
        .try_fold(Vec::new(), |mut accm, reference| {
            let ref_name = match reference {
                Ok(ref_name) => {
                    let name = ref_name.name().unwrap();
                    name.to_string()
                }
                Err(err) => return Err(err),
            };
            accm.push(Ref(ref_name));
            Ok(accm)
        })?;

    let (refs, missing) = parse_refs(
        ref_names.as_slice(),
        lorry_spec.ref_patterns.as_deref(),
        lorry_spec.ignore_patterns.as_deref(),
    )?;
    tracing::info!("pushing {} refs", refs.len());

    if refs.is_empty() {
        return Err(Error::NoMatchingRefspecs);
    }

    let repository_path = active_repo.repository_path();

    // Compute the LFS endpoint URL for the downstream, if LFS is enabled.
    // The /info/lfs path convention is defined by the Git LFS API spec:
    // https://github.com/git-lfs/git-lfs/blob/main/docs/api/README.md
    let lfs_url = if lorry_spec.lfs == Some(true) {
        let mut url = url_to_push_to.clone();
        url.set_path(&format!("{}/info/lfs", url.path()));
        Some(url)
    } else {
        None
    };

    if !arguments.disable_warnings.skip_lfs_refs_skipped {
        let check_lfs_refs_command = crate::check_lfs_refs::CheckLFSRefs {
            config_path: &arguments.git_config_path,
        };

        tracing::debug!("running CheckLFS command");
        let lfs_refs_present =
            match execute(&check_lfs_refs_command, repository_path.as_path()).await {
                Ok((stdout, _)) => crate::check_lfs_refs::CheckLFSRefs::parse_output(&stdout),
                Err(err) => {
                    tracing::warn!("CheckLFS command failed: {:?}", err);
                    false
                }
            };

        if lfs_refs_present && !matches!(lorry_spec.lfs, Some(true)) {
            warnings.push(Warning::LFSRefsSkipped)
        }
    }

    let push = &crate::push::Push {
        url: url_to_push_to,
        ref_names: refs.iter().map(|s| s.0.as_str()).collect(),
        config_path: &arguments.git_config_path,
        lfs_url: lfs_url.as_ref(),
    };

    tracing::debug!("running Push command");
    let push_refs = match execute(push, repository_path.as_path()).await {
        Ok((stdout, _)) => get_refs(refs.len(), &stdout),
        Err(err) => match err {
            ExecutionError::IO { command, source } => {
                tracing::warn!("Command failed to spawn: {:?}", source.to_string());
                Err(Error::Io {
                    command: command.clone(),
                })
            }
            ExecutionError::CommandError {
                command: _,
                status: _,
                stderr: _,
                stdout,
            } => get_refs(refs.len(), &stdout),
        },
    }?;

    Ok(MirrorStatus {
        push_refs,
        warnings: {
            if !arguments.disable_warnings.skip_only_ref_tags
                && !refs.iter().any(|refs| refs.0.contains("refs/heads"))
            {
                warnings.push(Warning::OnlyRefTags)
            }
            if !arguments.disable_warnings.skip_non_matching_ref_specs {
                warnings.extend(
                    missing
                        .iter()
                        .map(|pattern| Warning::NonMatchingRefSpecs {
                            pattern: pattern.0.to_string(),
                        })
                        .collect::<Vec<Warning>>(),
                );
            }
            warnings
        },
    })
}

/// Attempt to mirror a repository from an upstream host into the configured
/// downstream server.
///
/// Rough outline of the order of operations is:
///
/// * Attempt to fetch repository data from the downstream into the working
///   directory of the local mirror.
///
/// * Run git remote-ls on upstream to find a list of refs in the upstream.
///
/// * Fetch the upstream repository on top of the working directory of the
///   local mirror applying new updates.
///
/// * Push the local mirror back into the downstream updating it with new refs.
///
/// Fetching the downstream repository initially ensures that Lorry can work
/// in a stateless environment and will consider the downstream mirror the
/// source of truth.
///
/// This workflow applies to both LFS and normal Git mirrors. In the case of
/// LFS the raw files are also downloaded into the working directory prior to
/// running download operations.
///
/// In the event that the downstream repository becomes corrupted in someway
/// the procedure is to delete and reinitialize an empty repository in which
/// case the upstream mirror will be re-imported from scratch.
///
/// TODO: This code can be factored out better
pub async fn try_mirror(
    lorry_details: &LorrySpec,
    lorry_name: &str,
    downstream_url: &url::Url,
    workspace: &workspace::Workspace,
    arguments: &Arguments,
) -> Result<MirrorStatus, Error> {
    match lorry_details {
        LorrySpec::Git(single_lorry) => {
            tracing::info!("Ensuring local mirror is consistent with downstream");
            crate::fetch::Fetch {
                git_repo: workspace,
                target_url: downstream_url,
                use_git_binary: arguments.use_git_binary,
                git_config_path: arguments.git_config_path.as_path(),
                // all refs considered in downstream
                refs: None,
                lfs: false,
            }
            .fetch()
            .await?;
            let target_url = single_lorry.url.clone();
            let remote = Remote {
                workspace,
                use_git_binary: arguments.use_git_binary,
                git_config_path: arguments.git_config_path.as_path(),
            };
            tracing::info!("Matching ref-patterns with refs on upstream");
            let refs = remote.list(&target_url).await?;

            tracing::info!("Ensuring local mirror default branch is consistent with upstream");
            remote.set_head(&target_url).await?;

            tracing::info!("Fetching upstream repository into local mirror");
            // match upstream's remote ls response but do not error yet since
            // although the Lorry configuration may ask for refs which are
            // missing, we still want to pull and push what is available into
            // our downstream. push_to_mirror_server will flag any errors.
            let (matches, _) = parse_refs(
                refs.0.as_slice(),
                single_lorry.ref_patterns.as_deref(),
                single_lorry.ignore_patterns.as_deref(),
            )?;
            tracing::info!("Upstream contains {} matching refs", matches.len());
            crate::fetch::Fetch {
                git_repo: workspace,
                target_url: &target_url,
                use_git_binary: arguments.use_git_binary,
                git_config_path: arguments.git_config_path.as_path(),
                refs: Some(matches.as_slice()),
                lfs: single_lorry.lfs == Some(true),
            }
            .fetch()
            .await?;

            push_to_mirror_server(
                single_lorry,
                lorry_name,
                downstream_url,
                workspace,
                arguments,
            )
            .await
        }
        LorrySpec::RawFiles(raw_files) => {
            if matches!(downstream_url.scheme(), "file") {
                tracing::warn!("Raw file mirrors are not supported for local downstream");
                return Err(Error::RawFileNotSupported(lorry_name.to_string()));
            }
            // check for missing sha256sums if configuration disallows them
            if arguments.sha256sum_required {
                let missing_sha256sums = raw_files.missing_sha256sums();
                if !missing_sha256sums.is_empty() {
                    let mut message = String::default();
                    missing_sha256sums
                        .iter()
                        .for_each(|url| message.push_str(&format!("{url}\n")));
                    return Err(Error::Sha256sumsNotSpecified(message));
                }
            }
            tracing::info!("Fetching raw files from downstream to ensure consistency");
            let mut lfs_url = downstream_url.clone();
            lfs_url.set_path(&format!("{}/info/lfs", lfs_url.path()));
            tracing::debug!("running FetchDownstreamRawFiles command");
            let fetch_err = execute(
                &raw_files::FetchDownstreamRawFiles {
                    url: downstream_url,
                    lfs_url: &lfs_url,
                    worktree: workspace.lfs_data_path().as_path(),
                    config_path: arguments.git_config_path.as_path(),
                },
                &workspace.repository_path(),
            )
            .await;
            // TODO: I would prefer to explicitly detect if the repository
            // exists or not but the code calling the Gitlab API needs work.
            // BUG: Be aware that if the downstream fails for some other reason
            // like the server being down that this will create a new
            // repository and push its contents up causing a rejection error.
            // If this happens then you need to manually delete the mirror
            // directory and on the next run it will properly import the
            // downstream sources.
            if let Err(err) = fetch_err {
                tracing::warn!("Fetch failed but might not be an error: {}", err);
            } else {
                tracing::info!("Local mirror is consistent with downstream");
            }
            let importer = importer::Importer(raw_files.clone());
            let helper = raw_files::Helper(workspace.clone());

            // Initialize the LFS repository if it hasn't been already creating
            // the first commit which enables tracking of everything within.
            helper
                .initial_commit_if_missing(arguments.git_config_path.as_path())
                .await?;

            // Download any new raw files
            let modified = importer
                .ensure(&workspace.lfs_data_path(), arguments)
                .await?;
            if modified {
                // Import new data that was just downloaded
                helper
                    .import_data(arguments.git_config_path.as_path())
                    .await?;
            } else {
                tracing::info!("No new files were added or removed, nothing to do")
            };

            tracing::info!("Synchronizing local raw-file mirror to downstream");
            let mut lfs_url = downstream_url.clone();
            lfs_url.set_path(&format!("{}/info/lfs", lfs_url.path()));
            tracing::debug!("running PushRawFiles command");
            execute(
                &raw_files::PushRawFiles {
                    url: downstream_url,
                    lfs_url: &lfs_url,
                    worktree: workspace.lfs_data_path().as_path(),
                    config_path: arguments.git_config_path.as_path(),
                },
                workspace.repository_path().as_path(),
            )
            .await?;
            // LFS breaks porcelain git output so we just fake it since we only
            // supporting syncing to a single branch anyway.
            Ok(MirrorStatus {
                push_refs: PushRefs(vec![(DEFAULT_BRANCH_NAME.to_string(), RefStatus::NoPush)]),
                warnings: vec![],
            })
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;

    use git2::Repository;

    use crate::local::LocalRepositoryBuilder;
    use crate::test_server::{spawn_test_server, TestBuilder, TestRepo};

    #[test]
    fn test_parse_refs() {
        let (refs, missing) = parse_refs(
            &[
                Ref::from("refs/heads/master"),
                Ref::from("refs/tags/v1.0.0"),
                Ref::from("refs/tags/v1.0.1"),
                Ref::from("refs/tags/v1.0.2-rc1"),
                Ref::from("some-random-string"),
            ],
            Some(&[
                String::from("refs/heads/ma*"),
                String::from("v1.0.1"),
                String::from("refs/tags/v*"),
                String::from("notgonnamatch"),
            ]),
            // ignore RC releases
            Some(&[
                String::from("refs/tags/*-rc*"),
                String::from("*-rc*"),
                String::from("*ef*"), // Will not match refs/...
            ]),
        )
        .unwrap();
        assert!(refs.iter().any(|key| *key == "refs/heads/master".into()));
        assert!(refs.iter().any(|key| *key == "refs/tags/v1.0.0".into()));
        assert!(refs.iter().any(|key| *key == "refs/tags/v1.0.1".into()));
        assert!(!refs.iter().any(|key| *key == "some-random-string".into()));
        assert!(!refs.iter().any(|key| *key == "refs/tags/v1.0.2-rc1".into()));
        assert!(*missing.first().unwrap() == Ref::from("notgonnamatch"));
        assert!(missing.len() == 1);
    }

    #[test]
    fn test_parse_refs_conflict() {
        let (refs, missing) = parse_refs(
            &[
                Ref::from("refs/heads/master"),
                Ref::from("refs/heads/v1.0.0"),
                Ref::from("refs/tags/v1.0.0"),
            ],
            Some(&[String::from("v1.0.0")]),
            Some(&[String::from("master")]), // pointless
        )
        .unwrap();
        assert!(refs.len() == 2);
        assert!(refs.iter().any(|key| *key == "refs/tags/v1.0.0".into()));
        assert!(refs.iter().any(|key| *key == "refs/heads/v1.0.0".into()));
        assert!(missing.is_empty());
    }

    #[tokio::test]
    async fn test_try_mirror() {
        let upstream = TestBuilder::default().test_repo("hello.git");
        let upstream_repo = upstream.test_repo.unwrap();

        let downstream = TestBuilder::default().git_config().workspace("test-repo");
        let downstream_repo = TestRepo((String::from("hello.git"), vec![]));
        let repos_upstream_dir = upstream.dir.join("repos_upstream");
        let repos_downstream_dir = downstream.dir.join("repos_downstream");
        // Upstream server
        let upstream_address =
            spawn_test_server(repos_upstream_dir.as_path(), &[upstream_repo.clone()])
                .await
                .unwrap();
        // Downstream server
        let downstream_address =
            spawn_test_server(repos_downstream_dir.as_path(), &[downstream_repo.clone()])
                .await
                .unwrap();

        try_mirror(
            &LorrySpec::Git(SingleLorry {
                url: upstream_repo.address(&upstream_address),
                ref_patterns: None,
                ignore_patterns: None,
                lfs: None,
            }),
            "test_lorry",
            &downstream_repo.address(&downstream_address),
            &downstream.workspace.unwrap(),
            &Arguments {
                working_area: downstream.dir,
                use_git_binary: Some(true),
                git_config_path: downstream.git_config.unwrap(),
                ..Default::default()
            },
        )
        .await
        .unwrap();

        // ensure that downstream contains the git from the upstream
        let repository = Repository::open_bare(repos_downstream_dir.join("hello.git")).unwrap();
        let mut walk = repository.revwalk().unwrap();
        walk.push_head().unwrap();
        let last_commit_id = walk.next().unwrap().unwrap();
        let last_commit = repository.find_commit(last_commit_id).unwrap();
        let last_commit_message = last_commit.message().unwrap();
        assert!(last_commit_message == "Test Commit: 1/1")
    }

    #[tokio::test]
    async fn test_try_local_mirror() {
        let upstream = TestBuilder::default().test_repo("hello.git");
        let upstream_repo = upstream.test_repo.unwrap();

        let downstream = TestBuilder::default().git_config().workspace("test-repo");
        let downstream_repo = downstream.dir.join("mirrors");

        LocalRepositoryBuilder::new(&downstream_repo)
            .build("test")
            .unwrap();

        let repos_upstream_dir = upstream.dir.join("repos_upstream");
        // Upstream server
        let upstream_address =
            spawn_test_server(repos_upstream_dir.as_path(), &[upstream_repo.clone()])
                .await
                .unwrap();

        // Expect to mirror ref/heads/<DEFAULT_GIT_BRANCH_NAME>
        try_mirror(
            &LorrySpec::Git(SingleLorry {
                url: upstream_repo.address(&upstream_address),
                ref_patterns: None,
                ignore_patterns: None,
                lfs: None,
            }),
            "test_lorry",
            &Url::from_file_path(downstream_repo.join("test/git-repository")).unwrap(),
            &downstream.workspace.unwrap(),
            &Arguments {
                working_area: downstream.dir,
                use_git_binary: Some(true),
                git_config_path: downstream.git_config.unwrap(),
                ..Default::default()
            },
        )
        .await
        .unwrap();
    }
}