workerlib/
fetch.rs

1//!
2//! Fetch git sources into a workspace
3//!
4
5use crate::execute::{execute, Command, CommandBuilder, Error as ExecutionError};
6use crate::redact::redact;
7use crate::remote::Ref;
8use git2::{Error as GitError, FetchOptions, FetchPrune, Repository};
9use std::path::Path;
10
11const ALL_HEADS: &str = "+refs/heads/*:refs/heads/*";
12const ALL_TAGS: &str = "+refs/tags/*:refs/tags/*";
13
14/// Fetch all refs from the an upstream source into a file system directory
15struct FetchCommand<'a> {
16    pub url: &'a url::Url,
17
18    // Path to the global git configuration file
19    pub config_path: &'a Path,
20    pub ref_names: Option<&'a [Ref]>,
21}
22
23impl CommandBuilder for FetchCommand<'_> {
24    fn build(&self, current_dir: &Path) -> Command {
25        let mut cmd = Command::new("git");
26        cmd.current_dir(current_dir);
27        let mut args = vec![
28            String::from("--bare"),
29            String::from("fetch"),
30            String::from("--no-tags"),
31            // String::from("--prune"),
32            self.url.to_string(),
33        ];
34        cmd.envs([(
35            crate::git_config::GIT_CONFIG_GLOBAL,
36            self.config_path.to_string_lossy().as_ref(),
37        )]);
38
39        if let Some(ref_names) = self.ref_names {
40            args.extend(ref_names.iter().map(|remote_ref| remote_ref.git_ref_spec()));
41        } else {
42            args.extend(vec![String::from(ALL_HEADS), String::from(ALL_TAGS)])
43        }
44
45        cmd.args(args);
46        cmd
47    }
48}
49
50/// Fetch LFS objects from an upstream source into a bare mirror
51struct LfsFetchCommand<'a> {
52    pub url: &'a url::Url,
53
54    // Path to the global git configuration file
55    pub config_path: &'a Path,
56    pub ref_names: &'a [Ref],
57}
58
59impl CommandBuilder for LfsFetchCommand<'_> {
60    fn build(&self, current_dir: &Path) -> Command {
61        let mut cmd = Command::new("git");
62        cmd.current_dir(current_dir);
63        cmd.envs([(
64            crate::git_config::GIT_CONFIG_GLOBAL,
65            self.config_path.to_string_lossy().as_ref(),
66        )]);
67        cmd.args(["lfs", "fetch", self.url.as_str()]);
68        cmd.args(self.ref_names.iter().map(|r| r.0.as_str()));
69        cmd
70    }
71}
72
73/// An error occurred while fetching from the upstream repository
74#[derive(thiserror::Error, Debug)]
75pub enum Error {
76    /// Error fetching with git binary
77    #[error("Error fetching with the Git binary: {0}")]
78    Command(#[from] ExecutionError),
79
80    /// Error fetching with libgit2
81    #[error("Error fetching with Libgit2: {0}")]
82    Git(#[from] GitError),
83}
84
85/// Mirrors a git repo by pulling down the upstream. Depending on the
86/// configuration it will use libgit2 or the git binary
87pub struct Fetch<'a> {
88    /// The repository on disk to fetch into
89    pub git_repo: &'a crate::workspace::Workspace,
90    /// URL of the server being fetched from, must be a git compliant URL
91    pub target_url: &'a url::Url,
92    /// Whether to use git binary or libgit2 as the engine for fetch operations
93    pub use_git_binary: Option<bool>,
94    /// The path to Lorry's git configuration
95    pub git_config_path: &'a Path,
96    /// Refs to pull from
97    pub refs: Option<&'a [Ref]>,
98    /// Whether to also fetch LFS objects for the given refs from the target
99    pub lfs: bool,
100}
101
102impl Fetch<'_> {
103    async fn fetch_with_git(&self) -> Result<(), Error> {
104        tracing::debug!("running FetchCommand command");
105        execute(
106            &FetchCommand {
107                url: self.target_url,
108                config_path: self.git_config_path,
109                ref_names: self.refs,
110            },
111            &self.git_repo.repository_path(),
112        )
113        .await?;
114        Ok(())
115    }
116
117    async fn fetch_with_libgit2(&self) -> Result<(), Error> {
118        let git_repo_thread = self
119            .git_repo
120            .repository_path()
121            .to_string_lossy()
122            .to_string();
123        let upstream_url_thread = self.target_url.to_string();
124        let ref_names = if let Some(ref_names) = self.refs {
125            ref_names
126                .iter()
127                .map(|ref_name| ref_name.git_ref_spec())
128                .collect()
129        } else {
130            vec![String::from(ALL_HEADS), String::from(ALL_TAGS)]
131        };
132        let handle = tokio::task::spawn_blocking(move || {
133            let repository = Repository::open(git_repo_thread).unwrap();
134            let mut remote = repository.remote_anonymous(&upstream_url_thread).unwrap();
135            remote.fetch(
136                ref_names.as_slice(),
137                Some(
138                    FetchOptions::new()
139                        .prune(FetchPrune::On)
140                        .custom_headers(&[crate::LORRY_VERSION_HEADER]),
141                ),
142                None,
143            )
144        });
145        handle.await.unwrap()?;
146        Ok(())
147    }
148
149    async fn fetch_lfs(&self) -> Result<(), Error> {
150        let Some(refs) = self.refs else {
151            return Ok(());
152        };
153        tracing::info!("Fetching LFS objects from {}", redact(&self.target_url),);
154        execute(
155            &LfsFetchCommand {
156                url: self.target_url,
157                config_path: self.git_config_path,
158                ref_names: refs,
159            },
160            &self.git_repo.repository_path(),
161        )
162        .await?;
163        Ok(())
164    }
165
166    /// Fetch refs and git objects from a target repository
167    ///
168    /// Switches the engine used for fetch operations depending on arguments.
169    /// Git binary | Libgit2
170    ///
171    /// If `lfs` is set, also fetches LFS objects for the given refs.
172    pub async fn fetch(&self) -> Result<(), Error> {
173        if self
174            .use_git_binary
175            .is_some_and(|use_git_binary| use_git_binary)
176        {
177            tracing::info!(
178                "fetching repository {} with the git binary into {}",
179                redact(&self.target_url),
180                self.git_repo.repository_path().to_string_lossy(),
181            );
182            self.fetch_with_git().await
183        } else {
184            tracing::info!(
185                "fetching repository {} with libgit2 into {}",
186                redact(&self.target_url),
187                self.git_repo.repository_path().to_string_lossy(),
188            );
189            self.fetch_with_libgit2().await
190        }?;
191
192        if self.lfs {
193            self.fetch_lfs().await?;
194        }
195
196        Ok(())
197    }
198}
199
200#[cfg(test)]
201mod test {
202    use super::*;
203    use crate::git_config::Config;
204    use crate::test_server::{spawn_test_server, TestBuilder};
205
206    #[tokio::test]
207    async fn test_fetch_git_binary() {
208        let test = TestBuilder::default()
209            .git_config()
210            .test_repo("hello.git")
211            .workspace("test_fetch");
212        let test_repo = test.test_repo.unwrap();
213        let git_config_path = test.git_config.unwrap();
214        let repos_dir = test.dir.join("repos");
215        let address = spawn_test_server(repos_dir.as_path(), &[test_repo.clone()])
216            .await
217            .unwrap();
218        let fetch = Fetch {
219            git_repo: &test.workspace.unwrap(),
220            target_url: &test_repo.address(&address),
221            use_git_binary: Some(true),
222            git_config_path: git_config_path.as_path(),
223            refs: None,
224            lfs: false,
225        };
226        fetch.fetch().await.unwrap();
227    }
228
229    #[tokio::test]
230    async fn test_fetch_libgit2() {
231        let test = TestBuilder::default()
232            .test_repo("hello.git")
233            .workspace("test_fetch");
234        let test_repo = test.test_repo.unwrap();
235        let git_config_path = test.dir.join("gitconfig");
236        let git_config = Config(git_config_path.to_path_buf());
237        git_config
238            .setup(
239                ("Lorry", "hello@example.org"),
240                1,
241                false,
242                Some("HTTP/2"),
243                Path::new("/dev/null"),
244                None,
245                None,
246                None,
247                None,
248            )
249            .unwrap();
250        let repos_dir = test.dir.join("repos");
251        let address = spawn_test_server(repos_dir.as_path(), &[test_repo.clone()])
252            .await
253            .unwrap();
254        let fetch = Fetch {
255            git_repo: &test.workspace.unwrap(),
256            target_url: &test_repo.address(&address),
257            use_git_binary: Some(false),
258            git_config_path: git_config_path.as_path(),
259            refs: None,
260            lfs: false,
261        };
262        fetch.fetch().await.unwrap();
263    }
264
265    #[tokio::test]
266    async fn fetch_git_with_refs() {
267        let test = TestBuilder::default()
268            .test_repo("hello.git")
269            .workspace("test_fetch");
270        let test_repo = test.test_repo.unwrap();
271        let git_config_path = test.dir.join("gitconfig");
272        let git_config = Config(git_config_path.to_path_buf());
273        git_config
274            .setup(
275                ("Lorry", "hello@example.org"),
276                1,
277                false,
278                Some("HTTP/2"),
279                Path::new("/dev/null"),
280                None,
281                None,
282                None,
283                None,
284            )
285            .unwrap();
286        let repos_dir = test.dir.join("repos");
287        let address = spawn_test_server(repos_dir.as_path(), &[test_repo.clone()])
288            .await
289            .unwrap();
290        let refs = vec![Ref(String::from("refs/heads/main"))];
291        let fetch = Fetch {
292            git_repo: &test.workspace.unwrap(),
293            target_url: &test_repo.address(&address),
294            use_git_binary: Some(false),
295            git_config_path: git_config_path.as_path(),
296            refs: Some(refs.as_slice()),
297            lfs: false,
298        };
299        fetch.fetch().await.unwrap();
300    }
301}
workerlib/fetch.rs

workerlib/
fetch.rs
