workerlib/
lorry_specs.rs

1//!
2//! Representation of `.lorry` specification files
3//!
4
5use relative_path::RelativePathBuf;
6use serde::{Deserialize, Serialize};
7use std::collections::BTreeMap;
8use url::Url;
9
10/// The contents of a single .lorry configuration file
11#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
12#[serde(tag = "type")]
13#[serde(deny_unknown_fields)]
14pub enum LorrySpec {
15    /// Git mirror
16    #[serde(rename = "git")]
17    Git(SingleLorry),
18
19    /// Raw-file mirror
20    #[serde(rename = "raw-file")]
21    RawFiles(MultipleRawFiles),
22}
23
24/// Specification of a Git mirror
25///
26/// Explicitly a single lorry as git mirrors can only target a single
27/// repository.
28#[derive(Clone, Serialize, Deserialize, Debug, PartialEq)]
29#[serde(deny_unknown_fields)]
30pub struct SingleLorry {
31    /// Url of the target git repository
32    pub url: Url,
33    /// Patterns to match valid refs
34    pub ref_patterns: Option<Vec<String>>,
35    /// Refs to ignore
36    pub ignore_patterns: Option<Vec<String>>,
37    /// Enable large file storage
38    ///
39    /// TODO: Re-enable lfs for git mirrors
40    pub lfs: Option<bool>,
41}
42
43#[cfg(any(test, feature = "test"))]
44impl SingleLorry {
45    pub fn dummy() -> Self {
46        Self {
47            url: Url::parse("https://example-forge.com/repo.git").unwrap(),
48            ref_patterns: None,
49            ignore_patterns: None,
50            lfs: None,
51        }
52    }
53}
54
55/// Specification for a raw-file mirror
56///
57/// This can be a collection of raw files under the same identifier.
58#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
59#[serde(deny_unknown_fields)]
60pub struct MultipleRawFiles {
61    /// Target urls and sha256 for raw-files
62    pub urls: Vec<RawFileURLMapping>,
63}
64
65impl MultipleRawFiles {
66    /// Find the first missing sha256sum if any exist
67    pub fn missing_sha256sums(&self) -> Vec<String> {
68        self.urls
69            .iter()
70            .filter_map(|f| {
71                if f.sha256sum.is_none() {
72                    Some(f.url.to_string())
73                } else {
74                    None
75                }
76            })
77            .collect()
78    }
79}
80
81/// Specification for a raw-file
82#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
83pub struct RawFileURLMapping {
84    /// Destination on disk to store raw-file to
85    #[serde(default = "empty_relative_path")]
86    pub destination: RelativePathBuf,
87    /// Target url
88    pub url: url::Url,
89    /// SHA256 to verify that raw-file has imported successfully
90    pub sha256sum: Option<String>,
91}
92
93fn empty_relative_path() -> RelativePathBuf {
94    RelativePathBuf::new()
95}
96
97/// Attempts to convert a string-like, presumably the contents of a `.lorry` file, into the internal representation of a lorry spec.
98///
99/// This spec object can then be passed to `mirror`.
100pub fn extract_lorry_specs<S: AsRef<str>>(
101    s: S,
102) -> Result<BTreeMap<String, LorrySpec>, serde_yaml::Error> {
103    let lorries = serde_yaml::from_str(s.as_ref())?;
104    Ok(lorries)
105}
106
107#[test]
108fn lorries_extract() {
109    let lorry_dot_lorry = r"lorry:
110    type: git
111    url: https://gitlab.com/CodethinkLabs/lorry/lorry.git
112  ";
113
114    let extracted = extract_lorry_specs(lorry_dot_lorry).unwrap();
115
116    let mut correct: BTreeMap<String, LorrySpec> = BTreeMap::new();
117    correct.insert(
118        "lorry".to_string(),
119        LorrySpec::Git(SingleLorry {
120            url: "https://gitlab.com/CodethinkLabs/lorry/lorry.git"
121                .parse()
122                .unwrap(),
123            ref_patterns: None,
124            ignore_patterns: None,
125            lfs: None,
126        }),
127    );
128    assert_eq!(extracted, correct);
129}
130
131#[test]
132fn extract_multiple_lorries() {
133    let lorries_as_string = r"ARM-software/arm-trusted-firmware:
134    type: git
135    url: https://github.com/ARM-software/arm-trusted-firmware.git
136HarryMichal/go-version:
137    type: git
138    url: https://github.com/HarryMichal/go-version.git
139TelepathyIM/telepathy-glib:
140    type: git
141    url: https://github.com/TelepathyIM/telepathy-glib.git
142TelepathyIM/telepathy-logger:
143    type: git
144    url: https://github.com/TelepathyIM/telepathy-logger.git
145TelepathyIM/telepathy-mission-control:
146    type: git
147    url: https://github.com/TelepathyIM/telepathy-mission-control.git
148acobaugh/osrelease:
149    type: git
150    url: https://github.com/acobaugh/osrelease.git
151ayufan-rock64/pinebook-pro-keyboard-updater:
152    type: git
153    url: https://github.com/ayufan-rock64/pinebook-pro-keyboard-updater.git
154briandowns/spinner:
155    type: git
156    url: https://github.com/briandowns/spinner.git
157docker/go-units:
158    type: git
159    url: https://github.com/docker/go-units.git
160ebassi/graphene:
161    type: git
162    url: https://github.com/ebassi/graphene.git
163endlessm/eos-installer:
164    type: git
165    url: https://github.com/endlessm/eos-installer.git
166fatih/color:
167    type: git
168    url: https://github.com/fatih/color.git
169fsnotify/fsnotify:
170    type: git
171    url: https://github.com/fsnotify/fsnotify.git
172godbus/dbus:
173    type: git
174    url: https://github.com/godbus/dbus.git
175golang/crypto:
176    type: git
177    url: https://github.com/golang/crypto.git
178golang/sys:
179    type: git
180    url: https://github.com/golang/sys.git
181hughsie/appstream-glib:
182    type: git
183    url: https://github.com/hughsie/appstream-glib.git
184hughsie/libgusb:
185    type: git
186    url: https://github.com/hughsie/libgusb.git
187inconshreveable/mousetrap:
188    type: git
189    url: https://github.com/inconshreveable/mousetrap.git
190konsorten/go-windows-terminal-sequences:
191    type: git
192    url: https://github.com/konsorten/go-windows-terminal-sequences.git
193libhangul/libhangul:
194    type: git
195    url: https://github.com/libhangul/libhangul.git
196libpinyin/libpinyin:
197    type: git
198    url: https://github.com/libpinyin/libpinyin.git
199libsigcplusplus/libsigcplusplus:
200    type: git
201    url: https://github.com/libsigcplusplus/libsigcplusplus.git
202mattn/go-colorable:
203    type: git
204    url: https://github.com/mattn/go-colorable.git
205mattn/go-isatty:
206    type: git
207    url: https://github.com/mattn/go-isatty.git
208raspberrypi/userland:
209    type: git
210    url: https://github.com/raspberrypi/userland.git
211sirupsen/logrus:
212    type: git
213    url: https://github.com/sirupsen/logrus.git
214spf13/cobra:
215    type: git
216    url: https://github.com/spf13/cobra.git
217spf13/pflag:
218    type: git
219    url: https://github.com/spf13/pflag.git
220stevegrubb/libcap-ng:
221    type: git
222    url: https://github.com/stevegrubb/libcap-ng.git
223streambinder/vpnc:
224    type: git
225    url: https://github.com/streambinder/vpnc.git
226
227";
228
229    //example from https://gitlab.gnome.org/Infrastructure/Mirrors/mirroring-config/-/blob/main/github_com/github_com.lorry
230
231    let extracted = extract_lorry_specs(lorries_as_string).unwrap();
232
233    assert_eq!(extracted.len(), 31);
234}
235
236#[test]
237fn extract_raw_file_lorries() {
238    let lorries_as_string = "tar-files:
239    type: raw-file
240    urls:
241      - destination: debian/pool/main/a/anthy
242        url: http://http.debian.net/debian/pool/main/a/anthy/anthy_0.3.orig.tar.gz
243      - destination: debian/pool/main/d/db5.3
244        url: http://http.debian.net/debian/pool/main/d/db5.3/db5.3_5.3.28.orig.tar.xz
245  
246  
247  ";
248    //example from https://gitlab.gnome.org/Infrastructure/Mirrors/mirroring-config/-/blob/main/github_com/github_com.lorry
249
250    let extracted = extract_lorry_specs(lorries_as_string).unwrap();
251
252    assert_eq!(extracted.len(), 1);
253    assert_eq!(
254        match &extracted["tar-files"] {
255            LorrySpec::RawFiles(MultipleRawFiles { urls }) => urls.len(),
256            _ => panic!("non-rawfiles parsed as raw-files!"),
257        },
258        2
259    )
260}
workerlib/lorry_specs.rs

workerlib/
lorry_specs.rs
