workerlib/
lorry_specs.rs

1//!
2//! Representation of `.lorry` specification files
3//!
4
5use relative_path::RelativePathBuf;
6use serde::{Deserialize, Serialize};
7use std::collections::BTreeMap;
8use url::Url;
9
10/// The contents of a single .lorry configuration file
11#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
12#[serde(tag = "type")]
13#[serde(deny_unknown_fields)]
14pub enum LorrySpec {
15    /// Git mirror
16    #[serde(rename = "git")]
17    Git(SingleLorry),
18
19    /// Raw-file mirror
20    #[serde(rename = "raw-file")]
21    RawFiles(MultipleRawFiles),
22}
23
24/// Specification of a Git mirror
25///
26/// Explicitly a single lorry as git mirrors can only target a single
27/// repository.
28#[derive(Clone, Serialize, Deserialize, Debug, PartialEq)]
29#[serde(deny_unknown_fields)]
30pub struct SingleLorry {
31    /// Url of the target git repository
32    pub url: Url,
33    /// Patterns to match valid refs
34    pub ref_patterns: Option<Vec<String>>,
35    /// Refs to ignore
36    pub ignore_patterns: Option<Vec<String>>,
37    /// Enable large file storage
38    pub lfs: Option<bool>,
39}
40
41#[cfg(any(test, feature = "test"))]
42impl SingleLorry {
43    pub fn dummy() -> Self {
44        Self {
45            url: Url::parse("https://example-forge.com/repo.git").unwrap(),
46            ref_patterns: None,
47            ignore_patterns: None,
48            lfs: None,
49        }
50    }
51}
52
53/// Specification for a raw-file mirror
54///
55/// This can be a collection of raw files under the same identifier.
56#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
57#[serde(deny_unknown_fields)]
58pub struct MultipleRawFiles {
59    /// Target urls and sha256 for raw-files
60    pub urls: Vec<RawFileURLMapping>,
61}
62
63impl MultipleRawFiles {
64    /// Find the first missing sha256sum if any exist
65    pub fn missing_sha256sums(&self) -> Vec<String> {
66        self.urls
67            .iter()
68            .filter_map(|f| {
69                if f.sha256sum.is_none() {
70                    Some(f.url.to_string())
71                } else {
72                    None
73                }
74            })
75            .collect()
76    }
77}
78
79/// Specification for a raw-file
80#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
81pub struct RawFileURLMapping {
82    /// Destination on disk to store raw-file to
83    #[serde(default = "empty_relative_path")]
84    pub destination: RelativePathBuf,
85    /// Target url
86    pub url: url::Url,
87    /// SHA256 to verify that raw-file has imported successfully
88    pub sha256sum: Option<String>,
89}
90
91fn empty_relative_path() -> RelativePathBuf {
92    RelativePathBuf::new()
93}
94
95/// Attempts to convert a string-like, presumably the contents of a `.lorry` file, into the internal representation of a lorry spec.
96///
97/// This spec object can then be passed to `mirror`.
98pub fn extract_lorry_specs<S: AsRef<str>>(
99    s: S,
100) -> Result<BTreeMap<String, LorrySpec>, serde_yaml::Error> {
101    let lorries = serde_yaml::from_str(s.as_ref())?;
102    Ok(lorries)
103}
104
105#[test]
106fn lorries_extract() {
107    let lorry_dot_lorry = r"lorry:
108    type: git
109    url: https://gitlab.com/CodethinkLabs/lorry/lorry.git
110  ";
111
112    let extracted = extract_lorry_specs(lorry_dot_lorry).unwrap();
113
114    let mut correct: BTreeMap<String, LorrySpec> = BTreeMap::new();
115    correct.insert(
116        "lorry".to_string(),
117        LorrySpec::Git(SingleLorry {
118            url: "https://gitlab.com/CodethinkLabs/lorry/lorry.git"
119                .parse()
120                .unwrap(),
121            ref_patterns: None,
122            ignore_patterns: None,
123            lfs: None,
124        }),
125    );
126    assert_eq!(extracted, correct);
127}
128
129#[test]
130fn extract_multiple_lorries() {
131    let lorries_as_string = r"ARM-software/arm-trusted-firmware:
132    type: git
133    url: https://github.com/ARM-software/arm-trusted-firmware.git
134HarryMichal/go-version:
135    type: git
136    url: https://github.com/HarryMichal/go-version.git
137TelepathyIM/telepathy-glib:
138    type: git
139    url: https://github.com/TelepathyIM/telepathy-glib.git
140TelepathyIM/telepathy-logger:
141    type: git
142    url: https://github.com/TelepathyIM/telepathy-logger.git
143TelepathyIM/telepathy-mission-control:
144    type: git
145    url: https://github.com/TelepathyIM/telepathy-mission-control.git
146acobaugh/osrelease:
147    type: git
148    url: https://github.com/acobaugh/osrelease.git
149ayufan-rock64/pinebook-pro-keyboard-updater:
150    type: git
151    url: https://github.com/ayufan-rock64/pinebook-pro-keyboard-updater.git
152briandowns/spinner:
153    type: git
154    url: https://github.com/briandowns/spinner.git
155docker/go-units:
156    type: git
157    url: https://github.com/docker/go-units.git
158ebassi/graphene:
159    type: git
160    url: https://github.com/ebassi/graphene.git
161endlessm/eos-installer:
162    type: git
163    url: https://github.com/endlessm/eos-installer.git
164fatih/color:
165    type: git
166    url: https://github.com/fatih/color.git
167fsnotify/fsnotify:
168    type: git
169    url: https://github.com/fsnotify/fsnotify.git
170godbus/dbus:
171    type: git
172    url: https://github.com/godbus/dbus.git
173golang/crypto:
174    type: git
175    url: https://github.com/golang/crypto.git
176golang/sys:
177    type: git
178    url: https://github.com/golang/sys.git
179hughsie/appstream-glib:
180    type: git
181    url: https://github.com/hughsie/appstream-glib.git
182hughsie/libgusb:
183    type: git
184    url: https://github.com/hughsie/libgusb.git
185inconshreveable/mousetrap:
186    type: git
187    url: https://github.com/inconshreveable/mousetrap.git
188konsorten/go-windows-terminal-sequences:
189    type: git
190    url: https://github.com/konsorten/go-windows-terminal-sequences.git
191libhangul/libhangul:
192    type: git
193    url: https://github.com/libhangul/libhangul.git
194libpinyin/libpinyin:
195    type: git
196    url: https://github.com/libpinyin/libpinyin.git
197libsigcplusplus/libsigcplusplus:
198    type: git
199    url: https://github.com/libsigcplusplus/libsigcplusplus.git
200mattn/go-colorable:
201    type: git
202    url: https://github.com/mattn/go-colorable.git
203mattn/go-isatty:
204    type: git
205    url: https://github.com/mattn/go-isatty.git
206raspberrypi/userland:
207    type: git
208    url: https://github.com/raspberrypi/userland.git
209sirupsen/logrus:
210    type: git
211    url: https://github.com/sirupsen/logrus.git
212spf13/cobra:
213    type: git
214    url: https://github.com/spf13/cobra.git
215spf13/pflag:
216    type: git
217    url: https://github.com/spf13/pflag.git
218stevegrubb/libcap-ng:
219    type: git
220    url: https://github.com/stevegrubb/libcap-ng.git
221streambinder/vpnc:
222    type: git
223    url: https://github.com/streambinder/vpnc.git
224
225";
226
227    //example from https://gitlab.gnome.org/Infrastructure/Mirrors/mirroring-config/-/blob/main/github_com/github_com.lorry
228
229    let extracted = extract_lorry_specs(lorries_as_string).unwrap();
230
231    assert_eq!(extracted.len(), 31);
232}
233
234#[test]
235fn extract_raw_file_lorries() {
236    let lorries_as_string = "tar-files:
237    type: raw-file
238    urls:
239      - destination: debian/pool/main/a/anthy
240        url: http://http.debian.net/debian/pool/main/a/anthy/anthy_0.3.orig.tar.gz
241      - destination: debian/pool/main/d/db5.3
242        url: http://http.debian.net/debian/pool/main/d/db5.3/db5.3_5.3.28.orig.tar.xz
243  
244  
245  ";
246    //example from https://gitlab.gnome.org/Infrastructure/Mirrors/mirroring-config/-/blob/main/github_com/github_com.lorry
247
248    let extracted = extract_lorry_specs(lorries_as_string).unwrap();
249
250    assert_eq!(extracted.len(), 1);
251    assert_eq!(
252        match &extracted["tar-files"] {
253            LorrySpec::RawFiles(MultipleRawFiles { urls }) => urls.len(),
254            _ => panic!("non-rawfiles parsed as raw-files!"),
255        },
256        2
257    )
258}
workerlib/lorry_specs.rs

workerlib/
lorry_specs.rs
