workerlib/
lorry_specs.rs

1//!
2//! Representation of `.lorry` specification files
3//!
4
5use relative_path::RelativePathBuf;
6use serde::{Deserialize, Serialize};
7use std::collections::BTreeMap;
8
9/// The contents of a single .lorry configuration file
10#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
11#[serde(tag = "type")]
12pub enum LorrySpec {
13    /// Git mirror
14    #[serde(rename = "git")]
15    Git(SingleLorry),
16
17    /// Raw-file mirror
18    #[serde(rename = "raw-file")]
19    RawFiles(MultipleRawFiles),
20}
21
22/// Settings for git cleanup operations
23#[derive(Clone, Default, Serialize, Deserialize, Debug, PartialEq)]
24pub struct GCOptions {
25    /// Enable gc
26    pub enabled: bool,
27
28    /// TODO
29    pub window_size: Option<String>,
30}
31
32/// Specification of a Git mirror
33///
34/// Explicitly a single lorry as git mirrors can only target a single
35/// repository.
36#[derive(Clone, Default, Serialize, Deserialize, Debug, PartialEq)]
37pub struct SingleLorry {
38    /// Url of the target git repository
39    pub url: String,
40    /// Verify SSL certificates default to true
41    #[serde(rename = "check-certificates", default = "trivially_true")]
42    pub check_ssl_certificates: bool,
43    /// Deprecated refspec option
44    #[deprecated]
45    pub refspecs: Option<Vec<String>>,
46    /// Patterns to match valid refs
47    pub ref_patterns: Option<Vec<String>>,
48    /// Refs to ignore
49    pub ignore_patterns: Option<Vec<String>>,
50    /// Enable large file storage
51    ///
52    /// TODO: Re-enable lfs for git mirrors
53    pub lfs: Option<bool>,
54    /// Git cleanup settings
55    pub gc: Option<GCOptions>,
56}
57
58/// Specification for a raw-file mirror
59///
60/// This can be a collection of raw files under the same identifier.
61#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
62pub struct MultipleRawFiles {
63    /// Target urls and sha256 for raw-files
64    pub urls: Vec<RawFileURLMapping>,
65
66    /// Verify SSL certificates default to true
67    ///
68    /// TODO should this be per file or per-entry?
69    #[serde(rename = "check-certificates", default = "trivially_true")]
70    pub check_ssl_certificates: bool,
71
72    /// Git cleanup settings
73    pub gc: Option<GCOptions>,
74}
75
76impl MultipleRawFiles {
77    /// Find the first missing sha256sum if any exist
78    pub fn missing_sha256sums(&self) -> Vec<String> {
79        self.urls
80            .iter()
81            .filter_map(|f| {
82                if f.sha256sum.is_none() {
83                    Some(f.url.to_string())
84                } else {
85                    None
86                }
87            })
88            .collect()
89    }
90}
91
92/// Specification for a raw-file
93#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
94pub struct RawFileURLMapping {
95    /// Destination on disk to store raw-file to
96    #[serde(default = "empty_relative_path")]
97    pub destination: RelativePathBuf,
98    /// Target url
99    pub url: url::Url,
100    /// SHA256 to verify that raw-file has imported successfully
101    pub sha256sum: Option<String>,
102}
103
104fn trivially_true() -> bool {
105    true
106}
107
108fn empty_relative_path() -> RelativePathBuf {
109    RelativePathBuf::new()
110}
111
112/// Attempts to convert a string-like, presumably the contents of a `.lorry` file, into the internal representation of a lorry spec.
113///
114/// This spec object can then be passed to `mirror`.
115pub fn extract_lorry_specs<S: AsRef<str>>(
116    s: S,
117) -> Result<BTreeMap<String, LorrySpec>, serde_yaml::Error> {
118    let lorries = serde_yaml::from_str(s.as_ref())?;
119    Ok(lorries)
120}
121
122#[test]
123fn lorries_extract() {
124    let lorry_dot_lorry = r"lorry:
125    type: git
126    url: https://gitlab.com/CodethinkLabs/lorry/lorry.git
127  ";
128
129    let extracted = extract_lorry_specs(lorry_dot_lorry).unwrap();
130
131    let mut correct: BTreeMap<String, LorrySpec> = BTreeMap::new();
132    correct.insert(
133        "lorry".to_string(),
134        LorrySpec::Git(SingleLorry {
135            url: "https://gitlab.com/CodethinkLabs/lorry/lorry.git".to_string(),
136            check_ssl_certificates: true,
137            ..Default::default()
138        }),
139    );
140    assert_eq!(extracted, correct);
141}
142
143#[test]
144fn extract_multiple_lorries() {
145    let lorries_as_string = r"ARM-software/arm-trusted-firmware:
146    type: git
147    url: https://github.com/ARM-software/arm-trusted-firmware.git
148HarryMichal/go-version:
149    type: git
150    url: https://github.com/HarryMichal/go-version.git
151TelepathyIM/telepathy-glib:
152    type: git
153    url: https://github.com/TelepathyIM/telepathy-glib.git
154TelepathyIM/telepathy-logger:
155    type: git
156    url: https://github.com/TelepathyIM/telepathy-logger.git
157TelepathyIM/telepathy-mission-control:
158    type: git
159    url: https://github.com/TelepathyIM/telepathy-mission-control.git
160acobaugh/osrelease:
161    type: git
162    url: https://github.com/acobaugh/osrelease.git
163ayufan-rock64/pinebook-pro-keyboard-updater:
164    type: git
165    url: https://github.com/ayufan-rock64/pinebook-pro-keyboard-updater.git
166briandowns/spinner:
167    type: git
168    url: https://github.com/briandowns/spinner.git
169docker/go-units:
170    type: git
171    url: https://github.com/docker/go-units.git
172ebassi/graphene:
173    type: git
174    url: https://github.com/ebassi/graphene.git
175endlessm/eos-installer:
176    type: git
177    url: https://github.com/endlessm/eos-installer.git
178fatih/color:
179    type: git
180    url: https://github.com/fatih/color.git
181fsnotify/fsnotify:
182    type: git
183    url: https://github.com/fsnotify/fsnotify.git
184godbus/dbus:
185    type: git
186    url: https://github.com/godbus/dbus.git
187golang/crypto:
188    type: git
189    url: https://github.com/golang/crypto.git
190golang/sys:
191    type: git
192    url: https://github.com/golang/sys.git
193hughsie/appstream-glib:
194    type: git
195    url: https://github.com/hughsie/appstream-glib.git
196hughsie/libgusb:
197    type: git
198    url: https://github.com/hughsie/libgusb.git
199inconshreveable/mousetrap:
200    type: git
201    url: https://github.com/inconshreveable/mousetrap.git
202konsorten/go-windows-terminal-sequences:
203    type: git
204    url: https://github.com/konsorten/go-windows-terminal-sequences.git
205libhangul/libhangul:
206    type: git
207    url: https://github.com/libhangul/libhangul.git
208libpinyin/libpinyin:
209    type: git
210    url: https://github.com/libpinyin/libpinyin.git
211libsigcplusplus/libsigcplusplus:
212    type: git
213    url: https://github.com/libsigcplusplus/libsigcplusplus.git
214mattn/go-colorable:
215    type: git
216    url: https://github.com/mattn/go-colorable.git
217mattn/go-isatty:
218    type: git
219    url: https://github.com/mattn/go-isatty.git
220raspberrypi/userland:
221    type: git
222    url: https://github.com/raspberrypi/userland.git
223sirupsen/logrus:
224    type: git
225    url: https://github.com/sirupsen/logrus.git
226spf13/cobra:
227    type: git
228    url: https://github.com/spf13/cobra.git
229spf13/pflag:
230    type: git
231    url: https://github.com/spf13/pflag.git
232stevegrubb/libcap-ng:
233    type: git
234    url: https://github.com/stevegrubb/libcap-ng.git
235streambinder/vpnc:
236    type: git
237    url: https://github.com/streambinder/vpnc.git
238
239";
240
241    //example from https://gitlab.gnome.org/Infrastructure/Mirrors/mirroring-config/-/blob/main/github_com/github_com.lorry
242
243    let extracted = extract_lorry_specs(lorries_as_string).unwrap();
244
245    assert_eq!(extracted.len(), 31);
246}
247
248#[test]
249fn extract_raw_file_lorries() {
250    let lorries_as_string = "tar-files:
251    type: raw-file
252    urls:
253      - destination: debian/pool/main/a/anthy
254        url: http://http.debian.net/debian/pool/main/a/anthy/anthy_0.3.orig.tar.gz
255      - destination: debian/pool/main/d/db5.3
256        url: http://http.debian.net/debian/pool/main/d/db5.3/db5.3_5.3.28.orig.tar.xz
257  
258  
259  ";
260    //example from https://gitlab.gnome.org/Infrastructure/Mirrors/mirroring-config/-/blob/main/github_com/github_com.lorry
261
262    let extracted = extract_lorry_specs(lorries_as_string).unwrap();
263
264    assert_eq!(extracted.len(), 1);
265    assert_eq!(
266        match &extracted["tar-files"] {
267            LorrySpec::RawFiles(MultipleRawFiles {
268                urls,
269                check_ssl_certificates: _,
270                gc: _,
271            }) => urls.len(),
272            _ => panic!("non-rawfiles parsed as raw-files!"),
273        },
274        2
275    )
276}
workerlib/lorry_specs.rs

workerlib/
lorry_specs.rs
