nimbus_fml/util/
loaders.rs

Help
1/* This Source Code Form is subject to the terms of the Mozilla Public
2* License, v. 2.0. If a copy of the MPL was not distributed with this
3* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4use crate::{
5    error::{FMLError, Result},
6    SUPPORT_URL_LOADING,
7};
8
9use anyhow::anyhow;
10use reqwest::blocking::{Client, ClientBuilder};
11use std::{
12    collections::{hash_map::DefaultHasher, BTreeMap},
13    env,
14    fmt::Display,
15    hash::{Hash, Hasher},
16    path::{Path, PathBuf},
17};
18use url::Url;
19
20pub(crate) const GITHUB_USER_CONTENT_DOTCOM: &str = "https://raw.githubusercontent.com";
21pub(crate) const API_GITHUB_DOTCOM: &str = "https://api.github.com";
22
23#[derive(Clone)]
24pub struct LoaderConfig {
25    pub cwd: PathBuf,
26    pub repo_files: Vec<String>,
27    pub cache_dir: Option<PathBuf>,
28    pub refs: BTreeMap<String, String>,
29}
30
31impl LoaderConfig {
32    pub(crate) fn repo_and_path(f: &str) -> Option<(String, String)> {
33        if f.starts_with('@') {
34            let parts = f.splitn(3, '/').collect::<Vec<&str>>();
35            match parts.as_slice() {
36                [user, repo, path] => Some((format!("{user}/{repo}"), path.to_string())),
37                _ => None,
38            }
39        } else {
40            None
41        }
42    }
43}
44
45impl Default for LoaderConfig {
46    fn default() -> Self {
47        Self {
48            repo_files: Default::default(),
49            cache_dir: None,
50            cwd: env::current_dir().expect("Current Working Directory is not set"),
51            refs: Default::default(),
52        }
53    }
54}
55
56/// A FilePath for a file hosted in a GitHub repository with a specified ref.
57#[derive(Clone, Debug)]
58pub struct GitHubRepoFilePath {
59    /// The repository id, i.e,. `owner/repo`.
60    repo_id: String,
61
62    /// The Git ref.
63    git_ref: String,
64
65    /// A Url, which is only used so that we can re-use Url::join for paths
66    /// inside the repository.
67    ///
68    /// The URL scheme and host should not be referenced.
69    ///
70    /// Instead of this, you probably want [`Self::path()`] or
71    /// [`Self::default_download_url()`].
72    url: Url,
73}
74
75impl GitHubRepoFilePath {
76    pub fn new(repo_id: &str, git_ref: &str) -> Self {
77        Self {
78            repo_id: repo_id.into(),
79            git_ref: git_ref.into(),
80            url: Url::parse("invalid://do-not-use/").expect("This is a constant, valid URL"),
81        }
82    }
83
84    /// Return the repository ID.
85    pub fn repo_id(&self) -> &str {
86        &self.repo_id
87    }
88
89    /// Return the Git ref.
90    pub fn git_ref(&self) -> &str {
91        &self.git_ref
92    }
93
94    /// Return the path of the file in the GitHub repository.
95    pub fn path(&self) -> &str {
96        self.url.path()
97    }
98
99    pub fn join(&self, file: &str) -> Result<Self> {
100        Ok(Self {
101            repo_id: self.repo_id.clone(),
102            git_ref: self.git_ref.clone(),
103            url: self.url.join(file)?,
104        })
105    }
106
107    /// Return the default download URL, without a token, as a string.
108    ///
109    /// [`Self::default_download_url()`] can return an error, so this is
110    /// provided as a convenience for situations where an actual valid URL is
111    /// not required, such as in Display impls.
112    pub(crate) fn default_download_url_as_str(&self) -> String {
113        format!(
114            "{}/{}/{}{}",
115            GITHUB_USER_CONTENT_DOTCOM,
116            self.repo_id,
117            self.git_ref,
118            self.path() // begins with a /
119        )
120    }
121
122    /// Return the default download URL, without a token.
123    ///
124    /// This URL can only be used to download files from public repositories.
125    ///
126    /// Otherwise, the URL must be retrieved via the GitHub repository contents
127    /// API.
128    pub fn default_download_url(&self) -> Result<Url> {
129        Url::parse(&self.default_download_url_as_str()).map_err(Into::into)
130    }
131
132    pub fn contents_api_url(&self) -> Result<Url> {
133        // https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#get-repository-content
134        Url::parse(&format!(
135            "{}/repos/{}/contents{}?ref={}",
136            API_GITHUB_DOTCOM,
137            self.repo_id,
138            self.path(), // begins with a /
139            self.git_ref
140        ))
141        .map_err(Into::into)
142    }
143}
144
145/// A small enum for working with URLs and relative files
146#[derive(Clone, Debug)]
147pub enum FilePath {
148    Local(PathBuf),
149    Remote(Url),
150    GitHub(GitHubRepoFilePath),
151}
152
153impl FilePath {
154    pub fn new(cwd: &Path, file: &str) -> Result<Self> {
155        Ok(if file.contains("://") {
156            FilePath::Remote(Url::parse(file)?)
157        } else {
158            FilePath::Local(cwd.join(file))
159        })
160    }
161
162    /// Appends a suffix to a path.
163    /// If the `self` is a local file and the suffix is an absolute URL,
164    /// then the return is the URL.
165    pub fn join(&self, file: &str) -> Result<Self> {
166        if file.contains("://") {
167            return Ok(FilePath::Remote(Url::parse(file)?));
168        }
169        Ok(match self {
170            Self::Local(p) => Self::Local(
171                // We implement a join similar to Url::join.
172                // If the root is a directory, we append;
173                // if not we take the parent, then append.
174                if is_dir(p) {
175                    p.join(file)
176                } else {
177                    p.parent()
178                        .expect("a file within a parent directory")
179                        .join(file)
180                },
181            ),
182            Self::Remote(u) => Self::Remote(u.join(file)?),
183            Self::GitHub(p) => Self::GitHub(p.join(file)?),
184        })
185    }
186
187    pub fn canonicalize(&self) -> Result<Self> {
188        Ok(match self {
189            Self::Local(p) => Self::Local(p.canonicalize().map_err(|e| {
190                // We do this map_err here because the IO Error message that comes out of `canonicalize`
191                // doesn't include the problematic file path.
192                FMLError::InvalidPath(format!("{}: {}", e, p.as_path().display()))
193            })?),
194            _ => self.clone(),
195        })
196    }
197
198    pub fn extension(&self) -> Option<&str> {
199        Some(match self {
200            Self::Local(p) => {
201                let ext = p.extension()?;
202                ext.to_str()?
203            }
204            Self::GitHub(GitHubRepoFilePath { url, .. }) | Self::Remote(url) => {
205                let file = url.path_segments()?.next_back()?;
206                let (_, ext) = file.rsplit_once('.')?;
207                ext
208            }
209        })
210    }
211}
212
213impl Display for FilePath {
214    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
215        match self {
216            Self::Local(p) => p.display().fmt(f),
217            Self::Remote(u) => u.fmt(f),
218            Self::GitHub(p) => p.default_download_url_as_str().fmt(f),
219        }
220    }
221}
222
223impl From<&Path> for FilePath {
224    fn from(path: &Path) -> Self {
225        Self::Local(path.into())
226    }
227}
228
229#[cfg(not(test))]
230fn is_dir(path_buf: &Path) -> bool {
231    path_buf.is_dir()
232}
233
234// In tests, the directories don't always exist on-disk, so we cannot use the
235// `.is_dir()` method, which would call `stat` (or equivalent) on a non-existent
236// file. Instead, we check for the presence of a trailing slash, so all tests
237// that need to treat a path like a directory *must* append trailing slashes to
238// those paths.
239#[cfg(test)]
240fn is_dir(path_buf: &Path) -> bool {
241    path_buf.display().to_string().ends_with('/')
242}
243
244static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"),);
245
246/// Utility class to abstract away the differences between loading from file and network.
247///
248/// With a nod to offline developer experience, files which come from the network
249/// are cached on disk.
250///
251/// The cache directory should be in a directory that will get purged on a clean build.
252///
253/// This allows us to import files from another repository (via https) or include files
254/// from a local files.
255///
256/// The loader is able to resolve a shortcut syntax similar to other package managers.
257///
258/// By default a prefix of `@XXXX/YYYY`: resolves to the `main` branch `XXXX/YYYY` Github repo.
259///
260/// The config is a map of repository names to paths, URLs or branches.
261///
262/// Config files can be loaded
263#[derive(Clone, Debug)]
264pub struct FileLoader {
265    cache_dir: Option<PathBuf>,
266    fetch_client: Client,
267
268    /// A mapping of repository IDs (without the leading @) to the git refs that
269    /// should be used to download files.
270    repo_refs: BTreeMap<String, FilePath>,
271
272    // This is used for resolving relative paths when no other path
273    // information is available.
274    cwd: PathBuf,
275}
276
277impl TryFrom<&LoaderConfig> for FileLoader {
278    type Error = FMLError;
279
280    fn try_from(loader_config: &LoaderConfig) -> Result<Self, Self::Error> {
281        let cache_dir = loader_config.cache_dir.clone();
282        let cwd = loader_config.cwd.clone();
283
284        let mut file_loader = Self::new(cwd, cache_dir, Default::default())?;
285
286        for (repo_id, git_ref) in &loader_config.refs {
287            file_loader.add_repo(repo_id, git_ref)?;
288        }
289
290        for f in &loader_config.repo_files {
291            let path = file_loader.file_path(f)?;
292            file_loader.add_repo_file(&path)?;
293        }
294
295        Ok(file_loader)
296    }
297}
298
299impl FileLoader {
300    pub fn new(
301        cwd: PathBuf,
302        cache_dir: Option<PathBuf>,
303        repo_refs: BTreeMap<String, FilePath>,
304    ) -> Result<Self> {
305        let http_client = ClientBuilder::new()
306            .https_only(true)
307            .user_agent(USER_AGENT)
308            .build()?;
309
310        Ok(Self {
311            cache_dir,
312            fetch_client: http_client,
313            cwd,
314            repo_refs,
315        })
316    }
317
318    #[allow(clippy::should_implement_trait)]
319    #[cfg(test)]
320    pub fn default() -> Result<Self> {
321        let cwd = std::env::current_dir()?;
322        let cache_path = cwd.join("build/app/fml-cache");
323        Self::new(
324            std::env::current_dir().expect("Current Working Directory not set"),
325            Some(cache_path),
326            Default::default(),
327        )
328    }
329
330    /// Load a file containing mapping of repo names to `FilePath`s.
331    /// Repo files can be JSON or YAML in format.
332    /// Files are simple key value pair mappings of `repo_id` to repository locations,
333    /// where:
334    ///
335    /// - a repo id is of the format used on Github: `$ORGANIZATION/$PROJECT`, and
336    /// - location can be
337    ///     - a path to a directory on disk, or
338    ///     - a ref/branch/tag/commit hash in the repo stored on Github.
339    ///
340    /// Relative paths to on disk directories will be taken as relative to this file.
341    pub fn add_repo_file(&mut self, file: &FilePath) -> Result<()> {
342        let config: BTreeMap<String, String> = self.read(file)?;
343
344        for (k, v) in config {
345            self.add_repo_relative(file, &k, &v)?;
346        }
347
348        Ok(())
349    }
350
351    /// Add a repo and version/tag/ref/location.
352    /// `repo_id` is the github `$ORGANIZATION/$PROJECT` string, e.g. `mozilla/application-services`.
353    /// The `loc` string can be a:
354    /// 1. A branch, commit hash or release tag on a remote repository, hosted on Github
355    /// 2. A URL
356    /// 3. A relative path (to the current working directory) to a directory on the local disk.
357    /// 4. An absolute path to a directory on the local disk.
358    pub fn add_repo(&mut self, repo_id: &str, loc: &str) -> Result<()> {
359        self.add_repo_relative(&FilePath::Local(self.cwd.clone()), repo_id, loc)
360    }
361
362    fn add_repo_relative(&mut self, cwd: &FilePath, repo_id: &str, loc: &str) -> Result<()> {
363        // We're building up a mapping of repo_ids to `FilePath`s; recall: `FilePath` is an enum that is an
364        // absolute path or URL.
365
366        // Standardize the form of repo id. We accept `@user/repo` or `user/repo`, but store it as
367        // `user/repo`.
368        let repo_id = repo_id.strip_prefix('@').unwrap_or(repo_id);
369
370        // We construct the FilePath. We want to be able to tell the difference between a what `FilePath`s
371        // can already reason about (relative file paths, absolute file paths and URLs) and what git knows about (refs, tags, versions).
372        let file_path = if loc.starts_with('.')
373            || loc.starts_with('/')
374            || loc.contains(":\\")
375            || loc.contains("://")
376        {
377            // The `loc`, whatever the current working directory, is going to end up as a part of a path.
378            // A trailing slash ensures it gets treated like a directory, rather than a file.
379            // See Url::join.
380            let loc = if loc.ends_with('/') {
381                loc.to_string()
382            } else {
383                format!("{}/", loc)
384            };
385
386            // URLs, relative file paths, absolute paths.
387            cwd.join(&loc)?
388        } else {
389            // refs, commit hashes, tags, branches.
390            self.remote_file_path(repo_id, loc)
391        };
392
393        // Finally, add the absolute path that we use every time the user refers to @user/repo.
394        self.repo_refs.insert(repo_id.into(), file_path);
395        Ok(())
396    }
397
398    fn remote_file_path(&self, repo: &str, branch_or_tag: &str) -> FilePath {
399        FilePath::GitHub(GitHubRepoFilePath::new(repo, branch_or_tag))
400    }
401
402    fn default_remote_path(&self, key: String) -> FilePath {
403        self.remote_file_path(&key, "main")
404    }
405
406    /// This loads a text file from disk or the network.
407    ///
408    /// If it's coming from the network, then cache the file to disk (based on the URL).
409    ///
410    /// We don't worry about cache invalidation, because a clean build should blow the cache
411    /// away.
412    pub fn read_to_string(&self, file: &FilePath) -> Result<String> {
413        Ok(match file {
414            FilePath::Local(path) => std::fs::read_to_string(path)?,
415            FilePath::Remote(url) => self.fetch_and_cache(url)?,
416            FilePath::GitHub(p) => {
417                // If there is a GITHUB_BEARER_TOKEN environment variable
418                // present, we will use that to get the download URL from the
419                // GitHub contents API.
420                let api_key = match env::var("GITHUB_BEARER_TOKEN") {
421                    Ok(api_key) => Some(api_key),
422                    Err(env::VarError::NotPresent) => None,
423                    Err(env::VarError::NotUnicode(_)) => Err(FMLError::InvalidApiToken)?,
424                };
425
426                let download_url = if let Some(api_key) = api_key {
427                    let contents_api_url = p.contents_api_url()?;
428
429                    // The response format is documented here:
430                    // https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#get-repository-content
431                    self.fetch_client
432                        .get(contents_api_url)
433                        .bearer_auth(api_key)
434                        .send()?
435                        .error_for_status()?
436                        .json::<serde_json::Value>()?
437                        .get("download_url")
438                        .and_then(serde_json::Value::as_str)
439                        .ok_or_else(|| {
440                            anyhow!(
441                                "GitHub API did not return a download_url for @{}/{} at ref {}",
442                                p.repo_id(),
443                                p.path(),
444                                p.git_ref()
445                            )
446                        })
447                        .and_then(|u| Url::parse(u).map_err(Into::into))?
448                } else {
449                    p.default_download_url()?
450                };
451
452                self.fetch_and_cache(&download_url)?
453            }
454        })
455    }
456
457    pub fn read<T: serde::de::DeserializeOwned>(&self, file: &FilePath) -> Result<T> {
458        let string = self
459            .read_to_string(file)
460            .map_err(|e| FMLError::InvalidPath(format!("{file}: {e}")))?;
461
462        Ok(serde_yaml::from_str(&string)?)
463    }
464
465    fn fetch_and_cache(&self, url: &Url) -> Result<String> {
466        if !SUPPORT_URL_LOADING {
467            unimplemented!("Loading manifests from URLs is not yet supported ({})", url);
468        }
469        let path_buf = self.create_cache_path_buf(url);
470        Ok(if path_buf.exists() {
471            std::fs::read_to_string(path_buf)?
472        } else {
473            let res = self
474                .fetch_client
475                .get(url.clone())
476                .send()?
477                .error_for_status()?;
478            let text = res.text()?;
479
480            let parent = path_buf.parent().expect("Cache directory is specified");
481            if !parent.exists() {
482                std::fs::create_dir_all(parent)?;
483            }
484
485            std::fs::write(path_buf, &text)?;
486            text
487        })
488    }
489
490    fn create_cache_path_buf(&self, url: &Url) -> PathBuf {
491        // Method to look after the cache directory.
492        // We can organize this how we want: in this case we use a flat structure
493        // with a hash of the URL as a prefix of the directory.
494        let mut hasher = DefaultHasher::new();
495        url.hash(&mut hasher);
496        let checksum = hasher.finish();
497        let filename = match url.path_segments() {
498            Some(mut segments) => segments.next_back().unwrap_or("unknown.txt"),
499            None => "unknown.txt",
500        };
501        // Take the last 16 bytes of the hash to make sure our prefixes are still random, but
502        // not crazily long.
503        let filename = format!("{:x}_{}", (checksum & 0x000000000000FFFF) as u16, filename,);
504
505        self.cache_dir().join(filename)
506    }
507
508    fn cache_dir(&self) -> &Path {
509        match &self.cache_dir {
510            Some(d) => d,
511            _ => self.tmp_cache_dir(),
512        }
513    }
514
515    fn tmp_cache_dir<'a>(&self) -> &'a Path {
516        use std::time::SystemTime;
517        lazy_static::lazy_static! {
518            static ref CACHE_DIR_NAME: String = format!("nimbus-fml-manifests-{:x}", match SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) {
519                Ok(n) => n.as_micros() & 0x00ffffff,
520                Err(_) => 0,
521            });
522
523            static ref TMP_CACHE_DIR: PathBuf = std::env::temp_dir().join(CACHE_DIR_NAME.as_str());
524        }
525        &TMP_CACHE_DIR
526    }
527
528    /// Joins a path to a string, to make a new path.
529    ///
530    /// We want to be able to support local and remote files.
531    /// We also want to be able to support a configurable short cut format.
532    /// Following a pattern common in other package managers, `@XXXX/YYYY`
533    /// is used as short hand for the main branch in github repos.
534    ///
535    /// If `f` is a relative path, the result is relative to `base`.
536    pub fn join(&self, base: &FilePath, f: &str) -> Result<FilePath> {
537        Ok(if let Some(u) = self.resolve_url_shortcut(f)? {
538            u
539        } else {
540            base.join(f)?
541        })
542    }
543
544    /// Make a new path.
545    ///
546    /// We want to be able to support local and remote files.
547    /// We also want to be able to support a configurable short cut format.
548    /// Following a pattern common in other package managers, `@XXXX/YYYY`
549    /// is used as short hand for the main branch in github repos.
550    ///
551    /// If `f` is a relative path, the result is relative to `self.cwd`.
552    pub fn file_path(&self, f: &str) -> Result<FilePath> {
553        Ok(if let Some(u) = self.resolve_url_shortcut(f)? {
554            u
555        } else {
556            FilePath::new(&self.cwd, f)?
557        })
558    }
559
560    /// Checks that the given string has a @organization/repo/ prefix.
561    /// If it does, then use that as a `repo_id` to look up the `FilePath` prefix
562    /// if it exists in the `repo_refs`, and use the `main` branch of the github repo if it
563    /// doesn't exist.
564    fn resolve_url_shortcut(&self, f: &str) -> Result<Option<FilePath>> {
565        if f.starts_with('@') {
566            let f = f.replacen('@', "", 1);
567            let parts = f.splitn(3, '/').collect::<Vec<&str>>();
568            match parts.as_slice() {
569                [user, repo, path] => {
570                    let key = format!("{}/{}", user, repo);
571                    Ok(if let Some(repo) = self.lookup_repo_path(user, repo) {
572                        Some(repo.join(path)?)
573                    } else {
574                        let repo = self.default_remote_path(key);
575                        Some(repo.join(path)?)
576                    })
577                }
578                _ => Err(FMLError::InvalidPath(format!(
579                    "'{}' needs to include a username, a repo and a filepath",
580                    f
581                ))),
582            }
583        } else {
584            Ok(None)
585        }
586    }
587
588    fn lookup_repo_path(&self, user: &str, repo: &str) -> Option<&FilePath> {
589        let key = format!("{}/{}", user, repo);
590        self.repo_refs.get(&key)
591    }
592}
593
594impl Drop for FileLoader {
595    fn drop(&mut self) {
596        if self.cache_dir.is_some() {
597            return;
598        }
599        let cache_dir = self.tmp_cache_dir();
600        if cache_dir.exists() {
601            _ = std::fs::remove_dir_all(cache_dir);
602        }
603    }
604}
605
606#[cfg(test)]
607mod unit_tests {
608    use std::fs;
609
610    use crate::util::{build_dir, pkg_dir};
611
612    use super::*;
613
614    #[test]
615    fn test_relative_paths() -> Result<()> {
616        let tmp = std::env::temp_dir();
617
618        let file = tmp.join("foo/bar.txt");
619        let obs = FilePath::from(file.as_path());
620
621        assert!(matches!(obs, FilePath::Local(_)));
622        assert!(obs.to_string().ends_with("foo/bar.txt"));
623
624        let obs = obs.join("baz.txt")?;
625        assert!(obs.to_string().ends_with("foo/baz.txt"));
626
627        let obs = obs.join("./bam.txt")?;
628        // We'd prefer it to be like this:
629        // assert!(obs.to_string().ends_with("foo/bam.txt"));
630        // But there's no easy way to get this (because symlinks).
631        // This is most likely the correct thing for us to do.
632        // We put this test here for documentation purposes, and to
633        // highlight that with URLs, ../ and ./ do what you might
634        // expect.
635        assert!(obs.to_string().ends_with("foo/./bam.txt"));
636
637        let obs = obs.join("https://example.com/foo/bar.txt")?;
638        assert!(matches!(obs, FilePath::Remote(_)));
639        assert_eq!(obs.to_string(), "https://example.com/foo/bar.txt");
640
641        let obs = obs.join("baz.txt")?;
642        assert_eq!(obs.to_string(), "https://example.com/foo/baz.txt");
643
644        let obs = obs.join("./bam.txt")?;
645        assert_eq!(obs.to_string(), "https://example.com/foo/bam.txt");
646
647        let obs = obs.join("../brum/bram.txt")?;
648        assert_eq!(obs.to_string(), "https://example.com/brum/bram.txt");
649
650        Ok(())
651    }
652
653    #[test]
654    fn test_at_shorthand_with_no_at() -> Result<()> {
655        let files = create_loader()?;
656        let cwd = FilePath::Local(files.cwd.clone());
657        let src_file = cwd.join("base/old.txt")?;
658
659        // A source file asks for a destination file relative to it.
660        let obs = files.join(&src_file, "a/file.txt")?;
661        assert!(matches!(obs, FilePath::Local(_)));
662        assert_eq!(
663            obs.to_string(),
664            format!("{}/base/a/file.txt", remove_trailing_slash(&cwd))
665        );
666        Ok(())
667    }
668
669    #[test]
670    fn test_at_shorthand_default_branch() -> Result<()> {
671        let files = create_loader()?;
672        let cwd = FilePath::Local(files.cwd.clone());
673        let src_file = cwd.join("base/old.txt")?;
674
675        // A source file asks for a file in another repo. We haven't any specific configuration
676        // for this repo, so we default to the `main` branch.
677        let obs = files.join(&src_file, "@repo/unspecified/a/file.txt")?;
678        assert!(
679            matches!(obs, FilePath::GitHub(ref gh) if gh.repo_id() == "repo/unspecified" && gh.git_ref() == "main" && gh.path() == "/a/file.txt")
680        );
681        assert_eq!(
682            obs.to_string(),
683            "https://raw.githubusercontent.com/repo/unspecified/main/a/file.txt"
684        );
685        Ok(())
686    }
687
688    #[test]
689    fn test_at_shorthand_absolute_url() -> Result<()> {
690        let mut files = create_loader()?;
691        let cwd = FilePath::Local(files.cwd.clone());
692        let src_file = cwd.join("base/old.txt")?;
693
694        // A source file asks for a file in another repo. The loader uses an absolute
695        // URL as the base URL.
696        files.add_repo("@repos/url", "https://example.com/remote/directory/path")?;
697
698        let obs = files.join(&src_file, "@repos/url/a/file.txt")?;
699        assert!(matches!(obs, FilePath::Remote(_)));
700        assert_eq!(
701            obs.to_string(),
702            "https://example.com/remote/directory/path/a/file.txt"
703        );
704
705        let obs = files.file_path("@repos/url/b/file.txt")?;
706        assert!(matches!(obs, FilePath::Remote(_)));
707        assert_eq!(
708            obs.to_string(),
709            "https://example.com/remote/directory/path/b/file.txt"
710        );
711        Ok(())
712    }
713
714    #[test]
715    fn test_at_shorthand_specified_branch() -> Result<()> {
716        let mut files = create_loader()?;
717        let cwd = FilePath::Local(files.cwd.clone());
718        let src_file = cwd.join("base/old.txt")?;
719
720        // A source file asks for a file in another repo. The loader uses the branch/tag/ref
721        // specified.
722        files.add_repo("@repos/branch", "develop")?;
723        let obs = files.join(&src_file, "@repos/branch/a/file.txt")?;
724        assert!(
725            matches!(obs, FilePath::GitHub(ref gh) if gh.repo_id() == "repos/branch" && gh.git_ref() == "develop" && gh.path() == "/a/file.txt")
726        );
727        assert_eq!(
728            obs.to_string(),
729            "https://raw.githubusercontent.com/repos/branch/develop/a/file.txt"
730        );
731
732        let obs = files.file_path("@repos/branch/b/file.txt")?;
733        assert!(
734            matches!(obs, FilePath::GitHub(ref gh) if gh.repo_id() == "repos/branch" && gh.git_ref() == "develop" && gh.path() == "/b/file.txt")
735        );
736        assert_eq!(
737            obs.to_string(),
738            "https://raw.githubusercontent.com/repos/branch/develop/b/file.txt"
739        );
740        Ok(())
741    }
742
743    #[test]
744    fn test_at_shorthand_local_development() -> Result<()> {
745        let mut files = create_loader()?;
746        let cwd = FilePath::Local(files.cwd.clone());
747        let src_file = cwd.join("base/old.txt")?;
748
749        // A source file asks for a file in another repo. The loader is configured to
750        // give a file in a directory on the local filesystem.
751        let rel_dir = "../directory/path";
752        files.add_repo("@repos/local", rel_dir)?;
753
754        let obs = files.join(&src_file, "@repos/local/a/file.txt")?;
755        assert!(matches!(obs, FilePath::Local(_)));
756        assert_eq!(
757            obs.to_string(),
758            format!("{}/{}/a/file.txt", remove_trailing_slash(&cwd), rel_dir)
759        );
760
761        let obs = files.file_path("@repos/local/b/file.txt")?;
762        assert!(matches!(obs, FilePath::Local(_)));
763        assert_eq!(
764            obs.to_string(),
765            format!("{}/{}/b/file.txt", remove_trailing_slash(&cwd), rel_dir)
766        );
767
768        Ok(())
769    }
770
771    fn create_loader() -> Result<FileLoader, FMLError> {
772        let cache_dir = PathBuf::from(format!("{}/cache", build_dir()));
773        let repo_refs = Default::default();
774        let cwd = PathBuf::from(format!("{}/fixtures/", pkg_dir()));
775        let loader = FileLoader::new(cwd, Some(cache_dir), repo_refs)?;
776        Ok(loader)
777    }
778
779    #[test]
780    fn test_at_shorthand_from_config_file() -> Result<()> {
781        let cwd = PathBuf::from(pkg_dir());
782
783        let config = &LoaderConfig {
784            cwd,
785            cache_dir: None,
786            repo_files: vec![
787                "fixtures/loaders/config_files/remote.json".to_string(),
788                "fixtures/loaders/config_files/local.yaml".to_string(),
789            ],
790            refs: Default::default(),
791        };
792
793        let files: FileLoader = config.try_into()?;
794        let cwd = FilePath::Local(files.cwd.clone());
795
796        // This is a remote repo, specified in remote.json.
797        let tfr = files.file_path("@my/remote/file.txt")?;
798        assert_eq!(
799            tfr.to_string(),
800            "https://example.com/repo/branch/file.txt".to_string()
801        );
802
803        // This is a local file, specified in local.yaml
804        let tf1 = files.file_path("@test/nested1/test-file.txt")?;
805        assert_eq!(
806            tf1.to_string(),
807            format!(
808                "{}/fixtures/loaders/config_files/./nested-1/test-file.txt",
809                &cwd
810            )
811        );
812
813        // This is a remote repo, specified in remote.json, but overridden in local.yaml
814        let tf2 = files.file_path("@test/nested2/test-file.txt")?;
815        assert_eq!(
816            tf2.to_string(),
817            format!(
818                "{}/fixtures/loaders/config_files/./nested-2/test-file.txt",
819                &cwd
820            )
821        );
822
823        let tf1 = files.read_to_string(&tf1)?;
824        let tf2 = files.read_to_string(&tf2)?;
825
826        assert_eq!("test-file/1".to_string(), tf1);
827        assert_eq!("test-file/2".to_string(), tf2);
828
829        Ok(())
830    }
831
832    fn remove_trailing_slash(cwd: &FilePath) -> String {
833        let s = cwd.to_string();
834        let mut chars = s.chars();
835        if s.ends_with('/') {
836            chars.next_back();
837        }
838        chars.as_str().to_string()
839    }
840
841    #[test]
842    fn test_at_shorthand_override_via_cli() -> Result<()> {
843        let cwd = PathBuf::from(pkg_dir());
844
845        let config = &LoaderConfig {
846            cwd,
847            cache_dir: None,
848            repo_files: Default::default(),
849            refs: BTreeMap::from([("@my-remote/repo".to_string(), "cli-branch".to_string())]),
850        };
851
852        let files: FileLoader = config.try_into()?;
853
854        // This is a file from the remote repo
855        let tfr = files.file_path("@my-remote/repo/path/to/file.txt")?;
856        assert_eq!(
857            tfr.to_string(),
858            // We're going to fetch it from the `cli-branch` of the repo.
859            "https://raw.githubusercontent.com/my-remote/repo/cli-branch/path/to/file.txt"
860                .to_string()
861        );
862
863        Ok(())
864    }
865
866    #[test]
867    fn test_dropping_tmp_cache_dir() -> Result<()> {
868        let cwd = PathBuf::from(pkg_dir());
869        let config = &LoaderConfig {
870            cwd,
871            cache_dir: None,
872            repo_files: Default::default(),
873            refs: Default::default(),
874        };
875
876        let files: FileLoader = config.try_into()?;
877        let cache_dir = files.tmp_cache_dir();
878        fs::create_dir_all(cache_dir)?;
879
880        assert!(cache_dir.exists());
881        drop(files);
882
883        assert!(!cache_dir.exists());
884        Ok(())
885    }
886
887    #[test]
888    fn test_github_repo_file_path() -> Result<()> {
889        let gh = GitHubRepoFilePath::new("owner/repo-name", "ref").join("a/file.txt")?;
890        assert_eq!(
891            gh.contents_api_url()?.to_string(),
892            "https://api.github.com/repos/owner/repo-name/contents/a/file.txt?ref=ref",
893        );
894        assert_eq!(
895            gh.default_download_url()?.to_string(),
896            "https://raw.githubusercontent.com/owner/repo-name/ref/a/file.txt"
897        );
898
899        let gh = gh.join("/b/file.txt")?;
900        assert_eq!(
901            gh.contents_api_url()?.to_string(),
902            "https://api.github.com/repos/owner/repo-name/contents/b/file.txt?ref=ref",
903        );
904        assert_eq!(
905            gh.default_download_url()?.to_string(),
906            "https://raw.githubusercontent.com/owner/repo-name/ref/b/file.txt"
907        );
908
909        let gh = gh.join("/c/")?.join("file.txt")?;
910        assert_eq!(
911            gh.contents_api_url()?.to_string(),
912            "https://api.github.com/repos/owner/repo-name/contents/c/file.txt?ref=ref",
913        );
914        assert_eq!(
915            gh.default_download_url()?.to_string(),
916            "https://raw.githubusercontent.com/owner/repo-name/ref/c/file.txt"
917        );
918
919        let gh = gh.join("d/")?.join("file.txt")?;
920        assert_eq!(
921            gh.contents_api_url()?.to_string(),
922            "https://api.github.com/repos/owner/repo-name/contents/c/d/file.txt?ref=ref",
923        );
924        assert_eq!(
925            gh.default_download_url()?.to_string(),
926            "https://raw.githubusercontent.com/owner/repo-name/ref/c/d/file.txt"
927        );
928
929        Ok(())
930    }
931
932    #[test]
933    fn test_extension() -> Result<()> {
934        let path = FilePath::Local("file.json".into());
935        assert_eq!(path.extension(), Some("json"));
936
937        let path = FilePath::Local("file.fml.yaml".into());
938        assert_eq!(path.extension(), Some("yaml"));
939
940        let path = FilePath::Local("file".into());
941        assert_eq!(path.extension(), None);
942
943        // Remote paths
944        let path = FilePath::Remote("https://example.com/file.json".try_into()?);
945        assert_eq!(path.extension(), Some("json"));
946
947        let path = FilePath::Remote("https://example.com/file.fml.yaml".try_into()?);
948        assert_eq!(path.extension(), Some("yaml"));
949
950        let path = FilePath::Remote("https://example.com/".try_into()?);
951        assert_eq!(path.extension(), None);
952
953        let path = FilePath::Remote("https://example.com/file".try_into()?);
954        assert_eq!(path.extension(), None);
955
956        let path = FilePath::Remote("https://example.com/path/".try_into()?);
957        assert_eq!(path.extension(), None);
958
959        let path = FilePath::GitHub(GitHubRepoFilePath::new("example", "main"));
960        assert_eq!(path.extension(), None);
961
962        let path = path.join("./file.json")?;
963        assert_eq!(path.extension(), Some("json"));
964
965        let path = path.join("./file.fml.yaml")?;
966        assert_eq!(path.extension(), Some("yaml"));
967
968        Ok(())
969    }
970}
nimbus_fml/util/loaders.rs

nimbus_fml/util/
loaders.rs
