nimbus_fml/util/
loaders.rs

1/* This Source Code Form is subject to the terms of the Mozilla Public
2* License, v. 2.0. If a copy of the MPL was not distributed with this
3* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4use crate::{
5    error::{FMLError, Result},
6    SUPPORT_URL_LOADING,
7};
8
9use anyhow::anyhow;
10use std::{
11    collections::{hash_map::DefaultHasher, BTreeMap},
12    env,
13    fmt::Display,
14    hash::{Hash, Hasher},
15    path::{Path, PathBuf},
16};
17use url::Url;
18
19pub(crate) const GITHUB_USER_CONTENT_DOTCOM: &str = "https://raw.githubusercontent.com";
20pub(crate) const API_GITHUB_DOTCOM: &str = "https://api.github.com";
21
22#[derive(Clone)]
23pub struct LoaderConfig {
24    pub cwd: PathBuf,
25    pub repo_files: Vec<String>,
26    pub cache_dir: Option<PathBuf>,
27    pub refs: BTreeMap<String, String>,
28}
29
30impl LoaderConfig {
31    pub(crate) fn repo_and_path(f: &str) -> Option<(String, String)> {
32        if f.starts_with('@') {
33            let parts = f.splitn(3, '/').collect::<Vec<&str>>();
34            match parts.as_slice() {
35                [user, repo, path] => Some((format!("{user}/{repo}"), path.to_string())),
36                _ => None,
37            }
38        } else {
39            None
40        }
41    }
42}
43
44impl Default for LoaderConfig {
45    fn default() -> Self {
46        Self {
47            repo_files: Default::default(),
48            cache_dir: None,
49            cwd: env::current_dir().expect("Current Working Directory is not set"),
50            refs: Default::default(),
51        }
52    }
53}
54
55/// A FilePath for a file hosted in a GitHub repository with a specified ref.
56#[derive(Clone, Debug)]
57pub struct GitHubRepoFilePath {
58    /// The repository id, i.e,. `owner/repo`.
59    repo_id: String,
60
61    /// The Git ref.
62    git_ref: String,
63
64    /// A Url, which is only used so that we can re-use Url::join for paths
65    /// inside the repository.
66    ///
67    /// The URL scheme and host should not be referenced.
68    ///
69    /// Instead of this, you probably want [`Self::path()`] or
70    /// [`Self::default_download_url()`].
71    url: Url,
72}
73
74impl GitHubRepoFilePath {
75    pub fn new(repo_id: &str, git_ref: &str) -> Self {
76        Self {
77            repo_id: repo_id.into(),
78            git_ref: git_ref.into(),
79            url: Url::parse("invalid://do-not-use/").expect("This is a constant, valid URL"),
80        }
81    }
82
83    /// Return the repository ID.
84    pub fn repo_id(&self) -> &str {
85        &self.repo_id
86    }
87
88    /// Return the Git ref.
89    pub fn git_ref(&self) -> &str {
90        &self.git_ref
91    }
92
93    /// Return the path of the file in the GitHub repository.
94    pub fn path(&self) -> &str {
95        self.url.path()
96    }
97
98    pub fn join(&self, file: &str) -> Result<Self> {
99        Ok(Self {
100            repo_id: self.repo_id.clone(),
101            git_ref: self.git_ref.clone(),
102            url: self.url.join(file)?,
103        })
104    }
105
106    /// Return the default download URL, without a token, as a string.
107    ///
108    /// [`Self::default_download_url()`] can return an error, so this is
109    /// provided as a convenience for situations where an actual valid URL is
110    /// not required, such as in Display impls.
111    pub(crate) fn default_download_url_as_str(&self) -> String {
112        format!(
113            "{}/{}/{}{}",
114            GITHUB_USER_CONTENT_DOTCOM,
115            self.repo_id,
116            self.git_ref,
117            self.path() // begins with a /
118        )
119    }
120
121    /// Return the default download URL, without a token.
122    ///
123    /// This URL can only be used to download files from public repositories.
124    ///
125    /// Otherwise, the URL must be retrieved via the GitHub repository contents
126    /// API.
127    pub fn default_download_url(&self) -> Result<Url> {
128        Url::parse(&self.default_download_url_as_str()).map_err(Into::into)
129    }
130
131    pub fn contents_api_url(&self) -> Result<Url> {
132        // https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#get-repository-content
133        Url::parse(&format!(
134            "{}/repos/{}/contents{}?ref={}",
135            API_GITHUB_DOTCOM,
136            self.repo_id,
137            self.path(), // begins with a /
138            self.git_ref
139        ))
140        .map_err(Into::into)
141    }
142}
143
144/// A small enum for working with URLs and relative files
145#[derive(Clone, Debug)]
146pub enum FilePath {
147    Local(PathBuf),
148    Remote(Url),
149    GitHub(GitHubRepoFilePath),
150}
151
152impl FilePath {
153    pub fn new(cwd: &Path, file: &str) -> Result<Self> {
154        Ok(if file.contains("://") {
155            FilePath::Remote(Url::parse(file)?)
156        } else {
157            FilePath::Local(cwd.join(file))
158        })
159    }
160
161    /// Appends a suffix to a path.
162    /// If the `self` is a local file and the suffix is an absolute URL,
163    /// then the return is the URL.
164    pub fn join(&self, file: &str) -> Result<Self> {
165        if file.contains("://") {
166            return Ok(FilePath::Remote(Url::parse(file)?));
167        }
168        Ok(match self {
169            Self::Local(p) => Self::Local(
170                // We implement a join similar to Url::join.
171                // If the root is a directory, we append;
172                // if not we take the parent, then append.
173                if is_dir(p) {
174                    p.join(file)
175                } else {
176                    p.parent()
177                        .expect("a file within a parent directory")
178                        .join(file)
179                },
180            ),
181            Self::Remote(u) => Self::Remote(u.join(file)?),
182            Self::GitHub(p) => Self::GitHub(p.join(file)?),
183        })
184    }
185
186    pub fn canonicalize(&self) -> Result<Self> {
187        Ok(match self {
188            Self::Local(p) => Self::Local(p.canonicalize().map_err(|e| {
189                // We do this map_err here because the IO Error message that comes out of `canonicalize`
190                // doesn't include the problematic file path.
191                FMLError::InvalidPath(format!("{}: {}", e, p.as_path().display()))
192            })?),
193            _ => self.clone(),
194        })
195    }
196
197    pub fn extension(&self) -> Option<&str> {
198        Some(match self {
199            Self::Local(p) => {
200                let ext = p.extension()?;
201                ext.to_str()?
202            }
203            Self::GitHub(GitHubRepoFilePath { url, .. }) | Self::Remote(url) => {
204                let file = url.path_segments()?.next_back()?;
205                let (_, ext) = file.rsplit_once('.')?;
206                ext
207            }
208        })
209    }
210}
211
212impl Display for FilePath {
213    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
214        match self {
215            Self::Local(p) => p.display().fmt(f),
216            Self::Remote(u) => u.fmt(f),
217            Self::GitHub(p) => p.default_download_url_as_str().fmt(f),
218        }
219    }
220}
221
222impl From<&Path> for FilePath {
223    fn from(path: &Path) -> Self {
224        Self::Local(path.into())
225    }
226}
227
228#[cfg(not(test))]
229fn is_dir(path_buf: &Path) -> bool {
230    path_buf.is_dir()
231}
232
233// In tests, the directories don't always exist on-disk, so we cannot use the
234// `.is_dir()` method, which would call `stat` (or equivalent) on a non-existent
235// file. Instead, we check for the presence of a trailing slash, so all tests
236// that need to treat a path like a directory *must* append trailing slashes to
237// those paths.
238#[cfg(test)]
239fn is_dir(path_buf: &Path) -> bool {
240    path_buf.display().to_string().ends_with('/')
241}
242
243static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"),);
244
245/// Utility class to abstract away the differences between loading from file and network.
246///
247/// With a nod to offline developer experience, files which come from the network
248/// are cached on disk.
249///
250/// The cache directory should be in a directory that will get purged on a clean build.
251///
252/// This allows us to import files from another repository (via https) or include files
253/// from a local files.
254///
255/// The loader is able to resolve a shortcut syntax similar to other package managers.
256///
257/// By default a prefix of `@XXXX/YYYY`: resolves to the `main` branch `XXXX/YYYY` Github repo.
258///
259/// The config is a map of repository names to paths, URLs or branches.
260///
261/// Config files can be loaded
262#[derive(Clone, Debug)]
263pub struct FileLoader {
264    cache_dir: Option<PathBuf>,
265
266    /// A mapping of repository IDs (without the leading @) to the git refs that
267    /// should be used to download files.
268    repo_refs: BTreeMap<String, FilePath>,
269
270    // This is used for resolving relative paths when no other path
271    // information is available.
272    cwd: PathBuf,
273}
274
275impl TryFrom<&LoaderConfig> for FileLoader {
276    type Error = FMLError;
277
278    fn try_from(loader_config: &LoaderConfig) -> Result<Self, Self::Error> {
279        let cache_dir = loader_config.cache_dir.clone();
280        let cwd = loader_config.cwd.clone();
281
282        let mut file_loader = Self::new(cwd, cache_dir, Default::default())?;
283
284        for (repo_id, git_ref) in &loader_config.refs {
285            file_loader.add_repo(repo_id, git_ref)?;
286        }
287
288        for f in &loader_config.repo_files {
289            let path = file_loader.file_path(f)?;
290            file_loader.add_repo_file(&path)?;
291        }
292
293        Ok(file_loader)
294    }
295}
296
297impl FileLoader {
298    pub fn new(
299        cwd: PathBuf,
300        cache_dir: Option<PathBuf>,
301        repo_refs: BTreeMap<String, FilePath>,
302    ) -> Result<Self> {
303        Ok(Self {
304            cache_dir,
305            cwd,
306            repo_refs,
307        })
308    }
309
310    #[allow(clippy::should_implement_trait)]
311    #[cfg(test)]
312    pub fn default() -> Result<Self> {
313        let cwd = std::env::current_dir()?;
314        let cache_path = cwd.join("build/app/fml-cache");
315        Self::new(
316            std::env::current_dir().expect("Current Working Directory not set"),
317            Some(cache_path),
318            Default::default(),
319        )
320    }
321
322    /// Load a file containing mapping of repo names to `FilePath`s.
323    /// Repo files can be JSON or YAML in format.
324    /// Files are simple key value pair mappings of `repo_id` to repository locations,
325    /// where:
326    ///
327    /// - a repo id is of the format used on Github: `$ORGANIZATION/$PROJECT`, and
328    /// - location can be
329    ///     - a path to a directory on disk, or
330    ///     - a ref/branch/tag/commit hash in the repo stored on Github.
331    ///
332    /// Relative paths to on disk directories will be taken as relative to this file.
333    pub fn add_repo_file(&mut self, file: &FilePath) -> Result<()> {
334        let config: BTreeMap<String, String> = self.read(file)?;
335
336        for (k, v) in config {
337            self.add_repo_relative(file, &k, &v)?;
338        }
339
340        Ok(())
341    }
342
343    /// Add a repo and version/tag/ref/location.
344    /// `repo_id` is the github `$ORGANIZATION/$PROJECT` string, e.g. `mozilla/application-services`.
345    /// The `loc` string can be a:
346    /// 1. A branch, commit hash or release tag on a remote repository, hosted on Github
347    /// 2. A URL
348    /// 3. A relative path (to the current working directory) to a directory on the local disk.
349    /// 4. An absolute path to a directory on the local disk.
350    pub fn add_repo(&mut self, repo_id: &str, loc: &str) -> Result<()> {
351        self.add_repo_relative(&FilePath::Local(self.cwd.clone()), repo_id, loc)
352    }
353
354    fn add_repo_relative(&mut self, cwd: &FilePath, repo_id: &str, loc: &str) -> Result<()> {
355        // We're building up a mapping of repo_ids to `FilePath`s; recall: `FilePath` is an enum that is an
356        // absolute path or URL.
357
358        // Standardize the form of repo id. We accept `@user/repo` or `user/repo`, but store it as
359        // `user/repo`.
360        let repo_id = repo_id.strip_prefix('@').unwrap_or(repo_id);
361
362        // We construct the FilePath. We want to be able to tell the difference between a what `FilePath`s
363        // can already reason about (relative file paths, absolute file paths and URLs) and what git knows about (refs, tags, versions).
364        let file_path = if loc.starts_with('.')
365            || loc.starts_with('/')
366            || loc.contains(":\\")
367            || loc.contains("://")
368        {
369            // The `loc`, whatever the current working directory, is going to end up as a part of a path.
370            // A trailing slash ensures it gets treated like a directory, rather than a file.
371            // See Url::join.
372            let loc = if loc.ends_with('/') {
373                loc.to_string()
374            } else {
375                format!("{}/", loc)
376            };
377
378            // URLs, relative file paths, absolute paths.
379            cwd.join(&loc)?
380        } else {
381            // refs, commit hashes, tags, branches.
382            self.remote_file_path(repo_id, loc)
383        };
384
385        // Finally, add the absolute path that we use every time the user refers to @user/repo.
386        self.repo_refs.insert(repo_id.into(), file_path);
387        Ok(())
388    }
389
390    fn remote_file_path(&self, repo: &str, branch_or_tag: &str) -> FilePath {
391        FilePath::GitHub(GitHubRepoFilePath::new(repo, branch_or_tag))
392    }
393
394    fn default_remote_path(&self, key: String) -> FilePath {
395        self.remote_file_path(&key, "main")
396    }
397
398    /// This loads a text file from disk or the network.
399    ///
400    /// If it's coming from the network, then cache the file to disk (based on the URL).
401    ///
402    /// We don't worry about cache invalidation, because a clean build should blow the cache
403    /// away.
404    pub fn read_to_string(&self, file: &FilePath) -> Result<String> {
405        Ok(match file {
406            FilePath::Local(path) => std::fs::read_to_string(path)?,
407            FilePath::Remote(url) => self.fetch_and_cache(url)?,
408            FilePath::GitHub(p) => {
409                // If there is a GITHUB_BEARER_TOKEN environment variable
410                // present, we will use that to get the download URL from the
411                // GitHub contents API.
412                let api_key = match env::var("GITHUB_BEARER_TOKEN") {
413                    Ok(api_key) => Some(api_key),
414                    Err(env::VarError::NotPresent) => None,
415                    Err(env::VarError::NotUnicode(_)) => Err(FMLError::InvalidApiToken)?,
416                };
417
418                let download_url = if let Some(api_key) = api_key {
419                    let contents_api_url = p.contents_api_url()?;
420
421                    // The response format is documented here:
422                    // https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#get-repository-content
423                    viaduct::Request::get(contents_api_url)
424                        .header("Authorization", format!("Bearer {api_key}"))?
425                        .header("User-Agent", USER_AGENT)?
426                        .send()?
427                        .require_success()?
428                        .json::<serde_json::Value>()?
429                        .get("download_url")
430                        .and_then(serde_json::Value::as_str)
431                        .ok_or_else(|| {
432                            anyhow!(
433                                "GitHub API did not return a download_url for @{}/{} at ref {}",
434                                p.repo_id(),
435                                p.path(),
436                                p.git_ref()
437                            )
438                        })
439                        .and_then(|u| Url::parse(u).map_err(Into::into))?
440                } else {
441                    p.default_download_url()?
442                };
443
444                self.fetch_and_cache(&download_url)?
445            }
446        })
447    }
448
449    pub fn read<T: serde::de::DeserializeOwned>(&self, file: &FilePath) -> Result<T> {
450        let string = self
451            .read_to_string(file)
452            .map_err(|e| FMLError::InvalidPath(format!("{file}: {e}")))?;
453
454        Ok(serde_yaml::from_str(&string)?)
455    }
456
457    fn fetch_and_cache(&self, url: &Url) -> Result<String> {
458        if !SUPPORT_URL_LOADING {
459            unimplemented!("Loading manifests from URLs is not yet supported ({})", url);
460        }
461        let path_buf = self.create_cache_path_buf(url);
462        Ok(if path_buf.exists() {
463            std::fs::read_to_string(path_buf)?
464        } else {
465            let res = viaduct::Request::get(url.clone())
466                .header("User-Agent", USER_AGENT)?
467                .send()?
468                .require_success()?;
469            let text = String::from_utf8(res.body)?;
470
471            let parent = path_buf.parent().expect("Cache directory is specified");
472            if !parent.exists() {
473                std::fs::create_dir_all(parent)?;
474            }
475
476            std::fs::write(path_buf, &text)?;
477            text
478        })
479    }
480
481    fn create_cache_path_buf(&self, url: &Url) -> PathBuf {
482        // Method to look after the cache directory.
483        // We can organize this how we want: in this case we use a flat structure
484        // with a hash of the URL as a prefix of the directory.
485        let mut hasher = DefaultHasher::new();
486        url.hash(&mut hasher);
487        let checksum = hasher.finish();
488        let filename = match url.path_segments() {
489            Some(mut segments) => segments.next_back().unwrap_or("unknown.txt"),
490            None => "unknown.txt",
491        };
492        // Take the last 16 bytes of the hash to make sure our prefixes are still random, but
493        // not crazily long.
494        let filename = format!("{:x}_{}", (checksum & 0x000000000000FFFF) as u16, filename,);
495
496        self.cache_dir().join(filename)
497    }
498
499    fn cache_dir(&self) -> &Path {
500        match &self.cache_dir {
501            Some(d) => d,
502            _ => self.tmp_cache_dir(),
503        }
504    }
505
506    fn tmp_cache_dir<'a>(&self) -> &'a Path {
507        use std::time::SystemTime;
508        lazy_static::lazy_static! {
509            static ref CACHE_DIR_NAME: String = format!("nimbus-fml-manifests-{:x}", match SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) {
510                Ok(n) => n.as_micros() & 0x00ffffff,
511                Err(_) => 0,
512            });
513
514            static ref TMP_CACHE_DIR: PathBuf = std::env::temp_dir().join(CACHE_DIR_NAME.as_str());
515        }
516        &TMP_CACHE_DIR
517    }
518
519    /// Joins a path to a string, to make a new path.
520    ///
521    /// We want to be able to support local and remote files.
522    /// We also want to be able to support a configurable short cut format.
523    /// Following a pattern common in other package managers, `@XXXX/YYYY`
524    /// is used as short hand for the main branch in github repos.
525    ///
526    /// If `f` is a relative path, the result is relative to `base`.
527    pub fn join(&self, base: &FilePath, f: &str) -> Result<FilePath> {
528        Ok(if let Some(u) = self.resolve_url_shortcut(f)? {
529            u
530        } else {
531            base.join(f)?
532        })
533    }
534
535    /// Make a new path.
536    ///
537    /// We want to be able to support local and remote files.
538    /// We also want to be able to support a configurable short cut format.
539    /// Following a pattern common in other package managers, `@XXXX/YYYY`
540    /// is used as short hand for the main branch in github repos.
541    ///
542    /// If `f` is a relative path, the result is relative to `self.cwd`.
543    pub fn file_path(&self, f: &str) -> Result<FilePath> {
544        Ok(if let Some(u) = self.resolve_url_shortcut(f)? {
545            u
546        } else {
547            FilePath::new(&self.cwd, f)?
548        })
549    }
550
551    /// Checks that the given string has a @organization/repo/ prefix.
552    /// If it does, then use that as a `repo_id` to look up the `FilePath` prefix
553    /// if it exists in the `repo_refs`, and use the `main` branch of the github repo if it
554    /// doesn't exist.
555    fn resolve_url_shortcut(&self, f: &str) -> Result<Option<FilePath>> {
556        if f.starts_with('@') {
557            let f = f.replacen('@', "", 1);
558            let parts = f.splitn(3, '/').collect::<Vec<&str>>();
559            match parts.as_slice() {
560                [user, repo, path] => {
561                    let key = format!("{}/{}", user, repo);
562                    Ok(if let Some(repo) = self.lookup_repo_path(user, repo) {
563                        Some(repo.join(path)?)
564                    } else {
565                        let repo = self.default_remote_path(key);
566                        Some(repo.join(path)?)
567                    })
568                }
569                _ => Err(FMLError::InvalidPath(format!(
570                    "'{}' needs to include a username, a repo and a filepath",
571                    f
572                ))),
573            }
574        } else {
575            Ok(None)
576        }
577    }
578
579    fn lookup_repo_path(&self, user: &str, repo: &str) -> Option<&FilePath> {
580        let key = format!("{}/{}", user, repo);
581        self.repo_refs.get(&key)
582    }
583}
584
585impl Drop for FileLoader {
586    fn drop(&mut self) {
587        if self.cache_dir.is_some() {
588            return;
589        }
590        let cache_dir = self.tmp_cache_dir();
591        if cache_dir.exists() {
592            _ = std::fs::remove_dir_all(cache_dir);
593        }
594    }
595}
596
597#[cfg(test)]
598mod unit_tests {
599    use std::fs;
600
601    use crate::util::{build_dir, pkg_dir};
602
603    use super::*;
604
605    #[test]
606    fn test_relative_paths() -> Result<()> {
607        let tmp = std::env::temp_dir();
608
609        let file = tmp.join("foo/bar.txt");
610        let obs = FilePath::from(file.as_path());
611
612        assert!(matches!(obs, FilePath::Local(_)));
613        assert!(obs.to_string().ends_with("foo/bar.txt"));
614
615        let obs = obs.join("baz.txt")?;
616        assert!(obs.to_string().ends_with("foo/baz.txt"));
617
618        let obs = obs.join("./bam.txt")?;
619        // We'd prefer it to be like this:
620        // assert!(obs.to_string().ends_with("foo/bam.txt"));
621        // But there's no easy way to get this (because symlinks).
622        // This is most likely the correct thing for us to do.
623        // We put this test here for documentation purposes, and to
624        // highlight that with URLs, ../ and ./ do what you might
625        // expect.
626        assert!(obs.to_string().ends_with("foo/./bam.txt"));
627
628        let obs = obs.join("https://example.com/foo/bar.txt")?;
629        assert!(matches!(obs, FilePath::Remote(_)));
630        assert_eq!(obs.to_string(), "https://example.com/foo/bar.txt");
631
632        let obs = obs.join("baz.txt")?;
633        assert_eq!(obs.to_string(), "https://example.com/foo/baz.txt");
634
635        let obs = obs.join("./bam.txt")?;
636        assert_eq!(obs.to_string(), "https://example.com/foo/bam.txt");
637
638        let obs = obs.join("../brum/bram.txt")?;
639        assert_eq!(obs.to_string(), "https://example.com/brum/bram.txt");
640
641        Ok(())
642    }
643
644    #[test]
645    fn test_at_shorthand_with_no_at() -> Result<()> {
646        let files = create_loader()?;
647        let cwd = FilePath::Local(files.cwd.clone());
648        let src_file = cwd.join("base/old.txt")?;
649
650        // A source file asks for a destination file relative to it.
651        let obs = files.join(&src_file, "a/file.txt")?;
652        assert!(matches!(obs, FilePath::Local(_)));
653        assert_eq!(
654            obs.to_string(),
655            format!("{}/base/a/file.txt", remove_trailing_slash(&cwd))
656        );
657        Ok(())
658    }
659
660    #[test]
661    fn test_at_shorthand_default_branch() -> Result<()> {
662        let files = create_loader()?;
663        let cwd = FilePath::Local(files.cwd.clone());
664        let src_file = cwd.join("base/old.txt")?;
665
666        // A source file asks for a file in another repo. We haven't any specific configuration
667        // for this repo, so we default to the `main` branch.
668        let obs = files.join(&src_file, "@repo/unspecified/a/file.txt")?;
669        assert!(
670            matches!(obs, FilePath::GitHub(ref gh) if gh.repo_id() == "repo/unspecified" && gh.git_ref() == "main" && gh.path() == "/a/file.txt")
671        );
672        assert_eq!(
673            obs.to_string(),
674            "https://raw.githubusercontent.com/repo/unspecified/main/a/file.txt"
675        );
676        Ok(())
677    }
678
679    #[test]
680    fn test_at_shorthand_absolute_url() -> Result<()> {
681        let mut files = create_loader()?;
682        let cwd = FilePath::Local(files.cwd.clone());
683        let src_file = cwd.join("base/old.txt")?;
684
685        // A source file asks for a file in another repo. The loader uses an absolute
686        // URL as the base URL.
687        files.add_repo("@repos/url", "https://example.com/remote/directory/path")?;
688
689        let obs = files.join(&src_file, "@repos/url/a/file.txt")?;
690        assert!(matches!(obs, FilePath::Remote(_)));
691        assert_eq!(
692            obs.to_string(),
693            "https://example.com/remote/directory/path/a/file.txt"
694        );
695
696        let obs = files.file_path("@repos/url/b/file.txt")?;
697        assert!(matches!(obs, FilePath::Remote(_)));
698        assert_eq!(
699            obs.to_string(),
700            "https://example.com/remote/directory/path/b/file.txt"
701        );
702        Ok(())
703    }
704
705    #[test]
706    fn test_at_shorthand_specified_branch() -> Result<()> {
707        let mut files = create_loader()?;
708        let cwd = FilePath::Local(files.cwd.clone());
709        let src_file = cwd.join("base/old.txt")?;
710
711        // A source file asks for a file in another repo. The loader uses the branch/tag/ref
712        // specified.
713        files.add_repo("@repos/branch", "develop")?;
714        let obs = files.join(&src_file, "@repos/branch/a/file.txt")?;
715        assert!(
716            matches!(obs, FilePath::GitHub(ref gh) if gh.repo_id() == "repos/branch" && gh.git_ref() == "develop" && gh.path() == "/a/file.txt")
717        );
718        assert_eq!(
719            obs.to_string(),
720            "https://raw.githubusercontent.com/repos/branch/develop/a/file.txt"
721        );
722
723        let obs = files.file_path("@repos/branch/b/file.txt")?;
724        assert!(
725            matches!(obs, FilePath::GitHub(ref gh) if gh.repo_id() == "repos/branch" && gh.git_ref() == "develop" && gh.path() == "/b/file.txt")
726        );
727        assert_eq!(
728            obs.to_string(),
729            "https://raw.githubusercontent.com/repos/branch/develop/b/file.txt"
730        );
731        Ok(())
732    }
733
734    #[test]
735    fn test_at_shorthand_local_development() -> Result<()> {
736        let mut files = create_loader()?;
737        let cwd = FilePath::Local(files.cwd.clone());
738        let src_file = cwd.join("base/old.txt")?;
739
740        // A source file asks for a file in another repo. The loader is configured to
741        // give a file in a directory on the local filesystem.
742        let rel_dir = "../directory/path";
743        files.add_repo("@repos/local", rel_dir)?;
744
745        let obs = files.join(&src_file, "@repos/local/a/file.txt")?;
746        assert!(matches!(obs, FilePath::Local(_)));
747        assert_eq!(
748            obs.to_string(),
749            format!("{}/{}/a/file.txt", remove_trailing_slash(&cwd), rel_dir)
750        );
751
752        let obs = files.file_path("@repos/local/b/file.txt")?;
753        assert!(matches!(obs, FilePath::Local(_)));
754        assert_eq!(
755            obs.to_string(),
756            format!("{}/{}/b/file.txt", remove_trailing_slash(&cwd), rel_dir)
757        );
758
759        Ok(())
760    }
761
762    fn create_loader() -> Result<FileLoader, FMLError> {
763        let cache_dir = PathBuf::from(format!("{}/cache", build_dir()));
764        let repo_refs = Default::default();
765        let cwd = PathBuf::from(format!("{}/fixtures/", pkg_dir()));
766        let loader = FileLoader::new(cwd, Some(cache_dir), repo_refs)?;
767        Ok(loader)
768    }
769
770    #[test]
771    fn test_at_shorthand_from_config_file() -> Result<()> {
772        let cwd = PathBuf::from(pkg_dir());
773
774        let config = &LoaderConfig {
775            cwd,
776            cache_dir: None,
777            repo_files: vec![
778                "fixtures/loaders/config_files/remote.json".to_string(),
779                "fixtures/loaders/config_files/local.yaml".to_string(),
780            ],
781            refs: Default::default(),
782        };
783
784        let files: FileLoader = config.try_into()?;
785        let cwd = FilePath::Local(files.cwd.clone());
786
787        // This is a remote repo, specified in remote.json.
788        let tfr = files.file_path("@my/remote/file.txt")?;
789        assert_eq!(
790            tfr.to_string(),
791            "https://example.com/repo/branch/file.txt".to_string()
792        );
793
794        // This is a local file, specified in local.yaml
795        let tf1 = files.file_path("@test/nested1/test-file.txt")?;
796        assert_eq!(
797            tf1.to_string(),
798            format!(
799                "{}/fixtures/loaders/config_files/./nested-1/test-file.txt",
800                &cwd
801            )
802        );
803
804        // This is a remote repo, specified in remote.json, but overridden in local.yaml
805        let tf2 = files.file_path("@test/nested2/test-file.txt")?;
806        assert_eq!(
807            tf2.to_string(),
808            format!(
809                "{}/fixtures/loaders/config_files/./nested-2/test-file.txt",
810                &cwd
811            )
812        );
813
814        let tf1 = files.read_to_string(&tf1)?;
815        let tf2 = files.read_to_string(&tf2)?;
816
817        assert_eq!("test-file/1".to_string(), tf1);
818        assert_eq!("test-file/2".to_string(), tf2);
819
820        Ok(())
821    }
822
823    fn remove_trailing_slash(cwd: &FilePath) -> String {
824        let s = cwd.to_string();
825        let mut chars = s.chars();
826        if s.ends_with('/') {
827            chars.next_back();
828        }
829        chars.as_str().to_string()
830    }
831
832    #[test]
833    fn test_at_shorthand_override_via_cli() -> Result<()> {
834        let cwd = PathBuf::from(pkg_dir());
835
836        let config = &LoaderConfig {
837            cwd,
838            cache_dir: None,
839            repo_files: Default::default(),
840            refs: BTreeMap::from([("@my-remote/repo".to_string(), "cli-branch".to_string())]),
841        };
842
843        let files: FileLoader = config.try_into()?;
844
845        // This is a file from the remote repo
846        let tfr = files.file_path("@my-remote/repo/path/to/file.txt")?;
847        assert_eq!(
848            tfr.to_string(),
849            // We're going to fetch it from the `cli-branch` of the repo.
850            "https://raw.githubusercontent.com/my-remote/repo/cli-branch/path/to/file.txt"
851                .to_string()
852        );
853
854        Ok(())
855    }
856
857    #[test]
858    fn test_dropping_tmp_cache_dir() -> Result<()> {
859        let cwd = PathBuf::from(pkg_dir());
860        let config = &LoaderConfig {
861            cwd,
862            cache_dir: None,
863            repo_files: Default::default(),
864            refs: Default::default(),
865        };
866
867        let files: FileLoader = config.try_into()?;
868        let cache_dir = files.tmp_cache_dir();
869        fs::create_dir_all(cache_dir)?;
870
871        assert!(cache_dir.exists());
872        drop(files);
873
874        assert!(!cache_dir.exists());
875        Ok(())
876    }
877
878    #[test]
879    fn test_github_repo_file_path() -> Result<()> {
880        let gh = GitHubRepoFilePath::new("owner/repo-name", "ref").join("a/file.txt")?;
881        assert_eq!(
882            gh.contents_api_url()?.to_string(),
883            "https://api.github.com/repos/owner/repo-name/contents/a/file.txt?ref=ref",
884        );
885        assert_eq!(
886            gh.default_download_url()?.to_string(),
887            "https://raw.githubusercontent.com/owner/repo-name/ref/a/file.txt"
888        );
889
890        let gh = gh.join("/b/file.txt")?;
891        assert_eq!(
892            gh.contents_api_url()?.to_string(),
893            "https://api.github.com/repos/owner/repo-name/contents/b/file.txt?ref=ref",
894        );
895        assert_eq!(
896            gh.default_download_url()?.to_string(),
897            "https://raw.githubusercontent.com/owner/repo-name/ref/b/file.txt"
898        );
899
900        let gh = gh.join("/c/")?.join("file.txt")?;
901        assert_eq!(
902            gh.contents_api_url()?.to_string(),
903            "https://api.github.com/repos/owner/repo-name/contents/c/file.txt?ref=ref",
904        );
905        assert_eq!(
906            gh.default_download_url()?.to_string(),
907            "https://raw.githubusercontent.com/owner/repo-name/ref/c/file.txt"
908        );
909
910        let gh = gh.join("d/")?.join("file.txt")?;
911        assert_eq!(
912            gh.contents_api_url()?.to_string(),
913            "https://api.github.com/repos/owner/repo-name/contents/c/d/file.txt?ref=ref",
914        );
915        assert_eq!(
916            gh.default_download_url()?.to_string(),
917            "https://raw.githubusercontent.com/owner/repo-name/ref/c/d/file.txt"
918        );
919
920        Ok(())
921    }
922
923    #[test]
924    fn test_extension() -> Result<()> {
925        let path = FilePath::Local("file.json".into());
926        assert_eq!(path.extension(), Some("json"));
927
928        let path = FilePath::Local("file.fml.yaml".into());
929        assert_eq!(path.extension(), Some("yaml"));
930
931        let path = FilePath::Local("file".into());
932        assert_eq!(path.extension(), None);
933
934        // Remote paths
935        let path = FilePath::Remote("https://example.com/file.json".try_into()?);
936        assert_eq!(path.extension(), Some("json"));
937
938        let path = FilePath::Remote("https://example.com/file.fml.yaml".try_into()?);
939        assert_eq!(path.extension(), Some("yaml"));
940
941        let path = FilePath::Remote("https://example.com/".try_into()?);
942        assert_eq!(path.extension(), None);
943
944        let path = FilePath::Remote("https://example.com/file".try_into()?);
945        assert_eq!(path.extension(), None);
946
947        let path = FilePath::Remote("https://example.com/path/".try_into()?);
948        assert_eq!(path.extension(), None);
949
950        let path = FilePath::GitHub(GitHubRepoFilePath::new("example", "main"));
951        assert_eq!(path.extension(), None);
952
953        let path = path.join("./file.json")?;
954        assert_eq!(path.extension(), Some("json"));
955
956        let path = path.join("./file.fml.yaml")?;
957        assert_eq!(path.extension(), Some("yaml"));
958
959        Ok(())
960    }
961}
nimbus_fml/util/loaders.rs

nimbus_fml/util/
loaders.rs
