headless_lms_utils/file_store/
mod.rs

1//! Allows storing files to a file storage backend.
2pub mod file_utils;
3pub mod google_cloud_file_store;
4pub mod local_file_store;
5
6use std::{
7    os::unix::prelude::OsStrExt,
8    path::{Path, PathBuf},
9    pin::Pin,
10};
11
12use async_trait::async_trait;
13use bytes::Bytes;
14use futures::Stream;
15use rand::distr::SampleString;
16
17use uuid::Uuid;
18
19use crate::prelude::*;
20use headless_lms_base::config::ApplicationConfiguration;
21use headless_lms_base::jwt::{DownloadClaim, JwtKey, claimed_file_url};
22
23pub type GenericPayload = Pin<Box<dyn Stream<Item = Result<Bytes, anyhow::Error>>>>;
24/**
25Allows storing files to a file storage backend.
26*/
27#[async_trait(?Send)]
28// `Send + Sync` on the trait object, not on its futures: work that must cross threads (the spawned
29// CSV exports, the parallel seed) holds a `&dyn FileStore` across awaits, while the async methods
30// themselves stay `?Send`.
31pub trait FileStore: Send + Sync {
32    /// Upload a file that's in memory to a path.
33    async fn upload(&self, path: &Path, contents: Vec<u8>, mime_type: &str) -> UtilResult<()>;
34    /// Upload a file without loading the whole file to memory
35    async fn upload_stream(
36        &self,
37        path: &Path,
38        mut contents: GenericPayload,
39        mime_type: &str,
40    ) -> UtilResult<()>;
41    /// Download a file to memory.
42    async fn download(&self, path: &Path) -> UtilResult<Vec<u8>>;
43    /// Download a file without loading the whole file to memory.
44    async fn download_stream(
45        &self,
46        path: &Path,
47    ) -> UtilResult<Box<dyn Stream<Item = std::io::Result<Bytes>>>>;
48    /// Get a url that can be used to download the file without authentication for a while.
49    /// In most cases you probably want to use get_download_url() instead.
50    async fn get_direct_download_url(&self, path: &Path) -> UtilResult<String>;
51    /// Get a url for a file in FileStore that can be used to access the resource.
52    fn get_download_url(&self, path: &Path, app_conf: &ApplicationConfiguration) -> String {
53        format!(
54            "{}/api/v0/files/{}",
55            app_conf.base_url,
56            path.to_string_lossy()
57        )
58    }
59    /// Get a url for a file whose storage path must not be handed out, such as anything derived
60    /// from a student's answer. The claim-bearing sibling of [`Self::get_download_url`].
61    ///
62    /// The claim authorizes this one file and expires within the hour, so the url has to be minted
63    /// again on every read: it cannot be persisted, and a response carrying one cannot be cached
64    /// for another reader.
65    fn get_claimed_download_url(
66        &self,
67        file_upload_id: Uuid,
68        app_conf: &ApplicationConfiguration,
69    ) -> UtilResult<String> {
70        let jwt_key = JwtKey::new(&app_conf.jwt_password).map_err(|err| {
71            util_err!(
72                Other,
73                "Failed to build the JWT signing key.".to_string(),
74                err
75            )
76        })?;
77        claimed_file_url(
78            &app_conf.base_url,
79            &jwt_key,
80            DownloadClaim::expiring_in_1_hour(file_upload_id),
81        )
82        .map_err(|err| {
83            util_err!(
84                Other,
85                "Failed to sign a file download claim.".to_string(),
86                err
87            )
88        })
89    }
90    /// Delete a file.
91    async fn delete(&self, path: &Path) -> UtilResult<()>;
92
93    /// This function returns a path to a folder where downloaded files can be cached.
94    fn get_cache_files_folder_path(&self) -> UtilResult<&Path>;
95
96    async fn fetch_file_content_or_use_filesystem_cache(
97        &self,
98        file_path: &Path,
99    ) -> UtilResult<Vec<u8>> {
100        let cache_folder = self.get_cache_files_folder_path()?;
101        let hash = blake3::hash(file_path.as_os_str().as_bytes());
102        let cached_file_path = cache_folder.join(hash.to_hex().as_str());
103        match tokio::fs::read(&cached_file_path).await {
104            Ok(string) => return Ok(string),
105            Err(_) => {
106                info!(
107                    "File not found in cache, fetching from file store using path: {}",
108                    file_path.to_str().unwrap_or_default()
109                );
110            }
111        }
112
113        let random_filename = rand::distr::Alphanumeric.sample_string(&mut rand::rng(), 32);
114        let temp_path = cache_folder.join(random_filename.as_str());
115
116        let file_content = self.download(file_path).await?;
117
118        tokio::fs::write(&temp_path, &file_content).await?;
119        tokio::fs::rename(&temp_path, &cached_file_path).await?;
120        Ok(file_content.to_vec())
121    }
122}
123
124fn generate_cache_folder_dir() -> UtilResult<PathBuf> {
125    let cache_files_path =
126        std::env::var("HEADLESS_LMS_CACHE_FILES_PATH").map_err(|original_error| {
127            UtilError::new(
128                UtilErrorType::Other,
129                "You need to define the HEADLESS_LMS_CACHE_FILES_PATH environment variable."
130                    .to_string(),
131                Some(original_error.into()),
132            )
133        })?;
134    let path = PathBuf::from(cache_files_path).join("headlesss-lms-cached-files");
135    if !path.exists() {
136        std::fs::create_dir_all(&path)?;
137    }
138    Ok(path)
139}
140
141fn path_to_str(path: &Path) -> UtilResult<&str> {
142    let str = path.to_str();
143    match str {
144        Some(s) => Ok(s),
145        None => Err(UtilError::new(
146            UtilErrorType::Other,
147            "Could not convert path to string because it contained invalid UTF-8 characters."
148                .to_string(),
149            None,
150        )),
151    }
152}
153
154pub fn organization_image_path(organization_id: Uuid, image_name: &str) -> UtilResult<PathBuf> {
155    let path = PathBuf::from(format!(
156        "organizations/{}/images/{}",
157        organization_id, image_name
158    ));
159    Ok(path)
160}
161
162pub fn organization_audio_path(organization_id: Uuid, audio_name: &str) -> UtilResult<PathBuf> {
163    let path = PathBuf::from(format!(
164        "organizations/{}/audios/{}",
165        organization_id, audio_name
166    ));
167    Ok(path)
168}
169
170pub fn organization_file_path(organization_id: Uuid, file_name: &str) -> UtilResult<PathBuf> {
171    let path = PathBuf::from(format!(
172        "organizations/{}/files/{}",
173        organization_id, file_name
174    ));
175    Ok(path)
176}
177
178pub fn repository_exercise_path(repository_id: Uuid, repository_exercise_id: Uuid) -> PathBuf {
179    PathBuf::from(format!(
180        "repository_exercises/{repository_id}/{repository_exercise_id}",
181    ))
182}
headless_lms_utils/file_store/mod.rs

headless_lms_utils/file_store/
mod.rs
