headless_lms_utils/file_store/
mod.rs

1//! Allows storing files to a file storage backend.
2pub mod file_utils;
3pub mod google_cloud_file_store;
4pub mod local_file_store;
5pub mod media_uploads;
6
7use std::{
8    os::unix::prelude::OsStrExt,
9    path::{Path, PathBuf},
10    pin::Pin,
11};
12
13use async_trait::async_trait;
14use bytes::Bytes;
15use futures::Stream;
16use rand::distr::SampleString;
17
18use uuid::Uuid;
19
20use crate::prelude::*;
21use headless_lms_base::config::ApplicationConfiguration;
22use headless_lms_base::jwt::{DownloadClaim, JwtKey, claimed_file_url};
23
24pub type GenericPayload = Pin<Box<dyn Stream<Item = Result<Bytes, anyhow::Error>>>>;
25/**
26Allows storing files to a file storage backend.
27*/
28#[async_trait(?Send)]
29// `Send + Sync` on the trait object, not on its futures: work that must cross threads (the spawned
30// CSV exports, the parallel seed) holds a `&dyn FileStore` across awaits, while the async methods
31// themselves stay `?Send`.
32pub trait FileStore: Send + Sync {
33    /// Upload a file that's in memory to a path.
34    async fn upload(&self, path: &Path, contents: Vec<u8>, mime_type: &str) -> UtilResult<()>;
35    /// Upload a file without loading the whole file to memory
36    async fn upload_stream(
37        &self,
38        path: &Path,
39        mut contents: GenericPayload,
40        mime_type: &str,
41    ) -> UtilResult<()>;
42    /// Download a file to memory.
43    async fn download(&self, path: &Path) -> UtilResult<Vec<u8>>;
44    /// Download a file without loading the whole file to memory.
45    async fn download_stream(
46        &self,
47        path: &Path,
48    ) -> UtilResult<Box<dyn Stream<Item = std::io::Result<Bytes>>>>;
49    /// Get a url that can be used to download the file without authentication for a while.
50    /// In most cases you probably want to use get_download_url() instead.
51    async fn get_direct_download_url(&self, path: &Path) -> UtilResult<String>;
52    /// Like [`Self::get_direct_download_url`], but browsers save the file as `file_name` instead
53    /// of displaying it.
54    async fn get_direct_attachment_download_url(
55        &self,
56        path: &Path,
57        file_name: &str,
58    ) -> UtilResult<String>;
59    /// Get a url for a file in FileStore that can be used to access the resource.
60    fn get_download_url(&self, path: &Path, app_conf: &ApplicationConfiguration) -> String {
61        format!(
62            "{}/api/v0/files/{}",
63            app_conf.base_url,
64            path.to_string_lossy()
65        )
66    }
67    /// Get a url for a file whose storage path must not be handed out, such as anything derived
68    /// from a student's answer. The claim-bearing sibling of [`Self::get_download_url`].
69    ///
70    /// The claim authorizes this one file and expires within the hour, so the url has to be minted
71    /// again on every read: it cannot be persisted, and a response carrying one cannot be cached
72    /// for another reader.
73    fn get_claimed_download_url(
74        &self,
75        file_upload_id: Uuid,
76        app_conf: &ApplicationConfiguration,
77    ) -> UtilResult<String> {
78        let jwt_key = JwtKey::new(&app_conf.jwt_password).map_err(|err| {
79            util_err!(
80                Other,
81                "Failed to build the JWT signing key.".to_string(),
82                err
83            )
84        })?;
85        claimed_file_url(
86            &app_conf.base_url,
87            &jwt_key,
88            DownloadClaim::expiring_in_1_hour(file_upload_id),
89        )
90        .map_err(|err| {
91            util_err!(
92                Other,
93                "Failed to sign a file download claim.".to_string(),
94                err
95            )
96        })
97    }
98    /// Delete a file.
99    async fn delete(&self, path: &Path) -> UtilResult<()>;
100
101    /// This function returns a path to a folder where downloaded files can be cached.
102    fn get_cache_files_folder_path(&self) -> UtilResult<&Path>;
103
104    async fn fetch_file_content_or_use_filesystem_cache(
105        &self,
106        file_path: &Path,
107    ) -> UtilResult<Vec<u8>> {
108        let cache_folder = self.get_cache_files_folder_path()?;
109        let hash = blake3::hash(file_path.as_os_str().as_bytes());
110        let cached_file_path = cache_folder.join(hash.to_hex().as_str());
111        match tokio::fs::read(&cached_file_path).await {
112            Ok(string) => return Ok(string),
113            Err(_) => {
114                info!(
115                    "File not found in cache, fetching from file store using path: {}",
116                    file_path.to_str().unwrap_or_default()
117                );
118            }
119        }
120
121        let random_filename = rand::distr::Alphanumeric.sample_string(&mut rand::rng(), 32);
122        let temp_path = cache_folder.join(random_filename.as_str());
123
124        let file_content = self.download(file_path).await?;
125
126        tokio::fs::write(&temp_path, &file_content).await?;
127        tokio::fs::rename(&temp_path, &cached_file_path).await?;
128        Ok(file_content.to_vec())
129    }
130}
131
132fn generate_cache_folder_dir() -> UtilResult<PathBuf> {
133    let cache_files_path =
134        std::env::var("HEADLESS_LMS_CACHE_FILES_PATH").map_err(|original_error| {
135            UtilError::new(
136                UtilErrorType::Other,
137                "You need to define the HEADLESS_LMS_CACHE_FILES_PATH environment variable."
138                    .to_string(),
139                Some(original_error.into()),
140            )
141        })?;
142    let path = PathBuf::from(cache_files_path).join("headlesss-lms-cached-files");
143    if !path.exists() {
144        std::fs::create_dir_all(&path)?;
145    }
146    Ok(path)
147}
148
149/// `Content-Disposition` that makes browsers save the file as `file_name`. The plain `filename`
150/// is an ASCII fallback for clients that ignore the RFC 6266 `filename*`.
151pub fn attachment_content_disposition(file_name: &str) -> String {
152    let ascii_file_name: String = file_name
153        .chars()
154        .map(|c| {
155            if c.is_ascii() && !c.is_ascii_control() && c != '"' && c != '\\' {
156                c
157            } else {
158                '_'
159            }
160        })
161        .collect();
162    let encoded_file_name =
163        percent_encoding::utf8_percent_encode(file_name, percent_encoding::NON_ALPHANUMERIC);
164    format!("attachment; filename=\"{ascii_file_name}\"; filename*=UTF-8''{encoded_file_name}")
165}
166
167fn path_to_str(path: &Path) -> UtilResult<&str> {
168    let str = path.to_str();
169    match str {
170        Some(s) => Ok(s),
171        None => Err(UtilError::new(
172            UtilErrorType::Other,
173            "Could not convert path to string because it contained invalid UTF-8 characters."
174                .to_string(),
175            None,
176        )),
177    }
178}
179
180pub fn organization_image_path(organization_id: Uuid, image_name: &str) -> UtilResult<PathBuf> {
181    let path = PathBuf::from(format!(
182        "organizations/{}/images/{}",
183        organization_id, image_name
184    ));
185    Ok(path)
186}
187
188pub fn organization_audio_path(organization_id: Uuid, audio_name: &str) -> UtilResult<PathBuf> {
189    let path = PathBuf::from(format!(
190        "organizations/{}/audios/{}",
191        organization_id, audio_name
192    ));
193    Ok(path)
194}
195
196pub fn organization_file_path(organization_id: Uuid, file_name: &str) -> UtilResult<PathBuf> {
197    let path = PathBuf::from(format!(
198        "organizations/{}/files/{}",
199        organization_id, file_name
200    ));
201    Ok(path)
202}
203
204pub fn repository_exercise_path(repository_id: Uuid, repository_exercise_id: Uuid) -> PathBuf {
205    PathBuf::from(format!(
206        "repository_exercises/{repository_id}/{repository_exercise_id}",
207    ))
208}
headless_lms_utils/file_store/mod.rs

headless_lms_utils/file_store/
mod.rs
