headless_lms_utils/file_store/
mod.rs

1//! Allows storing files to a file storage backend.
2pub mod file_utils;
3pub mod google_cloud_file_store;
4pub mod local_file_store;
5
6use std::{
7    os::unix::prelude::OsStrExt,
8    path::{Path, PathBuf},
9    pin::Pin,
10};
11
12use async_trait::async_trait;
13use bytes::Bytes;
14use futures::Stream;
15use rand::distr::SampleString;
16
17use uuid::Uuid;
18
19use crate::prelude::*;
20use headless_lms_base::config::ApplicationConfiguration;
21
22pub type GenericPayload = Pin<Box<dyn Stream<Item = Result<Bytes, anyhow::Error>>>>;
23/**
24Allows storing files to a file storage backend.
25*/
26#[async_trait(?Send)]
27// `Send + Sync` on the trait object, not on its futures: work that must cross threads (the spawned
28// CSV exports, the parallel seed) holds a `&dyn FileStore` across awaits, while the async methods
29// themselves stay `?Send`.
30pub trait FileStore: Send + Sync {
31    /// Upload a file that's in memory to a path.
32    async fn upload(&self, path: &Path, contents: Vec<u8>, mime_type: &str) -> UtilResult<()>;
33    /// Upload a file without loading the whole file to memory
34    async fn upload_stream(
35        &self,
36        path: &Path,
37        mut contents: GenericPayload,
38        mime_type: &str,
39    ) -> UtilResult<()>;
40    /// Download a file to memory.
41    async fn download(&self, path: &Path) -> UtilResult<Vec<u8>>;
42    /// Download a file without loading the whole file to memory.
43    async fn download_stream(
44        &self,
45        path: &Path,
46    ) -> UtilResult<Box<dyn Stream<Item = std::io::Result<Bytes>>>>;
47    /// Get a url that can be used to download the file without authentication for a while.
48    /// In most cases you probably want to use get_download_url() instead.
49    async fn get_direct_download_url(&self, path: &Path) -> UtilResult<String>;
50    /// Get a url for a file in FileStore that can be used to access the resource.
51    fn get_download_url(&self, path: &Path, app_conf: &ApplicationConfiguration) -> String {
52        format!(
53            "{}/api/v0/files/{}",
54            app_conf.base_url,
55            path.to_string_lossy()
56        )
57    }
58    /// Delete a file.
59    async fn delete(&self, path: &Path) -> UtilResult<()>;
60
61    /// This function returns a path to a folder where downloaded files can be cached.
62    fn get_cache_files_folder_path(&self) -> UtilResult<&Path>;
63
64    async fn fetch_file_content_or_use_filesystem_cache(
65        &self,
66        file_path: &Path,
67    ) -> UtilResult<Vec<u8>> {
68        let cache_folder = self.get_cache_files_folder_path()?;
69        let hash = blake3::hash(file_path.as_os_str().as_bytes());
70        let cached_file_path = cache_folder.join(hash.to_hex().as_str());
71        match tokio::fs::read(&cached_file_path).await {
72            Ok(string) => return Ok(string),
73            Err(_) => {
74                info!(
75                    "File not found in cache, fetching from file store using path: {}",
76                    file_path.to_str().unwrap_or_default()
77                );
78            }
79        }
80
81        let random_filename = rand::distr::Alphanumeric.sample_string(&mut rand::rng(), 32);
82        let temp_path = cache_folder.join(random_filename.as_str());
83
84        let file_content = self.download(file_path).await?;
85
86        tokio::fs::write(&temp_path, &file_content).await?;
87        tokio::fs::rename(&temp_path, &cached_file_path).await?;
88        Ok(file_content.to_vec())
89    }
90}
91
92fn generate_cache_folder_dir() -> UtilResult<PathBuf> {
93    let cache_files_path =
94        std::env::var("HEADLESS_LMS_CACHE_FILES_PATH").map_err(|original_error| {
95            UtilError::new(
96                UtilErrorType::Other,
97                "You need to define the HEADLESS_LMS_CACHE_FILES_PATH environment variable."
98                    .to_string(),
99                Some(original_error.into()),
100            )
101        })?;
102    let path = PathBuf::from(cache_files_path).join("headlesss-lms-cached-files");
103    if !path.exists() {
104        std::fs::create_dir_all(&path)?;
105    }
106    Ok(path)
107}
108
109fn path_to_str(path: &Path) -> UtilResult<&str> {
110    let str = path.to_str();
111    match str {
112        Some(s) => Ok(s),
113        None => Err(UtilError::new(
114            UtilErrorType::Other,
115            "Could not convert path to string because it contained invalid UTF-8 characters."
116                .to_string(),
117            None,
118        )),
119    }
120}
121
122pub fn organization_image_path(organization_id: Uuid, image_name: &str) -> UtilResult<PathBuf> {
123    let path = PathBuf::from(format!(
124        "organizations/{}/images/{}",
125        organization_id, image_name
126    ));
127    Ok(path)
128}
129
130pub fn organization_audio_path(organization_id: Uuid, audio_name: &str) -> UtilResult<PathBuf> {
131    let path = PathBuf::from(format!(
132        "organizations/{}/audios/{}",
133        organization_id, audio_name
134    ));
135    Ok(path)
136}
137
138pub fn organization_file_path(organization_id: Uuid, file_name: &str) -> UtilResult<PathBuf> {
139    let path = PathBuf::from(format!(
140        "organizations/{}/files/{}",
141        organization_id, file_name
142    ));
143    Ok(path)
144}
145
146pub fn repository_exercise_path(repository_id: Uuid, repository_exercise_id: Uuid) -> PathBuf {
147    PathBuf::from(format!(
148        "repository_exercises/{repository_id}/{repository_exercise_id}",
149    ))
150}
headless_lms_utils/file_store/mod.rs

headless_lms_utils/file_store/
mod.rs
