cloud_storage/
lib.rs

1//! This crate aims to simplify interacting with the Google Cloud Storage JSON API. Use it until
2//! Google releases a Cloud Storage Client Library for Rust. Shoutout to
3//! [MyEmma](https://myemma.io/) for funding this free and open source project.
4//!
5//! Google Cloud Storage is a product by Google that allows for cheap file storage, with a
6//! relatively sophisticated API. The idea is that storage happens in `Bucket`s, which are
7//! filesystems with a globally unique name. You can make as many of these `Bucket`s as you like!
8//!
9//! This project talks to Google using a `Service Account`. A service account is an account that you
10//! must create in the [cloud storage console](https://console.cloud.google.com/). When the account
11//! is created, you can download the file `service-account-********.json`. Store this file somewhere
12//! on your machine, and place the path to this file in the environment parameter `SERVICE_ACCOUNT`.
13//! Environment parameters declared in the `.env` file are also registered. The service account can
14//! then be granted `Roles` in the cloud storage console. The roles required for this project to
15//! function are `Service Account Token Creator` and `Storage Object Admin`.
16//!
17//! # Quickstart
18//! Add the following line to your `Cargo.toml`
19//! ```toml
20//! [dependencies]
21//! cloud-storage = "0.10"
22//! ```
23//! The two most important concepts are [Buckets](bucket/struct.Bucket.html), which represent
24//! file systems, and [Objects](object/struct.Object.html), which represent files.
25//!
26//! ## Examples:
27//! Creating a new Bucket in Google Cloud Storage:
28//! ```rust
29//! # use cloud_storage::{Client, Bucket, NewBucket};
30//! # #[tokio::main]
31//! # async fn main() -> Result<(), Box<dyn std::error::Error>> {
32//! let client = Client::default();
33//! let bucket = client.bucket().create(&NewBucket {
34//!     name: "doctest-bucket".to_string(),
35//!     ..Default::default()
36//! }).await?;
37//! # client.bucket().delete(bucket).await?;
38//! # Ok(())
39//! # }
40//! ```
41//! Connecting to an existing Bucket in Google Cloud Storage:
42//! ```no_run
43//! # use cloud_storage::{Client, Bucket};
44//! # #[tokio::main]
45//! # async fn main() -> Result<(), Box<dyn std::error::Error>> {
46//! let client = Client::default();
47//! let bucket = client.bucket().read("mybucket").await?;
48//! # Ok(())
49//! # }
50//! ```
51//! Read a file from disk and store it on googles server:
52//! ```rust,no_run
53//! # use cloud_storage::{Client, Object};
54//! # use std::fs::File;
55//! # use std::io::Read;
56//! # #[tokio::main]
57//! # async fn main() -> Result<(), Box<dyn std::error::Error>> {
58//! let mut bytes: Vec<u8> = Vec::new();
59//! for byte in File::open("myfile.txt")?.bytes() {
60//!     bytes.push(byte?)
61//! }
62//! let client = Client::default();
63//! client.object().create("mybucket", bytes, "myfile.txt", "text/plain").await?;
64//! # Ok(())
65//! # }
66//! ```
67//! Renaming/moving a file
68//! ```rust,no_run
69//! # use cloud_storage::{Client, Object};
70//! # #[tokio::main]
71//! # async fn main() -> Result<(), Box<dyn std::error::Error>> {
72//! let client = Client::default();
73//! let mut object = client.object().read("mybucket", "myfile").await?;
74//! object.content_type = Some("application/xml".to_string());
75//! client.object().update(&object).await?;
76//! # Ok(())
77//! # }
78//! ```
79//! Removing a file
80//! ```rust,no_run
81//! # use cloud_storage::{Client, Object};
82//! # #[tokio::main]
83//! # async fn main() -> Result<(), Box<dyn std::error::Error>> {
84//! let client = Client::default();
85//! client.object().delete("mybucket", "myfile").await?;
86//! # Ok(())
87//! # }
88//! ```
89#![forbid(unsafe_code, missing_docs)]
90
91pub mod client;
92#[cfg(feature = "sync")]
93pub mod sync;
94
95mod download_options;
96mod error;
97/// Contains objects as represented by Google, to be used for serialization and deserialization.
98mod resources;
99mod token;
100
101use crate::resources::service_account::ServiceAccount;
102pub use crate::{
103    client::Client,
104    error::*,
105    resources::{
106        bucket::{Bucket, NewBucket},
107        object::{ListRequest, Object},
108        *,
109    },
110    token::{Token, TokenCache},
111};
112pub use download_options::DownloadOptions;
113use tokio::sync::Mutex;
114
115lazy_static::lazy_static! {
116    static ref IAM_TOKEN_CACHE: Mutex<Token> = Mutex::new(Token::new(
117        "https://www.googleapis.com/auth/iam"
118    ));
119
120    /// The struct is the parsed service account json file. It is publicly exported to enable easier
121    /// debugging of which service account is currently used. It is of the type
122    /// [ServiceAccount](service_account/struct.ServiceAccount.html).
123    pub static ref SERVICE_ACCOUNT: ServiceAccount = ServiceAccount::get();
124}
125
126#[cfg(feature = "global-client")]
127lazy_static::lazy_static! {
128    static ref CLOUD_CLIENT: client::Client = client::Client::default();
129}
130
131/// A type alias where the error is set to be `cloud_storage::Error`.
132pub type Result<T> = std::result::Result<T, crate::Error>;
133
134const BASE_URL: &str = "https://storage.googleapis.com/storage/v1";
135
136fn from_str<'de, T, D>(deserializer: D) -> std::result::Result<T, D::Error>
137where
138    T: std::str::FromStr,
139    T::Err: std::fmt::Display,
140    D: serde::Deserializer<'de>,
141{
142    use serde::de::Deserialize;
143    let s = String::deserialize(deserializer)?;
144    T::from_str(&s).map_err(serde::de::Error::custom)
145}
146
147fn from_str_opt<'de, T, D>(deserializer: D) -> std::result::Result<Option<T>, D::Error>
148where
149    T: std::str::FromStr,
150    T::Err: std::fmt::Display,
151    D: serde::Deserializer<'de>,
152{
153    let s: std::result::Result<serde_json::Value, _> =
154        serde::Deserialize::deserialize(deserializer);
155    match s {
156        Ok(serde_json::Value::String(s)) => T::from_str(&s)
157            .map_err(serde::de::Error::custom)
158            .map(Option::from),
159        Ok(serde_json::Value::Number(num)) => T::from_str(&num.to_string())
160            .map_err(serde::de::Error::custom)
161            .map(Option::from),
162        Ok(_value) => Err(serde::de::Error::custom("Incorrect type")),
163        Err(_) => Ok(None),
164    }
165}
166
167#[cfg(all(test, feature = "global-client", feature = "sync"))]
168fn read_test_bucket_sync() -> Bucket {
169    crate::runtime().unwrap().block_on(read_test_bucket())
170}
171
172#[cfg(all(test, feature = "global-client"))]
173async fn read_test_bucket() -> Bucket {
174    dotenv::dotenv().ok();
175    let name = std::env::var("TEST_BUCKET").unwrap();
176    match Bucket::read(&name).await {
177        Ok(bucket) => bucket,
178        Err(_not_found) => Bucket::create(&NewBucket {
179            name,
180            ..NewBucket::default()
181        })
182        .await
183        .unwrap(),
184    }
185}
186
187// since all tests run in parallel, we need to make sure we do not create multiple buckets with
188// the same name in each test.
189#[cfg(all(test, feature = "global-client", feature = "sync"))]
190fn create_test_bucket_sync(name: &str) -> Bucket {
191    crate::runtime().unwrap().block_on(create_test_bucket(name))
192}
193
194// since all tests run in parallel, we need to make sure we do not create multiple buckets with
195// the same name in each test.
196#[cfg(all(test, feature = "global-client"))]
197async fn create_test_bucket(name: &str) -> Bucket {
198    std::thread::sleep(std::time::Duration::from_millis(1500)); // avoid getting rate limited
199
200    dotenv::dotenv().ok();
201    let base_name = std::env::var("TEST_BUCKET").unwrap();
202    let name = format!("{}-{}", base_name, name);
203    let new_bucket = NewBucket {
204        name,
205        ..NewBucket::default()
206    };
207    match Bucket::create(&new_bucket).await {
208        Ok(bucket) => bucket,
209        Err(_alread_exists) => Bucket::read(&new_bucket.name).await.unwrap(),
210    }
211}
212
213#[cfg(feature = "sync")]
214fn runtime() -> Result<tokio::runtime::Runtime> {
215    Ok(tokio::runtime::Builder::new_current_thread()
216        .thread_name("cloud-storage-worker")
217        .enable_time()
218        .enable_io()
219        .build()?)
220}
cloud_storage/lib.rs

cloud_storage/
lib.rs
