gcs.h
1 // Copyright 2019, Beeri 15. All rights reserved.
2 // Author: Roman Gershman (romange@gmail.com)
3 //
4 
5 #pragma once
6 
7 #include <memory>
8 
9 #include <boost/asio/ssl.hpp>
10 #include <boost/beast/core/flat_buffer.hpp>
11 #include <boost/beast/http/buffer_body.hpp>
12 #include <boost/beast/http/empty_body.hpp>
13 #include <boost/beast/http/parser.hpp>
14 
15 #include "absl/types/optional.h"
16 
17 #include "file/file.h"
18 #include "strings/stringpiece.h"
19 #include "util/gce/gce.h"
20 #include "util/status.h"
21 
22 namespace util {
23 
24 namespace http {
25 
26 class HttpsClient;
27 class HttpsClientPool;
28 
29 } // namespace http
30 
31 class IoContext;
32 
33 // Single threaded, fiber blocking class. Should be created 1 instance per http connection.
34 // All IO functions must run from IoContext thread passed to c'tor.
35 class GCS {
36  const GCE& gce_;
37  IoContext& io_context_;
38 
39  public:
43  using error_code = ::boost::system::error_code;
44 
45  GCS(const GCE& gce, ::boost::asio::ssl::context* ssl_cntx, IoContext* io_context);
46  ~GCS();
47 
48  util::Status Connect(unsigned msec);
49 
50  ListBucketResult ListBuckets();
51 
53  using ListObjectCb = std::function<void(size_t, absl::string_view)>;
54 
57  ListObjectResult List(absl::string_view bucket, absl::string_view prefix, bool fs_mode,
58  ListObjectCb cb);
59 
60  ReadObjectResult Read(absl::string_view bucket, absl::string_view path, size_t ofs,
61  const strings::MutableByteRange& range);
62 
65  static bool SplitToBucketPath(absl::string_view input, absl::string_view* bucket,
66  absl::string_view* path);
67 
69  static std::string ToGcsPath(absl::string_view bucket, absl::string_view obj_path);
70 
71  private:
72 
73  using Request = ::boost::beast::http::request<::boost::beast::http::empty_body>;
74  template <typename Body> using Response = ::boost::beast::http::response<Body>;
75  template <typename Body> using Parser = ::boost::beast::http::response_parser<Body>;
76 
77  // Parser can not be reset, so we use absl::optional to workaround.
78  using ReusableParser = absl::optional<Parser<boost::beast::http::buffer_body>>;
79 
80  util::Status RefreshToken(Request* req);
81 
82  std::string BuildGetObjUrl(absl::string_view bucket, absl::string_view path);
83  util::Status PrepareConnection();
84 
86  template <typename RespBody> util::Status SendWithToken(Request* req, Response<RespBody>* resp);
87 
88  uint32_t native_handle();
89 
90  std::string access_token_header_;
91  std::unique_ptr<http::HttpsClient> https_client_;
92 };
93 
94 bool IsGcsPath(absl::string_view path);
95 
107 StatusObject<file::WriteFile*> OpenGcsWriteFile(
108  absl::string_view full_path, const GCE& gce, http::HttpsClientPool* pool);
109 
110 
123 StatusObject<file::ReadonlyFile*> OpenGcsReadFile(
124  absl::string_view full_path, const GCE& gce, http::HttpsClientPool* pool,
125  const file::ReadonlyFile::Options& opts = file::ReadonlyFile::Options{});
126 } // namespace util
ListObjectResult List(absl::string_view bucket, absl::string_view prefix, bool fs_mode, ListObjectCb cb)
Definition: gcs.cc:192
Definition: gce.h:16
static std::string ToGcsPath(absl::string_view bucket, absl::string_view obj_path)
Inverse function to SplitToBucketPath. Returns full gcs URI that starts with "gs://"".
Definition: gcs.cc:342
std::function< void(size_t, absl::string_view)> ListObjectCb
Called with (size, key_name) pairs.
Definition: gcs.h:53
Definition: gcs.h:35
static bool SplitToBucketPath(absl::string_view input, absl::string_view *bucket, absl::string_view *path)
Definition: gcs.cc:331