Skip to content

Commit a2df8bb

Browse files
ggcrggcr
committed
Add std::expected to S3 calls
1 parent e20be86 commit a2df8bb

File tree

4 files changed

+159
-60
lines changed

4 files changed

+159
-60
lines changed

‎main.cpp‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,24 @@
1+
#include <s3cpp/s3.h>
12

23
int main() {
3-
return 0;
4+
S3Client client("minio_access", "minio_secret");
5+
ListObjectsPaginator paginator(client, "my-bucket", "path/to/", 100);
6+
7+
int totalObjects = 0;
8+
9+
while (paginator.HasMorePages()) {
10+
std::expected<ListBucketResult, Error> page = paginator.NextPage();
11+
12+
if (!page) {
13+
std::println("Error: {}", page.error().Message);
14+
return 1;
15+
}
16+
17+
totalObjects += page->KeyCount;
18+
19+
for (const auto& obj : page->Contents) {
20+
std::println("Key: {}", obj.Key);
21+
}
22+
}
23+
return 0;
424
}

‎src/s3cpp/s3.cpp‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1+
#include <expected>
12
#include <s3cpp/s3.h>
23

3-
ListBucketResult S3Client::ListObjects(const std::string& bucket, const std::string& prefix, int maxKeys, const std::string& continuationToken) {
4+
std::expected<ListBucketResult, Error> S3Client::ListObjects(const std::string& bucket, const std::string& prefix, int maxKeys, const std::string& continuationToken) {
45
// Silent-ly accept maxKeys > 1000, even though we will return 1K at most
56
// Pagination is opt-in as in the Go SDK, the user must be aware of this
67
const std::string baseUrl = buildURL(bucket);
@@ -10,15 +11,17 @@ ListBucketResult S3Client::ListObjects(const std::string& bucket, const std::str
1011
} else {
1112
url = baseUrl + std::format("?list-type=2&prefix={}&max-keys={}", prefix, maxKeys);
1213
}
14+
1315
HttpRequest req = Client.get(url).header("Host", getHostHeader(bucket));
1416
Signer.sign(req);
1517
HttpResponse res = req.execute();
16-
ListBucketResult response = deserializeListBucketResult(Parser.parse(res.body()), maxKeys);
18+
19+
const std::vector<XMLNode>& XMLBody = Parser.parse(res.body());
20+
std::expected<ListBucketResult, Error> response = deserializeListBucketResult(XMLBody, maxKeys);
1721
return response;
1822
}
1923

20-
ListBucketResult S3Client::deserializeListBucketResult(const std::vector<XMLNode>& nodes, const int maxKeys) {
21-
// TODO(cristian): Detect and parse errors
24+
std::expected<ListBucketResult, Error> S3Client::deserializeListBucketResult(const std::vector<XMLNode>& nodes, const int maxKeys) {
2225
ListBucketResult response;
2326
response.Contents.reserve(maxKeys);
2427
response.CommonPrefixes.reserve(maxKeys);
@@ -98,6 +101,10 @@ ListBucketResult S3Client::deserializeListBucketResult(const std::vector<XMLNode
98101
} else if (node.tag == "ListBucketResult.Contents.StorageClass") {
99102
response.Contents[contentsIdx].StorageClass = std::move(node.value);
100103
} else {
104+
// Detect and parse error
105+
if (node.tag == "Error.Code") {
106+
return std::unexpected<Error>(deserializeError(nodes));
107+
}
101108
throw std::runtime_error(std::format("No case for ListBucketResult response found for: {}", node.tag));
102109
}
103110

@@ -119,3 +126,27 @@ ListBucketResult S3Client::deserializeListBucketResult(const std::vector<XMLNode
119126

120127
return response;
121128
}
129+
130+
Error S3Client::deserializeError(const std::vector<XMLNode>& nodes) {
131+
Error error;
132+
133+
for (const auto& node : nodes) {
134+
/* Sigh... no reflection */
135+
136+
if (node.tag == "Error.Code") {
137+
error.Code = std::move(node.value);
138+
} else if (node.tag == "Error.Message") {
139+
error.Message = std::move(node.value);
140+
} else if (node.tag == "Error.Resource") {
141+
error.Resource = std::move(node.value);
142+
} else if (node.tag == "Error.RequestId") {
143+
error.RequestId = Parser.parseNumber<int>(std::move(node.value));
144+
} else if (node.tag == "Error.HostId") {
145+
error.HostId = std::move(node.value);
146+
} else {
147+
throw std::runtime_error(std::format("No case for Error response found for: {}", node.tag));
148+
}
149+
}
150+
151+
return error;
152+
}

‎src/s3cpp/s3.h‎

Lines changed: 37 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#include <charconv>
2+
#include <expected>
23
#include <print>
34
#include <s3cpp/auth.h>
45
#include <s3cpp/xml.hpp>
@@ -47,6 +48,14 @@ struct ListBucketResult {
4748
std::string StartAfter;
4849
};
4950

51+
struct Error {
52+
std::string Code;
53+
std::string Message;
54+
std::string Resource;
55+
int RequestId;
56+
std::string HostId;
57+
};
58+
5059
enum class S3AddressingStyle {
5160
VirtualHosted,
5261
PathStyle
@@ -79,28 +88,30 @@ class S3Client {
7988
, addressing_style_(style) {
8089
}
8190

82-
// TODO(cristian): Implement deserialization
83-
// TODO(cristian): Wrap onto std::expected
84-
ListBucketResult GetObject(const std::string& bucket, const std::string& key) {
85-
std::string url = buildURL(bucket) + std::format("/{}", key);
86-
HttpRequest req = Client.get(url).header("Host", getHostHeader(bucket));
87-
Signer.sign(req);
88-
HttpResponse res = req.execute();
89-
std::println("{}", res.body());
90-
ListBucketResult response = deserializeListBucketResult(Parser.parse(res.body()), 1000);
91-
return response;
92-
}
93-
94-
ListBucketResult ListObjects(const std::string& bucket) { return ListObjects(bucket, "/", 1000, ""); }
95-
ListBucketResult ListObjects(const std::string& bucket, const std::string& prefix) { return ListObjects(bucket, prefix, 1000, ""); }
96-
ListBucketResult ListObjects(const std::string& bucket, const std::string& prefix, int maxKeys) { return ListObjects(bucket, prefix, maxKeys, ""); }
97-
ListBucketResult ListObjects(const std::string& bucket, const std::string& prefix, int maxKeys, const std::string& continuationToken);
98-
99-
ListBucketResult deserializeListBucketResult(const std::vector<XMLNode>& nodes, const int maxKeys);
91+
// TODO(cristian): Implement deserialization
92+
// TODO(cristian): Wrap onto std::expected
93+
std::expected<ListBucketResult, Error> GetObject(const std::string& bucket, const std::string& key) {
94+
std::string url = buildURL(bucket) + std::format("/{}", key);
95+
HttpRequest req = Client.get(url).header("Host", getHostHeader(bucket));
96+
Signer.sign(req);
97+
HttpResponse res = req.execute();
98+
std::println("{}", res.body());
99+
std::expected<ListBucketResult, Error> response = deserializeListBucketResult(Parser.parse(res.body()), 1000);
100+
return response;
101+
}
102+
// TODO(cristian): HeadBucket and HeadObject
103+
104+
std::expected<ListBucketResult, Error> ListObjects(const std::string& bucket) { return ListObjects(bucket, "/", 1000, ""); }
105+
std::expected<ListBucketResult, Error> ListObjects(const std::string& bucket, const std::string& prefix) { return ListObjects(bucket, prefix, 1000, ""); }
106+
std::expected<ListBucketResult, Error> ListObjects(const std::string& bucket, const std::string& prefix, int maxKeys) { return ListObjects(bucket, prefix, maxKeys, ""); }
107+
std::expected<ListBucketResult, Error> ListObjects(const std::string& bucket, const std::string& prefix, int maxKeys, const std::string& continuationToken);
108+
109+
std::expected<ListBucketResult, Error> deserializeListBucketResult(const std::vector<XMLNode>& nodes, const int maxKeys);
110+
Error deserializeError(const std::vector<XMLNode>& nodes);
100111

101112
private:
102-
HttpClient Client;
103-
AWSSigV4Signer Signer;
113+
HttpClient Client;
114+
AWSSigV4Signer Signer;
104115
XMLParser Parser;
105116
std::string endpoint_;
106117
S3AddressingStyle addressing_style_;
@@ -139,10 +150,12 @@ class ListObjectsPaginator {
139150

140151
bool HasMorePages() const { return hasMorePages_; }
141152

142-
ListBucketResult NextPage() {
143-
ListBucketResult response = client_.ListObjects(bucket_, prefix_, maxKeys_, continuationToken_);
144-
hasMorePages_ = response.IsTruncated;
145-
continuationToken_ = response.NextContinuationToken;
153+
std::expected<ListBucketResult, Error> NextPage() {
154+
auto response = client_.ListObjects(bucket_, prefix_, maxKeys_, continuationToken_);
155+
if (response.has_value()) {
156+
hasMorePages_ = response.value().IsTruncated;
157+
continuationToken_ = response.value().NextContinuationToken;
158+
}
146159
return response;
147160
}
148161

‎test/s3_test.cpp‎

Lines changed: 66 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,32 @@
11
#include "gtest/gtest.h"
22
#include <s3cpp/s3.h>
33

4-
TEST(S3, ListObjectsNoPrefix) {
4+
TEST(S3, ListObjectsBucket) {
55
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
66
try {
77
// Assuming the bucket has the 10K objects
88
// Once we implement PutObject we will do this ourselves with s3cpp
9-
ListBucketResult res = client.ListObjects("my-bucket");
10-
EXPECT_EQ(res.Contents.size(), 0);
9+
std::expected<ListBucketResult, Error> res = client.ListObjects("my-bucket");
10+
if (!res)
11+
GTEST_FAIL();
12+
EXPECT_EQ(res->Contents.size(), 0);
13+
} catch (const std::exception& e) {
14+
const std::string emsg = e.what();
15+
if (emsg == "libcurl error: Could not connect to server" || emsg == "libcurl error: Couldn't connect to server") {
16+
GTEST_SKIP_("Skipping MinIOBasicRequest: Server not up");
17+
}
18+
throw;
19+
}
20+
}
21+
22+
TEST(S3, ListObjectsBucketNotExists) {
23+
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
24+
try {
25+
std::expected<ListBucketResult, Error> res = client.ListObjects("Does-not-exist");
26+
if (res.has_value()) // We must return error
27+
GTEST_FAIL();
28+
Error error = res.error();
29+
// EXPECT_EQ(error.Code, "InvalidBucketName");
1130
} catch (const std::exception& e) {
1231
const std::string emsg = e.what();
1332
if (emsg == "libcurl error: Could not connect to server" || emsg == "libcurl error: Couldn't connect to server") {
@@ -21,8 +40,10 @@ TEST(S3, ListObjectsFilePrefix) {
2140
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
2241
try {
2342
// path/to/file_1.txt must exist
24-
ListBucketResult res = client.ListObjects("my-bucket", "path/to/file_1.txt");
25-
EXPECT_EQ(res.Contents.size(), 1);
43+
std::expected<ListBucketResult, Error> res = client.ListObjects("my-bucket", "path/to/file_1.txt");
44+
if (!res)
45+
GTEST_FAIL();
46+
EXPECT_EQ(res->Contents.size(), 1);
2647
} catch (const std::exception& e) {
2748
const std::string emsg = e.what();
2849
if (emsg == "libcurl error: Could not connect to server" || emsg == "libcurl error: Couldn't connect to server") {
@@ -35,9 +56,11 @@ TEST(S3, ListObjectsFilePrefix) {
3556
TEST(S3, ListObjectsDirPrefix) {
3657
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
3758
try {
38-
// Get 100 keys
39-
ListBucketResult res = client.ListObjects("my-bucket", "path/to/", 100);
40-
EXPECT_EQ(res.Contents.size(), 100);
59+
// Get 100 keys
60+
std::expected<ListBucketResult, Error> res = client.ListObjects("my-bucket", "path/to/", 100);
61+
if (!res)
62+
GTEST_FAIL();
63+
EXPECT_EQ(res->Contents.size(), 100);
4164
} catch (const std::exception& e) {
4265
const std::string emsg = e.what();
4366
if (emsg == "libcurl error: Could not connect to server" || emsg == "libcurl error: Couldn't connect to server") {
@@ -50,8 +73,10 @@ TEST(S3, ListObjectsDirPrefix) {
5073
TEST(S3, ListObjectsDirPrefixMaxKeys) {
5174
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
5275
try {
53-
ListBucketResult res = client.ListObjects("my-bucket", "path/to/", 1);
54-
EXPECT_EQ(res.Contents.size(), 1);
76+
std::expected<ListBucketResult, Error> res = client.ListObjects("my-bucket", "path/to/", 1);
77+
if (!res)
78+
GTEST_FAIL();
79+
EXPECT_EQ(res->Contents.size(), 1);
5580
} catch (const std::exception& e) {
5681
const std::string emsg = e.what();
5782
if (emsg == "libcurl error: Could not connect to server" || emsg == "libcurl error: Couldn't connect to server") {
@@ -64,24 +89,26 @@ TEST(S3, ListObjectsDirPrefixMaxKeys) {
6489
TEST(S3, ListObjectsCheckFields) {
6590
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
6691
try {
67-
ListBucketResult res = client.ListObjects("my-bucket", "path/to/", 2);
92+
std::expected<ListBucketResult, Error> res = client.ListObjects("my-bucket", "path/to/", 2);
93+
if (!res)
94+
GTEST_FAIL();
6895

69-
EXPECT_EQ(res.Name, "my-bucket");
70-
EXPECT_EQ(res.Prefix, "path/to/");
71-
EXPECT_EQ(res.MaxKeys, 2);
72-
EXPECT_EQ(res.IsTruncated, true);
73-
EXPECT_FALSE(res.NextContinuationToken.empty());
96+
EXPECT_EQ(res->Name, "my-bucket");
97+
EXPECT_EQ(res->Prefix, "path/to/");
98+
EXPECT_EQ(res->MaxKeys, 2);
99+
EXPECT_EQ(res->IsTruncated, true);
100+
EXPECT_FALSE(res->NextContinuationToken.empty());
74101

75102
// Should have exactly 2 keys
76-
EXPECT_EQ(res.Contents.size(), 2);
103+
EXPECT_EQ(res->Contents.size(), 2);
77104

78-
EXPECT_EQ(res.Contents[0].Key, "path/to/file_1.txt");
79-
EXPECT_EQ(res.Contents[0].Size, 26);
80-
EXPECT_EQ(res.Contents[0].StorageClass, "STANDARD");
105+
EXPECT_EQ(res->Contents[0].Key, "path/to/file_1.txt");
106+
EXPECT_EQ(res->Contents[0].Size, 26);
107+
EXPECT_EQ(res->Contents[0].StorageClass, "STANDARD");
81108

82-
EXPECT_EQ(res.Contents[1].Key, "path/to/file_10.txt");
83-
EXPECT_EQ(res.Contents[1].Size, 27);
84-
EXPECT_EQ(res.Contents[1].StorageClass, "STANDARD");
109+
EXPECT_EQ(res->Contents[1].Key, "path/to/file_10.txt");
110+
EXPECT_EQ(res->Contents[1].Size, 27);
111+
EXPECT_EQ(res->Contents[1].StorageClass, "STANDARD");
85112

86113
} catch (const std::exception& e) {
87114
const std::string emsg = e.what();
@@ -96,8 +123,10 @@ TEST(S3, ListObjectsCheckLenKeys) {
96123
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
97124
try {
98125
// has 10K objects - limit is 1000 keys
99-
ListBucketResult response = client.ListObjects("my-bucket", "path/to/");
100-
EXPECT_EQ(response.Contents.size(), 1000);
126+
std::expected<ListBucketResult, Error> res = client.ListObjects("my-bucket", "path/to/");
127+
if (!res)
128+
GTEST_FAIL();
129+
EXPECT_EQ(res->Contents.size(), 1000);
101130
} catch (const std::exception& e) {
102131
const std::string emsg = e.what();
103132
if (emsg == "libcurl error: Could not connect to server" || emsg == "libcurl error: Couldn't connect to server") {
@@ -117,14 +146,17 @@ TEST(S3, ListObjectsPaginator) {
117146
int pageCount = 0;
118147

119148
while (paginator.HasMorePages()) {
120-
ListBucketResult page = paginator.NextPage();
121-
totalObjects += page.Contents.size();
122-
if (page.Contents.size() > 0)
149+
std::expected<ListBucketResult, Error> page = paginator.NextPage();
150+
if (!page) {
151+
GTEST_FAIL();
152+
}
153+
totalObjects += page->Contents.size();
154+
if (page->Contents.size() > 0)
123155
pageCount++;
124156

125157
if (paginator.HasMorePages()) {
126-
EXPECT_EQ(page.Contents.size(), 100);
127-
EXPECT_TRUE(page.IsTruncated);
158+
EXPECT_EQ(page->Contents.size(), 100);
159+
EXPECT_TRUE(page->IsTruncated);
128160
}
129161
}
130162

@@ -142,7 +174,10 @@ TEST(S3, ListObjectsPaginator) {
142174
TEST(S3, GetObjectExists) {
143175
S3Client client("minio_access", "minio_secret", "127.0.0.1:9000", S3AddressingStyle::PathStyle);
144176
try {
145-
ListBucketResult response = client.GetObject("my-bucket", "path/to/file_1.txt");
177+
auto response = client.GetObject("my-bucket", "path/to/file_1.txt");
178+
if (!response) {
179+
GTEST_FAIL();
180+
}
146181
} catch (const std::exception& e) {
147182
const std::string emsg = e.what();
148183
if (emsg == "libcurl error: Could not connect to server" || emsg == "libcurl error: Couldn't connect to server") {

0 commit comments

Comments
 (0)