Skip to content

Commit bf39964

Browse files
Alan4506Alan4506
committed
reporting refinement based on PR comments
1 parent 991e012 commit bf39964

File tree

8 files changed

+128
-115
lines changed

8 files changed

+128
-115
lines changed

‎tests/performance-tests/include/performance_tests/reporting/JsonReportingMetrics.h

Lines changed: 42 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,18 @@ struct PerformanceMetricRecord {
6262
*/
6363
class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface {
6464
public:
65+
/**
66+
* Constructor that initializes the metrics collector with configuration parameters.
67+
* @param monitoredOperations Set of operations to monitor (empty means monitor all)
68+
* @param productId Product identifier (e.g., "cpp1")
69+
* @param sdkVersion SDK version string
70+
* @param commitId Git commit identifier
71+
* @param outputFilename Path to output file (e.g., "s3-perf-results.json")
72+
*/
73+
JsonReportingMetrics(const Aws::Set<Aws::String>& monitoredOperations = Aws::Set<Aws::String>(), const Aws::String& productId = "unknown",
74+
const Aws::String& sdkVersion = "unknown", const Aws::String& commitId = "unknown",
75+
const Aws::String& outputFilename = "performance-test-results.json");
76+
6577
~JsonReportingMetrics() override;
6678

6779
/**
@@ -120,41 +132,18 @@ class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface {
120132
void OnFinish(const Aws::String& serviceName, const Aws::String& requestName,
121133
const std::shared_ptr<const Aws::Http::HttpRequest>& request, void* context) const override;
122134

123-
/**
124-
* Sets test dimensions that will be included with all performance records.
125-
* @param dimensions Vector of key-value pairs representing test dimensions (e.g., size, bucket type)
126-
*/
127-
static void SetTestContext(const Aws::Map<Aws::String, Aws::String>& dimensions);
128-
129-
/**
130-
* Registers specific operations to monitor. If empty, all operations are monitored.
131-
* @param operations Vector of operation names to track (e.g., "PutObject", "GetItem")
132-
*/
133-
static void RegisterOperationsToMonitor(const Aws::Vector<Aws::String>& operations);
134-
135-
/**
136-
* Sets product information to include in the JSON output.
137-
* @param productId Product identifier (e.g., "cpp1")
138-
* @param sdkVersion SDK version string
139-
* @param commitId Git commit identifier
140-
*/
141-
static void SetProductInfo(const Aws::String& productId, const Aws::String& sdkVersion, const Aws::String& commitId);
142-
143-
/**
144-
* Sets the output filename for the JSON performance report.
145-
* @param filename Path to output file (e.g., "s3-perf-results.json")
146-
*/
147-
static void SetOutputFilename(const Aws::String& filename);
148-
149135
private:
150136
/**
151-
* Adds a performance record for a completed AWS service operation.
137+
* Adds a performance record with a specified duration.
152138
* @param serviceName Name of the AWS service (e.g., "S3", "DynamoDB")
153139
* @param requestName Name of the operation (e.g., "PutObject", "GetItem")
154140
* @param metricsFromCore Core metrics collection containing latency data
141+
* @param request HTTP request object (optional, for extracting test metadata)
142+
* @param durationMs Duration of the request in milliseconds (default: 0)
155143
*/
156144
void AddPerformanceRecord(const Aws::String& serviceName, const Aws::String& requestName,
157-
const Aws::Monitoring::CoreMetricsCollection& metricsFromCore) const;
145+
const Aws::Monitoring::CoreMetricsCollection& metricsFromCore,
146+
const std::shared_ptr<const Aws::Http::HttpRequest>& request = nullptr, int64_t durationMs = 0) const;
158147

159148
/**
160149
* Outputs aggregated performance metrics to JSON file.
@@ -163,12 +152,11 @@ class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface {
163152
void DumpJson() const;
164153

165154
mutable Aws::Vector<PerformanceMetricRecord> m_performanceRecords;
166-
static Aws::Map<Aws::String, Aws::String> TestDimensions;
167-
static Aws::Set<Aws::String> MonitoredOperations;
168-
static Aws::String ProductId;
169-
static Aws::String SdkVersion;
170-
static Aws::String CommitId;
171-
static Aws::String OutputFilename;
155+
Aws::Set<Aws::String> m_monitoredOperations;
156+
Aws::String m_productId;
157+
Aws::String m_sdkVersion;
158+
Aws::String m_commitId;
159+
Aws::String m_outputFilename;
172160
};
173161

174162
/**
@@ -177,12 +165,32 @@ class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface {
177165
*/
178166
class JsonReportingMetricsFactory : public Aws::Monitoring::MonitoringFactory {
179167
public:
168+
/**
169+
* Constructor that initializes the factory with configuration parameters.
170+
* @param monitoredOperations Set of operations to monitor (empty means monitor all)
171+
* @param productId Product identifier (e.g., "cpp1")
172+
* @param sdkVersion SDK version string
173+
* @param commitId Git commit identifier
174+
* @param outputFilename Path to output file (e.g., "s3-perf-results.json")
175+
*/
176+
JsonReportingMetricsFactory(const Aws::Set<Aws::String>& monitoredOperations = Aws::Set<Aws::String>(),
177+
const Aws::String& productId = "unknown", const Aws::String& sdkVersion = "unknown",
178+
const Aws::String& commitId = "unknown", const Aws::String& outputFilename = "performance-test-results.json");
179+
180180
~JsonReportingMetricsFactory() override = default;
181+
181182
/**
182183
* Creates a new JsonReportingMetrics instance for performance monitoring.
183184
* @return Unique pointer to monitoring interface implementation
184185
*/
185186
Aws::UniquePtr<Aws::Monitoring::MonitoringInterface> CreateMonitoringInstance() const override;
187+
188+
private:
189+
Aws::Set<Aws::String> m_monitoredOperations;
190+
Aws::String m_productId;
191+
Aws::String m_sdkVersion;
192+
Aws::String m_commitId;
193+
Aws::String m_outputFilename;
186194
};
187195
} // namespace Reporting
188196
} // namespace PerformanceTest

‎tests/performance-tests/include/performance_tests/services/dynamodb/DynamoDBTestConfig.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,15 @@
55

66
#pragma once
77

8+
#include <aws/core/utils/memory/stl/AWSSet.h>
89
#include <aws/core/utils/memory/stl/AWSString.h>
910
#include <aws/core/utils/memory/stl/AWSVector.h>
1011

1112
namespace PerformanceTest {
1213
namespace Services {
1314
namespace DynamoDB {
1415
namespace TestConfig {
15-
const Aws::Vector<Aws::String> Operations = {"PutItem", "GetItem"};
16+
const Aws::Set<Aws::String> Operations = {"PutItem", "GetItem"};
1617

1718
const Aws::Vector<TestCase> TestMatrix = {{"8KB", 8 * 1024}, {"64KB", 64 * 1024}, {"392KB", 392 * 1024}};
1819

‎tests/performance-tests/include/performance_tests/services/s3/S3TestConfig.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,15 @@
55

66
#pragma once
77

8+
#include <aws/core/utils/memory/stl/AWSSet.h>
89
#include <aws/core/utils/memory/stl/AWSString.h>
910
#include <aws/core/utils/memory/stl/AWSVector.h>
1011

1112
namespace PerformanceTest {
1213
namespace Services {
1314
namespace S3 {
1415
namespace TestConfig {
15-
const Aws::Vector<Aws::String> Operations = {"PutObject", "GetObject"};
16+
const Aws::Set<Aws::String> Operations = {"PutObject", "GetObject"};
1617

1718
const Aws::Vector<TestCase> TestMatrix = {{"8KB", 8 * 1024, "s3-standard"}, {"64KB", 64 * 1024, "s3-standard"},
1819
{"1MB", 1024 * 1024, "s3-standard"}, {"8KB", 8 * 1024, "s3-express"},

‎tests/performance-tests/src/reporting/JsonReportingMetrics.cpp

Lines changed: 58 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -26,85 +26,96 @@
2626

2727
using namespace PerformanceTest::Reporting;
2828

29-
Aws::Map<Aws::String, Aws::String> JsonReportingMetrics::TestDimensions;
30-
Aws::Set<Aws::String> JsonReportingMetrics::MonitoredOperations;
31-
Aws::String JsonReportingMetrics::ProductId = "unknown";
32-
Aws::String JsonReportingMetrics::SdkVersion = "unknown";
33-
Aws::String JsonReportingMetrics::CommitId = "unknown";
34-
Aws::String JsonReportingMetrics::OutputFilename = "performance-test-results.json";
35-
36-
void JsonReportingMetrics::SetTestContext(const Aws::Map<Aws::String, Aws::String>& dimensions) { TestDimensions = dimensions; }
37-
38-
void JsonReportingMetrics::RegisterOperationsToMonitor(const Aws::Vector<Aws::String>& operations) {
39-
MonitoredOperations.clear();
40-
for (const auto& operation : operations) {
41-
MonitoredOperations.insert(operation);
42-
}
43-
}
44-
45-
void JsonReportingMetrics::SetProductInfo(const Aws::String& productId, const Aws::String& sdkVersion, const Aws::String& commitId) {
46-
ProductId = productId;
47-
SdkVersion = sdkVersion;
48-
CommitId = commitId;
49-
}
50-
51-
void JsonReportingMetrics::SetOutputFilename(const Aws::String& filename) { OutputFilename = filename; }
29+
struct RequestContext {
30+
Aws::Utils::DateTime requestStartTime;
31+
};
32+
33+
JsonReportingMetrics::JsonReportingMetrics(const Aws::Set<Aws::String>& monitoredOperations, const Aws::String& productId,
34+
const Aws::String& sdkVersion, const Aws::String& commitId, const Aws::String& outputFilename)
35+
: m_monitoredOperations(monitoredOperations),
36+
m_productId(productId),
37+
m_sdkVersion(sdkVersion),
38+
m_commitId(commitId),
39+
m_outputFilename(outputFilename) {}
40+
41+
JsonReportingMetricsFactory::JsonReportingMetricsFactory(const Aws::Set<Aws::String>& monitoredOperations, const Aws::String& productId,
42+
const Aws::String& sdkVersion, const Aws::String& commitId,
43+
const Aws::String& outputFilename)
44+
: m_monitoredOperations(monitoredOperations),
45+
m_productId(productId),
46+
m_sdkVersion(sdkVersion),
47+
m_commitId(commitId),
48+
m_outputFilename(outputFilename) {}
5249

5350
JsonReportingMetrics::~JsonReportingMetrics() { DumpJson(); }
5451

5552
Aws::UniquePtr<Aws::Monitoring::MonitoringInterface> JsonReportingMetricsFactory::CreateMonitoringInstance() const {
56-
return Aws::MakeUnique<JsonReportingMetrics>("JsonReportingMetrics");
53+
return Aws::MakeUnique<JsonReportingMetrics>("JsonReportingMetrics", m_monitoredOperations, m_productId, m_sdkVersion, m_commitId,
54+
m_outputFilename);
5755
}
5856

5957
void JsonReportingMetrics::AddPerformanceRecord(const Aws::String& serviceName, const Aws::String& requestName,
60-
const Aws::Monitoring::CoreMetricsCollection& metricsFromCore) const {
58+
const Aws::Monitoring::CoreMetricsCollection&,
59+
const std::shared_ptr<const Aws::Http::HttpRequest>& request, int64_t durationMs) const {
6160
// If no operations are registered, monitor all operations. Otherwise, only monitor registered operations
62-
if (!MonitoredOperations.empty() && MonitoredOperations.find(requestName) == MonitoredOperations.end()) {
61+
if (!m_monitoredOperations.empty() && m_monitoredOperations.find(requestName) == m_monitoredOperations.end()) {
6362
return;
6463
}
6564

66-
int64_t durationMs = 0;
67-
Aws::String const latencyKey = Aws::Monitoring::GetHttpClientMetricNameByType(Aws::Monitoring::HttpClientMetricsType::RequestLatency);
68-
69-
auto iterator = metricsFromCore.httpClientMetrics.find(latencyKey);
70-
if (iterator != metricsFromCore.httpClientMetrics.end()) {
71-
durationMs = iterator->second;
72-
}
73-
7465
PerformanceMetricRecord record;
7566
record.name =
7667
Aws::Utils::StringUtils::ToLower(serviceName.c_str()) + "." + Aws::Utils::StringUtils::ToLower(requestName.c_str()) + ".latency";
7768
record.description = "Time to complete " + requestName + " operation";
7869
record.unit = "Milliseconds";
7970
record.date = Aws::Utils::DateTime::Now();
8071
record.measurements.emplace_back(durationMs);
81-
record.dimensions = TestDimensions;
72+
73+
if (request) {
74+
auto headers = request->GetHeaders();
75+
for (const auto& header : headers) {
76+
if (header.first.find("test-dimension-") == 0) {
77+
Aws::String const key = header.first.substr(15);
78+
record.dimensions[key] = header.second;
79+
}
80+
}
81+
}
8282

8383
m_performanceRecords.push_back(record);
8484
}
8585

8686
void* JsonReportingMetrics::OnRequestStarted(const Aws::String&, const Aws::String&,
8787
const std::shared_ptr<const Aws::Http::HttpRequest>&) const {
88-
return nullptr;
88+
auto context = Aws::New<RequestContext>("JsonReportingMetrics");
89+
context->requestStartTime = Aws::Utils::DateTime::Now();
90+
return context;
8991
}
9092

9193
void JsonReportingMetrics::OnRequestSucceeded(const Aws::String& serviceName, const Aws::String& requestName,
92-
const std::shared_ptr<const Aws::Http::HttpRequest>&, const Aws::Client::HttpResponseOutcome&,
93-
const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void*) const {
94-
AddPerformanceRecord(serviceName, requestName, metricsFromCore);
94+
const std::shared_ptr<const Aws::Http::HttpRequest>& request,
95+
const Aws::Client::HttpResponseOutcome&,
96+
const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void* context) const {
97+
RequestContext* requestContext = static_cast<RequestContext*>(context);
98+
int64_t durationMs = (Aws::Utils::DateTime::Now() - requestContext->requestStartTime).count();
99+
AddPerformanceRecord(serviceName, requestName, metricsFromCore, request, durationMs);
95100
}
96101

97102
void JsonReportingMetrics::OnRequestFailed(const Aws::String& serviceName, const Aws::String& requestName,
98-
const std::shared_ptr<const Aws::Http::HttpRequest>&, const Aws::Client::HttpResponseOutcome&,
99-
const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void*) const {
100-
AddPerformanceRecord(serviceName, requestName, metricsFromCore);
103+
const std::shared_ptr<const Aws::Http::HttpRequest>& request,
104+
const Aws::Client::HttpResponseOutcome&,
105+
const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void* context) const {
106+
RequestContext* requestContext = static_cast<RequestContext*>(context);
107+
int64_t durationMs = (Aws::Utils::DateTime::Now() - requestContext->requestStartTime).count();
108+
AddPerformanceRecord(serviceName, requestName, metricsFromCore, request, durationMs);
101109
}
102110

103111
void JsonReportingMetrics::OnRequestRetry(const Aws::String&, const Aws::String&, const std::shared_ptr<const Aws::Http::HttpRequest>&,
104112
void*) const {}
105113

106114
void JsonReportingMetrics::OnFinish(const Aws::String&, const Aws::String&, const std::shared_ptr<const Aws::Http::HttpRequest>&,
107-
void*) const {}
115+
void* context) const {
116+
RequestContext* requestContext = static_cast<RequestContext*>(context);
117+
Aws::Delete(requestContext);
118+
}
108119

109120
void JsonReportingMetrics::DumpJson() const {
110121
if (m_performanceRecords.empty()) {
@@ -131,9 +142,9 @@ void JsonReportingMetrics::DumpJson() const {
131142

132143
// Create the JSON output
133144
Aws::Utils::Json::JsonValue root;
134-
root.WithString("productId", ProductId);
135-
root.WithString("sdkVersion", SdkVersion);
136-
root.WithString("commitId", CommitId);
145+
root.WithString("productId", m_productId);
146+
root.WithString("sdkVersion", m_sdkVersion);
147+
root.WithString("commitId", m_commitId);
137148

138149
Aws::Utils::Array<Aws::Utils::Json::JsonValue> results(aggregatedRecords.size());
139150
size_t index = 0;
@@ -175,7 +186,7 @@ void JsonReportingMetrics::DumpJson() const {
175186

176187
root.WithArray("results", std::move(results));
177188

178-
std::ofstream outFile(OutputFilename.c_str());
189+
std::ofstream outFile(m_outputFilename.c_str());
179190
if (outFile.is_open()) {
180191
outFile << root.View().WriteReadable();
181192
}

‎tests/performance-tests/src/services/dynamodb/DynamoDBPerformanceTest.cpp

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,6 @@
2929
#include <thread>
3030

3131
bool PerformanceTest::Services::DynamoDB::RunTest(Aws::DynamoDB::DynamoDBClient& dynamodb, const TestCase& config, int iterations) {
32-
Aws::Map<Aws::String, Aws::String> dimensions;
33-
dimensions["Size"] = config.sizeLabel;
34-
PerformanceTest::Reporting::JsonReportingMetrics::SetTestContext(dimensions);
35-
3632
Aws::String tableName;
3733
Aws::String const rawUUID = Aws::Utils::UUID::RandomUUID();
3834
Aws::String const tableId = Aws::Utils::StringUtils::ToLower(rawUUID.c_str()).substr(0, 8);
@@ -80,6 +76,7 @@ bool PerformanceTest::Services::DynamoDB::RunTest(Aws::DynamoDB::DynamoDBClient&
8076
for (int i = 0; i < iterations; i++) {
8177
Aws::DynamoDB::Model::PutItemRequest putItemRequest;
8278
putItemRequest.SetTableName(tableName);
79+
putItemRequest.SetAdditionalCustomHeaderValue("test-dimension-size", config.sizeLabel);
8380

8481
Aws::Map<Aws::String, Aws::DynamoDB::Model::AttributeValue> item;
8582
item["id"].SetS(Aws::String("test-key-") + Aws::Utils::StringUtils::to_string(i));
@@ -96,6 +93,7 @@ bool PerformanceTest::Services::DynamoDB::RunTest(Aws::DynamoDB::DynamoDBClient&
9693
for (int i = 0; i < iterations; i++) {
9794
Aws::DynamoDB::Model::GetItemRequest getItemRequest;
9895
getItemRequest.SetTableName(tableName);
96+
getItemRequest.SetAdditionalCustomHeaderValue("test-dimension-size", config.sizeLabel);
9997

10098
Aws::Map<Aws::String, Aws::DynamoDB::Model::AttributeValue> key;
10199
key["id"].SetS(Aws::String("test-key-") + Aws::Utils::StringUtils::to_string(i));

‎tests/performance-tests/src/services/dynamodb/main.cpp

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -33,22 +33,19 @@ int main(int argc, char** argv) {
3333
}
3434

3535
Aws::SDKOptions options;
36-
options.monitoringOptions.customizedMonitoringFactory_create_fn = {[]() -> Aws::UniquePtr<Aws::Monitoring::MonitoringFactory> {
37-
return Aws::MakeUnique<PerformanceTest::Reporting::JsonReportingMetricsFactory>("JsonReportingMetricsFactory");
36+
Aws::SDKOptions::SDKVersion const version;
37+
Aws::String const versionStr = Aws::Utils::StringUtils::to_string(static_cast<int>(version.major)) + "." +
38+
Aws::Utils::StringUtils::to_string(static_cast<int>(version.minor)) + "." +
39+
Aws::Utils::StringUtils::to_string(static_cast<int>(version.patch));
40+
41+
options.monitoringOptions.customizedMonitoringFactory_create_fn = {[&]() -> Aws::UniquePtr<Aws::Monitoring::MonitoringFactory> {
42+
return Aws::MakeUnique<PerformanceTest::Reporting::JsonReportingMetricsFactory>(
43+
"JsonReportingMetricsFactory", PerformanceTest::Services::DynamoDB::TestConfig::Operations, "cpp1", versionStr, commitId,
44+
PerformanceTest::Services::DynamoDB::TestConfig::OutputFilename);
3845
}};
3946

4047
Aws::InitAPI(options);
4148

42-
PerformanceTest::Reporting::JsonReportingMetrics::RegisterOperationsToMonitor(
43-
PerformanceTest::Services::DynamoDB::TestConfig::Operations);
44-
45-
Aws::SDKOptions::SDKVersion const version;
46-
Aws::String const versionStr = Aws::Utils::StringUtils::to_string(version.major) + "." +
47-
Aws::Utils::StringUtils::to_string(version.minor) + "." +
48-
Aws::Utils::StringUtils::to_string(version.patch);
49-
PerformanceTest::Reporting::JsonReportingMetrics::SetProductInfo("cpp1", versionStr, commitId);
50-
PerformanceTest::Reporting::JsonReportingMetrics::SetOutputFilename(PerformanceTest::Services::DynamoDB::TestConfig::OutputFilename);
51-
5249
{
5350
Aws::Client::ClientConfiguration cfg;
5451
cfg.region = region;

0 commit comments

Comments
 (0)