From c68520306113947cbfbcb1c2928f8581fcd5b490 Mon Sep 17 00:00:00 2001 From: YUXUAN CHEN Date: Tue, 24 Jun 2025 16:23:07 -0400 Subject: [PATCH 1/3] implement the monitoring interface for performance tests --- .../reporting/JsonReportingMetrics.h | 165 ++++++++++++++++ .../src/reporting/JsonReportingMetrics.cpp | 179 ++++++++++++++++++ 2 files changed, 344 insertions(+) create mode 100644 tests/performance-tests/include/performance_tests/reporting/JsonReportingMetrics.h create mode 100644 tests/performance-tests/src/reporting/JsonReportingMetrics.cpp diff --git a/tests/performance-tests/include/performance_tests/reporting/JsonReportingMetrics.h b/tests/performance-tests/include/performance_tests/reporting/JsonReportingMetrics.h new file mode 100644 index 00000000000..2191c7851a6 --- /dev/null +++ b/tests/performance-tests/include/performance_tests/reporting/JsonReportingMetrics.h @@ -0,0 +1,165 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace PerformanceTest { +namespace Reporting { +/** + * Container for a single performance metric record that stores measurement data and associated metadata. + */ +struct PerformanceMetricRecord { + Aws::String name; + Aws::String description; + Aws::String unit; + int64_t date; + Aws::Vector measurements; + Aws::Vector> dimensions; +}; + +/** + * An implementation of the MonitoringInterface that collects performance metrics + * and reports them in a JSON format. + */ +class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface { + public: + ~JsonReportingMetrics() override; + + /** + * Called when an AWS request is started. Returns context for tracking. + * @param serviceName Name of the AWS service + * @param requestName Name of the operation + * @param request HTTP request object + * @return Context pointer (always returns nullptr) + */ + void* OnRequestStarted(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request) const override; + + /** + * Called when an AWS request succeeds. Records performance metrics. + * @param serviceName Name of the AWS service + * @param requestName Name of the operation + * @param request HTTP request object + * @param outcome HTTP response outcome + * @param metrics Core metrics collection containing latency data + * @param context Request context (unused) + */ + void OnRequestSucceeded(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request, const Aws::Client::HttpResponseOutcome& outcome, + const Aws::Monitoring::CoreMetricsCollection& metrics, void* context) const override; + + /** + * Called when an AWS request fails. Records performance metrics. + * @param serviceName Name of the AWS service + * @param requestName Name of the operation + * @param request HTTP request object + * @param outcome HTTP response outcome + * @param metrics Core metrics collection containing latency data + * @param context Request context (unused) + */ + void OnRequestFailed(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request, const Aws::Client::HttpResponseOutcome& outcome, + const Aws::Monitoring::CoreMetricsCollection& metrics, void* context) const override; + + /** + * Called when an AWS request is retried. No action taken. + * @param serviceName Name of the AWS service + * @param requestName Name of the operation + * @param request HTTP request object + * @param context Request context (unused) + */ + void OnRequestRetry(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request, void* context) const override; + + /** + * Called when an AWS request finishes. No action taken. + * @param serviceName Name of the AWS service + * @param requestName Name of the operation + * @param request HTTP request object + * @param context Request context (unused) + */ + void OnFinish(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request, void* context) const override; + + /** + * Sets test dimensions that will be included with all performance records. + * @param dimensions Vector of key-value pairs representing test dimensions (e.g., size, bucket type) + */ + static void SetTestContext(const Aws::Vector>& dimensions); + + /** + * Registers specific operations to monitor. If empty, all operations are monitored. + * @param operations Vector of operation names to track (e.g., "PutObject", "GetItem") + */ + static void RegisterOperationsToMonitor(const Aws::Vector& operations); + + /** + * Sets product information to include in the JSON output. + * @param productId Product identifier (e.g., "cpp1") + * @param sdkVersion SDK version string + * @param commitId Git commit identifier + */ + static void SetProductInfo(const Aws::String& productId, const Aws::String& sdkVersion, const Aws::String& commitId); + + /** + * Sets the output filename for the JSON performance report. + * @param filename Path to output file (e.g., "s3-perf-results.json") + */ + static void SetOutputFilename(const Aws::String& filename); + + private: + /** + * Adds a performance record for a completed AWS service operation. + * @param serviceName Name of the AWS service (e.g., "S3", "DynamoDB") + * @param requestName Name of the operation (e.g., "PutObject", "GetItem") + * @param metricsFromCore Core metrics collection containing latency data + */ + void AddPerformanceRecord(const Aws::String& serviceName, const Aws::String& requestName, + const Aws::Monitoring::CoreMetricsCollection& metricsFromCore) const; + + /** + * Outputs aggregated performance metrics to JSON file. + * Groups records by name and dimensions, then writes to configured output file. + */ + void DumpJson() const; + + mutable Aws::Vector m_performanceRecords; + static Aws::Vector> TestDimensions; + static Aws::Set MonitoredOperations; + static Aws::String ProductId; + static Aws::String SdkVersion; + static Aws::String CommitId; + static Aws::String OutputFilename; +}; + +/** + * A factory for creating instances of JsonReportingMetrics. + * Used by the AWS SDK monitoring system to instantiate performance metrics collectors. + */ +class JsonReportingMetricsFactory : public Aws::Monitoring::MonitoringFactory { + public: + ~JsonReportingMetricsFactory() override = default; + /** + * Creates a new JsonReportingMetrics instance for performance monitoring. + * @return Unique pointer to monitoring interface implementation + */ + Aws::UniquePtr CreateMonitoringInstance() const override; +}; +} // namespace Reporting +} // namespace PerformanceTest \ No newline at end of file diff --git a/tests/performance-tests/src/reporting/JsonReportingMetrics.cpp b/tests/performance-tests/src/reporting/JsonReportingMetrics.cpp new file mode 100644 index 00000000000..a1a46f0035e --- /dev/null +++ b/tests/performance-tests/src/reporting/JsonReportingMetrics.cpp @@ -0,0 +1,179 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace PerformanceTest::Reporting; + +Aws::Vector> JsonReportingMetrics::TestDimensions; +Aws::Set JsonReportingMetrics::MonitoredOperations; +Aws::String JsonReportingMetrics::ProductId = "unknown"; +Aws::String JsonReportingMetrics::SdkVersion = "unknown"; +Aws::String JsonReportingMetrics::CommitId = "unknown"; +Aws::String JsonReportingMetrics::OutputFilename = "performance-test-results.json"; + +void JsonReportingMetrics::SetTestContext(const Aws::Vector>& dimensions) { + TestDimensions = dimensions; +} + +void JsonReportingMetrics::RegisterOperationsToMonitor(const Aws::Vector& operations) { + MonitoredOperations.clear(); + for (const auto& operation : operations) { + MonitoredOperations.insert(operation); + } +} + +void JsonReportingMetrics::SetProductInfo(const Aws::String& productId, const Aws::String& sdkVersion, const Aws::String& commitId) { + ProductId = productId; + SdkVersion = sdkVersion; + CommitId = commitId; +} + +void JsonReportingMetrics::SetOutputFilename(const Aws::String& filename) { OutputFilename = filename; } + +JsonReportingMetrics::~JsonReportingMetrics() { DumpJson(); } + +Aws::UniquePtr JsonReportingMetricsFactory::CreateMonitoringInstance() const { + return Aws::MakeUnique("JsonReportingMetrics"); +} + +void JsonReportingMetrics::AddPerformanceRecord(const Aws::String& serviceName, const Aws::String& requestName, + const Aws::Monitoring::CoreMetricsCollection& metricsFromCore) const { + // If no operations are registered, monitor all operations. Otherwise, only monitor registered operations + if (!MonitoredOperations.empty() && MonitoredOperations.find(requestName) == MonitoredOperations.end()) { + return; + } + + int64_t durationMs = 0; + Aws::String const latencyKey = Aws::Monitoring::GetHttpClientMetricNameByType(Aws::Monitoring::HttpClientMetricsType::RequestLatency); + + auto iterator = metricsFromCore.httpClientMetrics.find(latencyKey); + if (iterator != metricsFromCore.httpClientMetrics.end()) { + durationMs = iterator->second; + } + + PerformanceMetricRecord record; + record.name = + Aws::Utils::StringUtils::ToLower(serviceName.c_str()) + "." + Aws::Utils::StringUtils::ToLower(requestName.c_str()) + ".latency"; + record.description = "Time to complete " + requestName + " operation"; + record.unit = "Milliseconds"; + record.date = Aws::Utils::DateTime::Now().Seconds(); + record.measurements.push_back(durationMs); + record.dimensions = TestDimensions; + + m_performanceRecords.push_back(record); +} + +void* JsonReportingMetrics::OnRequestStarted(const Aws::String&, const Aws::String&, + const std::shared_ptr&) const { + return nullptr; +} + +void JsonReportingMetrics::OnRequestSucceeded(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr&, const Aws::Client::HttpResponseOutcome&, + const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void*) const { + AddPerformanceRecord(serviceName, requestName, metricsFromCore); +} + +void JsonReportingMetrics::OnRequestFailed(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr&, const Aws::Client::HttpResponseOutcome&, + const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void*) const { + AddPerformanceRecord(serviceName, requestName, metricsFromCore); +} + +void JsonReportingMetrics::OnRequestRetry(const Aws::String&, const Aws::String&, const std::shared_ptr&, + void*) const {} + +void JsonReportingMetrics::OnFinish(const Aws::String&, const Aws::String&, const std::shared_ptr&, + void*) const {} + +void JsonReportingMetrics::DumpJson() const { + if (m_performanceRecords.empty()) { + return; + } + + // Group performance records by name and dimensions + Aws::Map aggregatedRecords; + + for (const auto& record : m_performanceRecords) { + Aws::String key = record.name; + for (const auto& dim : record.dimensions) { + key += ":" + Aws::String(dim.first) + "=" + Aws::String(dim.second); + } + + if (aggregatedRecords.find(key) == aggregatedRecords.end()) { + aggregatedRecords[key] = record; + } else { + for (const auto& measurement : record.measurements) { + aggregatedRecords[key].measurements.push_back(measurement); + } + } + } + + // Create the JSON output + Aws::Utils::Json::JsonValue root; + root.WithString("productId", ProductId); + root.WithString("sdkVersion", SdkVersion); + root.WithString("commitId", CommitId); + + Aws::Utils::Array results(aggregatedRecords.size()); + size_t index = 0; + + for (const auto& pair : aggregatedRecords) { + const auto& record = pair.second; + Aws::Utils::Json::JsonValue jsonMetric; + jsonMetric.WithString("name", record.name); + jsonMetric.WithString("description", record.description); + jsonMetric.WithString("unit", record.unit); + jsonMetric.WithInt64("date", record.date); + + if (!record.dimensions.empty()) { + Aws::Utils::Array dimensionsArray(record.dimensions.size()); + for (size_t j = 0; j < record.dimensions.size(); ++j) { + Aws::Utils::Json::JsonValue dimension; + dimension.WithString("name", record.dimensions[j].first); + dimension.WithString("value", record.dimensions[j].second); + dimensionsArray[j] = std::move(dimension); + } + jsonMetric.WithArray("dimensions", std::move(dimensionsArray)); + } + + Aws::Utils::Array measurementsArray(record.measurements.size()); + for (size_t j = 0; j < record.measurements.size(); ++j) { + Aws::Utils::Json::JsonValue measurementValue; + measurementValue.AsInt64(record.measurements[j]); + measurementsArray[j] = std::move(measurementValue); + } + jsonMetric.WithArray("measurements", std::move(measurementsArray)); + + results[index++] = std::move(jsonMetric); + } + + root.WithArray("results", std::move(results)); + + std::ofstream outFile(OutputFilename.c_str()); + if (outFile.is_open()) { + outFile << root.View().WriteReadable(); + } +} \ No newline at end of file From 178831bb8675995c2bc319d976471a229479b9dc Mon Sep 17 00:00:00 2001 From: YUXUAN CHEN Date: Fri, 27 Jun 2025 14:57:51 -0400 Subject: [PATCH 2/3] address issues from PR comments --- .../reporting/JsonReportingMetrics.h | 115 +++++++----- .../src/reporting/JsonReportingMetrics.cpp | 165 +++++++++++------- 2 files changed, 175 insertions(+), 105 deletions(-) rename tests/performance-tests/include/{performance_tests => performance-tests}/reporting/JsonReportingMetrics.h (60%) diff --git a/tests/performance-tests/include/performance_tests/reporting/JsonReportingMetrics.h b/tests/performance-tests/include/performance-tests/reporting/JsonReportingMetrics.h similarity index 60% rename from tests/performance-tests/include/performance_tests/reporting/JsonReportingMetrics.h rename to tests/performance-tests/include/performance-tests/reporting/JsonReportingMetrics.h index 2191c7851a6..dbd76e60890 100644 --- a/tests/performance-tests/include/performance_tests/reporting/JsonReportingMetrics.h +++ b/tests/performance-tests/include/performance-tests/reporting/JsonReportingMetrics.h @@ -10,17 +10,21 @@ #include #include #include +#include +#include #include +#include #include #include #include #include #include -#include +#include namespace PerformanceTest { namespace Reporting { + /** * Container for a single performance metric record that stores measurement data and associated metadata. */ @@ -28,9 +32,9 @@ struct PerformanceMetricRecord { Aws::String name; Aws::String description; Aws::String unit; - int64_t date; - Aws::Vector measurements; - Aws::Vector> dimensions; + Aws::Utils::DateTime date; + Aws::Vector> measurements; + Aws::Map dimensions; }; /** @@ -39,6 +43,18 @@ struct PerformanceMetricRecord { */ class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface { public: + /** + * Constructor that initializes the metrics collector with configuration parameters. + * @param monitoredOperations Set of operations to monitor (empty means monitor all) + * @param productId Product identifier (e.g., "cpp1") + * @param sdkVersion SDK version string + * @param commitId Git commit identifier + * @param outputFilename Path to output file (e.g., "s3-perf-results.json") + */ + JsonReportingMetrics(const Aws::Set& monitoredOperations = Aws::Set(), const Aws::String& productId = "unknown", + const Aws::String& sdkVersion = "unknown", const Aws::String& commitId = "unknown", + const Aws::String& outputFilename = "performance-test-results.json"); + ~JsonReportingMetrics() override; /** @@ -58,7 +74,7 @@ class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface { * @param request HTTP request object * @param outcome HTTP response outcome * @param metrics Core metrics collection containing latency data - * @param context Request context (unused) + * @param context Request context */ void OnRequestSucceeded(const Aws::String& serviceName, const Aws::String& requestName, const std::shared_ptr& request, const Aws::Client::HttpResponseOutcome& outcome, @@ -71,7 +87,7 @@ class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface { * @param request HTTP request object * @param outcome HTTP response outcome * @param metrics Core metrics collection containing latency data - * @param context Request context (unused) + * @param context Request context */ void OnRequestFailed(const Aws::String& serviceName, const Aws::String& requestName, const std::shared_ptr& request, const Aws::Client::HttpResponseOutcome& outcome, @@ -82,7 +98,7 @@ class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface { * @param serviceName Name of the AWS service * @param requestName Name of the operation * @param request HTTP request object - * @param context Request context (unused) + * @param context Request context */ void OnRequestRetry(const Aws::String& serviceName, const Aws::String& requestName, const std::shared_ptr& request, void* context) const override; @@ -92,46 +108,33 @@ class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface { * @param serviceName Name of the AWS service * @param requestName Name of the operation * @param request HTTP request object - * @param context Request context (unused) + * @param context Request context */ void OnFinish(const Aws::String& serviceName, const Aws::String& requestName, const std::shared_ptr& request, void* context) const override; + private: /** - * Sets test dimensions that will be included with all performance records. - * @param dimensions Vector of key-value pairs representing test dimensions (e.g., size, bucket type) - */ - static void SetTestContext(const Aws::Vector>& dimensions); - - /** - * Registers specific operations to monitor. If empty, all operations are monitored. - * @param operations Vector of operation names to track (e.g., "PutObject", "GetItem") - */ - static void RegisterOperationsToMonitor(const Aws::Vector& operations); - - /** - * Sets product information to include in the JSON output. - * @param productId Product identifier (e.g., "cpp1") - * @param sdkVersion SDK version string - * @param commitId Git commit identifier - */ - static void SetProductInfo(const Aws::String& productId, const Aws::String& sdkVersion, const Aws::String& commitId); - - /** - * Sets the output filename for the JSON performance report. - * @param filename Path to output file (e.g., "s3-perf-results.json") + * Helper method to process request metrics and store in context. + * @param serviceName Name of the AWS service + * @param requestName Name of the operation + * @param request HTTP request object + * @param metricsFromCore Core metrics collection containing latency data + * @param context Request context */ - static void SetOutputFilename(const Aws::String& filename); + void StoreLatencyInContext(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request, + const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void* context) const; - private: /** - * Adds a performance record for a completed AWS service operation. - * @param serviceName Name of the AWS service (e.g., "S3", "DynamoDB") - * @param requestName Name of the operation (e.g., "PutObject", "GetItem") - * @param metricsFromCore Core metrics collection containing latency data + * Adds a performance record with a specified duration. + * @param serviceName Name of the AWS service + * @param requestName Name of the operation + * @param request HTTP request object + * @param durationMs Duration of the request in milliseconds */ void AddPerformanceRecord(const Aws::String& serviceName, const Aws::String& requestName, - const Aws::Monitoring::CoreMetricsCollection& metricsFromCore) const; + const std::shared_ptr& request, const std::variant& durationMs) const; /** * Outputs aggregated performance metrics to JSON file. @@ -139,13 +142,19 @@ class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface { */ void DumpJson() const; + /** + * Writes JSON to the output file. + * @param root The JSON root object to write + */ + void WriteJsonToFile(const Aws::Utils::Json::JsonValue& root) const; + mutable Aws::Vector m_performanceRecords; - static Aws::Vector> TestDimensions; - static Aws::Set MonitoredOperations; - static Aws::String ProductId; - static Aws::String SdkVersion; - static Aws::String CommitId; - static Aws::String OutputFilename; + Aws::Set m_monitoredOperations; + Aws::String m_productId; + Aws::String m_sdkVersion; + Aws::String m_commitId; + Aws::String m_outputFilename; + mutable bool m_hasInvalidLatency; }; /** @@ -154,12 +163,32 @@ class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface { */ class JsonReportingMetricsFactory : public Aws::Monitoring::MonitoringFactory { public: + /** + * Constructor that initializes the factory with configuration parameters. + * @param monitoredOperations Set of operations to monitor (empty means monitor all) + * @param productId Product identifier (e.g., "cpp1") + * @param sdkVersion SDK version string + * @param commitId Git commit identifier + * @param outputFilename Path to output file (e.g., "s3-perf-results.json") + */ + JsonReportingMetricsFactory(const Aws::Set& monitoredOperations = Aws::Set(), + const Aws::String& productId = "unknown", const Aws::String& sdkVersion = "unknown", + const Aws::String& commitId = "unknown", const Aws::String& outputFilename = "performance-test-results.json"); + ~JsonReportingMetricsFactory() override = default; + /** * Creates a new JsonReportingMetrics instance for performance monitoring. * @return Unique pointer to monitoring interface implementation */ Aws::UniquePtr CreateMonitoringInstance() const override; + + private: + Aws::Set m_monitoredOperations; + Aws::String m_productId; + Aws::String m_sdkVersion; + Aws::String m_commitId; + Aws::String m_outputFilename; }; } // namespace Reporting } // namespace PerformanceTest \ No newline at end of file diff --git a/tests/performance-tests/src/reporting/JsonReportingMetrics.cpp b/tests/performance-tests/src/reporting/JsonReportingMetrics.cpp index a1a46f0035e..1c4df4a7742 100644 --- a/tests/performance-tests/src/reporting/JsonReportingMetrics.cpp +++ b/tests/performance-tests/src/reporting/JsonReportingMetrics.cpp @@ -16,100 +16,139 @@ #include #include #include -#include +#include #include #include #include #include #include +#include using namespace PerformanceTest::Reporting; -Aws::Vector> JsonReportingMetrics::TestDimensions; -Aws::Set JsonReportingMetrics::MonitoredOperations; -Aws::String JsonReportingMetrics::ProductId = "unknown"; -Aws::String JsonReportingMetrics::SdkVersion = "unknown"; -Aws::String JsonReportingMetrics::CommitId = "unknown"; -Aws::String JsonReportingMetrics::OutputFilename = "performance-test-results.json"; +struct RequestContext { + Aws::String serviceName; + Aws::String requestName; + std::shared_ptr request; + std::variant durationMs = int64_t(0); +}; + +JsonReportingMetrics::JsonReportingMetrics(const Aws::Set& monitoredOperations, const Aws::String& productId, + const Aws::String& sdkVersion, const Aws::String& commitId, const Aws::String& outputFilename) + : m_monitoredOperations(monitoredOperations), + m_productId(productId), + m_sdkVersion(sdkVersion), + m_commitId(commitId), + m_outputFilename(outputFilename), + m_hasInvalidLatency(false) {} -void JsonReportingMetrics::SetTestContext(const Aws::Vector>& dimensions) { - TestDimensions = dimensions; -} +JsonReportingMetrics::~JsonReportingMetrics() { DumpJson(); } -void JsonReportingMetrics::RegisterOperationsToMonitor(const Aws::Vector& operations) { - MonitoredOperations.clear(); - for (const auto& operation : operations) { - MonitoredOperations.insert(operation); - } -} +JsonReportingMetricsFactory::JsonReportingMetricsFactory(const Aws::Set& monitoredOperations, const Aws::String& productId, + const Aws::String& sdkVersion, const Aws::String& commitId, + const Aws::String& outputFilename) + : m_monitoredOperations(monitoredOperations), + m_productId(productId), + m_sdkVersion(sdkVersion), + m_commitId(commitId), + m_outputFilename(outputFilename) {} -void JsonReportingMetrics::SetProductInfo(const Aws::String& productId, const Aws::String& sdkVersion, const Aws::String& commitId) { - ProductId = productId; - SdkVersion = sdkVersion; - CommitId = commitId; +Aws::UniquePtr JsonReportingMetricsFactory::CreateMonitoringInstance() const { + return Aws::MakeUnique("JsonReportingMetrics", m_monitoredOperations, m_productId, m_sdkVersion, m_commitId, + m_outputFilename); } -void JsonReportingMetrics::SetOutputFilename(const Aws::String& filename) { OutputFilename = filename; } - -JsonReportingMetrics::~JsonReportingMetrics() { DumpJson(); } +void JsonReportingMetrics::StoreLatencyInContext(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request, + const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void* context) const { + RequestContext* requestContext = static_cast(context); -Aws::UniquePtr JsonReportingMetricsFactory::CreateMonitoringInstance() const { - return Aws::MakeUnique("JsonReportingMetrics"); + Aws::String const latencyKey = Aws::Monitoring::GetHttpClientMetricNameByType(Aws::Monitoring::HttpClientMetricsType::RequestLatency); + auto iterator = metricsFromCore.httpClientMetrics.find(latencyKey); + if (iterator != metricsFromCore.httpClientMetrics.end() && iterator->second > 0) { + requestContext->serviceName = serviceName; + requestContext->requestName = requestName; + requestContext->request = request; + requestContext->durationMs = iterator->second; + } else { + m_hasInvalidLatency = true; + } } void JsonReportingMetrics::AddPerformanceRecord(const Aws::String& serviceName, const Aws::String& requestName, - const Aws::Monitoring::CoreMetricsCollection& metricsFromCore) const { + const std::shared_ptr& request, + const std::variant& durationMs) const { // If no operations are registered, monitor all operations. Otherwise, only monitor registered operations - if (!MonitoredOperations.empty() && MonitoredOperations.find(requestName) == MonitoredOperations.end()) { + if (!m_monitoredOperations.empty() && m_monitoredOperations.find(requestName) == m_monitoredOperations.end()) { return; } - int64_t durationMs = 0; - Aws::String const latencyKey = Aws::Monitoring::GetHttpClientMetricNameByType(Aws::Monitoring::HttpClientMetricsType::RequestLatency); - - auto iterator = metricsFromCore.httpClientMetrics.find(latencyKey); - if (iterator != metricsFromCore.httpClientMetrics.end()) { - durationMs = iterator->second; - } - PerformanceMetricRecord record; record.name = Aws::Utils::StringUtils::ToLower(serviceName.c_str()) + "." + Aws::Utils::StringUtils::ToLower(requestName.c_str()) + ".latency"; record.description = "Time to complete " + requestName + " operation"; record.unit = "Milliseconds"; - record.date = Aws::Utils::DateTime::Now().Seconds(); - record.measurements.push_back(durationMs); - record.dimensions = TestDimensions; + record.date = Aws::Utils::DateTime::Now(); + record.measurements.emplace_back(durationMs); + + if (request) { + auto headers = request->GetHeaders(); + for (const auto& header : headers) { + if (header.first.find("test-dimension-") == 0) { + Aws::String const key = header.first.substr(15); + record.dimensions[key] = header.second; + } + } + } m_performanceRecords.push_back(record); } void* JsonReportingMetrics::OnRequestStarted(const Aws::String&, const Aws::String&, const std::shared_ptr&) const { - return nullptr; + auto context = Aws::New("RequestContext"); + return context; } void JsonReportingMetrics::OnRequestSucceeded(const Aws::String& serviceName, const Aws::String& requestName, - const std::shared_ptr&, const Aws::Client::HttpResponseOutcome&, - const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void*) const { - AddPerformanceRecord(serviceName, requestName, metricsFromCore); + const std::shared_ptr& request, + const Aws::Client::HttpResponseOutcome&, + const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void* context) const { + StoreLatencyInContext(serviceName, requestName, request, metricsFromCore, context); } void JsonReportingMetrics::OnRequestFailed(const Aws::String& serviceName, const Aws::String& requestName, - const std::shared_ptr&, const Aws::Client::HttpResponseOutcome&, - const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void*) const { - AddPerformanceRecord(serviceName, requestName, metricsFromCore); + const std::shared_ptr& request, + const Aws::Client::HttpResponseOutcome&, + const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void* context) const { + StoreLatencyInContext(serviceName, requestName, request, metricsFromCore, context); } void JsonReportingMetrics::OnRequestRetry(const Aws::String&, const Aws::String&, const std::shared_ptr&, void*) const {} void JsonReportingMetrics::OnFinish(const Aws::String&, const Aws::String&, const std::shared_ptr&, - void*) const {} + void* context) const { + RequestContext* requestContext = static_cast(context); + + if (std::visit([](auto&& v) { return v > 0; }, requestContext->durationMs)) { + AddPerformanceRecord(requestContext->serviceName, requestContext->requestName, requestContext->request, requestContext->durationMs); + } + + Aws::Delete(requestContext); +} void JsonReportingMetrics::DumpJson() const { - if (m_performanceRecords.empty()) { + Aws::Utils::Json::JsonValue root; + root.WithString("productId", m_productId); + root.WithString("sdkVersion", m_sdkVersion); + root.WithString("commitId", m_commitId); + + // If there is an invalid latency or there are no records, then use an empty results array + if (m_hasInvalidLatency || m_performanceRecords.empty()) { + root.WithArray("results", Aws::Utils::Array(0)); + WriteJsonToFile(root); return; } @@ -131,12 +170,6 @@ void JsonReportingMetrics::DumpJson() const { } } - // Create the JSON output - Aws::Utils::Json::JsonValue root; - root.WithString("productId", ProductId); - root.WithString("sdkVersion", SdkVersion); - root.WithString("commitId", CommitId); - Aws::Utils::Array results(aggregatedRecords.size()); size_t index = 0; @@ -146,24 +179,29 @@ void JsonReportingMetrics::DumpJson() const { jsonMetric.WithString("name", record.name); jsonMetric.WithString("description", record.description); jsonMetric.WithString("unit", record.unit); - jsonMetric.WithInt64("date", record.date); + jsonMetric.WithInt64("date", record.date.Seconds()); if (!record.dimensions.empty()) { Aws::Utils::Array dimensionsArray(record.dimensions.size()); - for (size_t j = 0; j < record.dimensions.size(); ++j) { + size_t dimensionIndex = 0; + for (const auto& dim : record.dimensions) { Aws::Utils::Json::JsonValue dimension; - dimension.WithString("name", record.dimensions[j].first); - dimension.WithString("value", record.dimensions[j].second); - dimensionsArray[j] = std::move(dimension); + dimension.WithString("name", dim.first); + dimension.WithString("value", dim.second); + dimensionsArray[dimensionIndex++] = std::move(dimension); } jsonMetric.WithArray("dimensions", std::move(dimensionsArray)); } Aws::Utils::Array measurementsArray(record.measurements.size()); - for (size_t j = 0; j < record.measurements.size(); ++j) { + for (size_t measurementIndex = 0; measurementIndex < record.measurements.size(); ++measurementIndex) { Aws::Utils::Json::JsonValue measurementValue; - measurementValue.AsInt64(record.measurements[j]); - measurementsArray[j] = std::move(measurementValue); + if (std::holds_alternative(record.measurements[measurementIndex])) { + measurementValue.AsDouble(std::get(record.measurements[measurementIndex])); + } else { + measurementValue.AsInt64(std::get(record.measurements[measurementIndex])); + } + measurementsArray[measurementIndex] = std::move(measurementValue); } jsonMetric.WithArray("measurements", std::move(measurementsArray)); @@ -171,8 +209,11 @@ void JsonReportingMetrics::DumpJson() const { } root.WithArray("results", std::move(results)); + WriteJsonToFile(root); +} - std::ofstream outFile(OutputFilename.c_str()); +void JsonReportingMetrics::WriteJsonToFile(const Aws::Utils::Json::JsonValue& root) const { + std::ofstream outFile(m_outputFilename.c_str()); if (outFile.is_open()) { outFile << root.View().WriteReadable(); } From 94aa414332b234bf69b8060ba96ecd9df6a7b5b4 Mon Sep 17 00:00:00 2001 From: YUXUAN CHEN Date: Fri, 27 Jun 2025 15:12:20 -0400 Subject: [PATCH 3/3] fix formatting --- .../include/performance-tests/reporting/JsonReportingMetrics.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/performance-tests/include/performance-tests/reporting/JsonReportingMetrics.h b/tests/performance-tests/include/performance-tests/reporting/JsonReportingMetrics.h index dbd76e60890..87ebb6d9374 100644 --- a/tests/performance-tests/include/performance-tests/reporting/JsonReportingMetrics.h +++ b/tests/performance-tests/include/performance-tests/reporting/JsonReportingMetrics.h @@ -134,7 +134,8 @@ class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface { * @param durationMs Duration of the request in milliseconds */ void AddPerformanceRecord(const Aws::String& serviceName, const Aws::String& requestName, - const std::shared_ptr& request, const std::variant& durationMs) const; + const std::shared_ptr& request, + const std::variant& durationMs) const; /** * Outputs aggregated performance metrics to JSON file.