diff --git a/tests/performance-tests/include/performance-tests/reporting/JsonReportingMetrics.h b/tests/performance-tests/include/performance-tests/reporting/JsonReportingMetrics.h new file mode 100644 index 00000000000..87ebb6d9374 --- /dev/null +++ b/tests/performance-tests/include/performance-tests/reporting/JsonReportingMetrics.h @@ -0,0 +1,195 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace PerformanceTest { +namespace Reporting { + +/** + * Container for a single performance metric record that stores measurement data and associated metadata. + */ +struct PerformanceMetricRecord { + Aws::String name; + Aws::String description; + Aws::String unit; + Aws::Utils::DateTime date; + Aws::Vector> measurements; + Aws::Map dimensions; +}; + +/** + * An implementation of the MonitoringInterface that collects performance metrics + * and reports them in a JSON format. + */ +class JsonReportingMetrics : public Aws::Monitoring::MonitoringInterface { + public: + /** + * Constructor that initializes the metrics collector with configuration parameters. + * @param monitoredOperations Set of operations to monitor (empty means monitor all) + * @param productId Product identifier (e.g., "cpp1") + * @param sdkVersion SDK version string + * @param commitId Git commit identifier + * @param outputFilename Path to output file (e.g., "s3-perf-results.json") + */ + JsonReportingMetrics(const Aws::Set& monitoredOperations = Aws::Set(), const Aws::String& productId = "unknown", + const Aws::String& sdkVersion = "unknown", const Aws::String& commitId = "unknown", + const Aws::String& outputFilename = "performance-test-results.json"); + + ~JsonReportingMetrics() override; + + /** + * Called when an AWS request is started. Returns context for tracking. + * @param serviceName Name of the AWS service + * @param requestName Name of the operation + * @param request HTTP request object + * @return Context pointer (always returns nullptr) + */ + void* OnRequestStarted(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request) const override; + + /** + * Called when an AWS request succeeds. Records performance metrics. + * @param serviceName Name of the AWS service + * @param requestName Name of the operation + * @param request HTTP request object + * @param outcome HTTP response outcome + * @param metrics Core metrics collection containing latency data + * @param context Request context + */ + void OnRequestSucceeded(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request, const Aws::Client::HttpResponseOutcome& outcome, + const Aws::Monitoring::CoreMetricsCollection& metrics, void* context) const override; + + /** + * Called when an AWS request fails. Records performance metrics. + * @param serviceName Name of the AWS service + * @param requestName Name of the operation + * @param request HTTP request object + * @param outcome HTTP response outcome + * @param metrics Core metrics collection containing latency data + * @param context Request context + */ + void OnRequestFailed(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request, const Aws::Client::HttpResponseOutcome& outcome, + const Aws::Monitoring::CoreMetricsCollection& metrics, void* context) const override; + + /** + * Called when an AWS request is retried. No action taken. + * @param serviceName Name of the AWS service + * @param requestName Name of the operation + * @param request HTTP request object + * @param context Request context + */ + void OnRequestRetry(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request, void* context) const override; + + /** + * Called when an AWS request finishes. No action taken. + * @param serviceName Name of the AWS service + * @param requestName Name of the operation + * @param request HTTP request object + * @param context Request context + */ + void OnFinish(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request, void* context) const override; + + private: + /** + * Helper method to process request metrics and store in context. + * @param serviceName Name of the AWS service + * @param requestName Name of the operation + * @param request HTTP request object + * @param metricsFromCore Core metrics collection containing latency data + * @param context Request context + */ + void StoreLatencyInContext(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request, + const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void* context) const; + + /** + * Adds a performance record with a specified duration. + * @param serviceName Name of the AWS service + * @param requestName Name of the operation + * @param request HTTP request object + * @param durationMs Duration of the request in milliseconds + */ + void AddPerformanceRecord(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request, + const std::variant& durationMs) const; + + /** + * Outputs aggregated performance metrics to JSON file. + * Groups records by name and dimensions, then writes to configured output file. + */ + void DumpJson() const; + + /** + * Writes JSON to the output file. + * @param root The JSON root object to write + */ + void WriteJsonToFile(const Aws::Utils::Json::JsonValue& root) const; + + mutable Aws::Vector m_performanceRecords; + Aws::Set m_monitoredOperations; + Aws::String m_productId; + Aws::String m_sdkVersion; + Aws::String m_commitId; + Aws::String m_outputFilename; + mutable bool m_hasInvalidLatency; +}; + +/** + * A factory for creating instances of JsonReportingMetrics. + * Used by the AWS SDK monitoring system to instantiate performance metrics collectors. + */ +class JsonReportingMetricsFactory : public Aws::Monitoring::MonitoringFactory { + public: + /** + * Constructor that initializes the factory with configuration parameters. + * @param monitoredOperations Set of operations to monitor (empty means monitor all) + * @param productId Product identifier (e.g., "cpp1") + * @param sdkVersion SDK version string + * @param commitId Git commit identifier + * @param outputFilename Path to output file (e.g., "s3-perf-results.json") + */ + JsonReportingMetricsFactory(const Aws::Set& monitoredOperations = Aws::Set(), + const Aws::String& productId = "unknown", const Aws::String& sdkVersion = "unknown", + const Aws::String& commitId = "unknown", const Aws::String& outputFilename = "performance-test-results.json"); + + ~JsonReportingMetricsFactory() override = default; + + /** + * Creates a new JsonReportingMetrics instance for performance monitoring. + * @return Unique pointer to monitoring interface implementation + */ + Aws::UniquePtr CreateMonitoringInstance() const override; + + private: + Aws::Set m_monitoredOperations; + Aws::String m_productId; + Aws::String m_sdkVersion; + Aws::String m_commitId; + Aws::String m_outputFilename; +}; +} // namespace Reporting +} // namespace PerformanceTest \ No newline at end of file diff --git a/tests/performance-tests/src/reporting/JsonReportingMetrics.cpp b/tests/performance-tests/src/reporting/JsonReportingMetrics.cpp new file mode 100644 index 00000000000..1c4df4a7742 --- /dev/null +++ b/tests/performance-tests/src/reporting/JsonReportingMetrics.cpp @@ -0,0 +1,220 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +using namespace PerformanceTest::Reporting; + +struct RequestContext { + Aws::String serviceName; + Aws::String requestName; + std::shared_ptr request; + std::variant durationMs = int64_t(0); +}; + +JsonReportingMetrics::JsonReportingMetrics(const Aws::Set& monitoredOperations, const Aws::String& productId, + const Aws::String& sdkVersion, const Aws::String& commitId, const Aws::String& outputFilename) + : m_monitoredOperations(monitoredOperations), + m_productId(productId), + m_sdkVersion(sdkVersion), + m_commitId(commitId), + m_outputFilename(outputFilename), + m_hasInvalidLatency(false) {} + +JsonReportingMetrics::~JsonReportingMetrics() { DumpJson(); } + +JsonReportingMetricsFactory::JsonReportingMetricsFactory(const Aws::Set& monitoredOperations, const Aws::String& productId, + const Aws::String& sdkVersion, const Aws::String& commitId, + const Aws::String& outputFilename) + : m_monitoredOperations(monitoredOperations), + m_productId(productId), + m_sdkVersion(sdkVersion), + m_commitId(commitId), + m_outputFilename(outputFilename) {} + +Aws::UniquePtr JsonReportingMetricsFactory::CreateMonitoringInstance() const { + return Aws::MakeUnique("JsonReportingMetrics", m_monitoredOperations, m_productId, m_sdkVersion, m_commitId, + m_outputFilename); +} + +void JsonReportingMetrics::StoreLatencyInContext(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request, + const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void* context) const { + RequestContext* requestContext = static_cast(context); + + Aws::String const latencyKey = Aws::Monitoring::GetHttpClientMetricNameByType(Aws::Monitoring::HttpClientMetricsType::RequestLatency); + auto iterator = metricsFromCore.httpClientMetrics.find(latencyKey); + if (iterator != metricsFromCore.httpClientMetrics.end() && iterator->second > 0) { + requestContext->serviceName = serviceName; + requestContext->requestName = requestName; + requestContext->request = request; + requestContext->durationMs = iterator->second; + } else { + m_hasInvalidLatency = true; + } +} + +void JsonReportingMetrics::AddPerformanceRecord(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request, + const std::variant& durationMs) const { + // If no operations are registered, monitor all operations. Otherwise, only monitor registered operations + if (!m_monitoredOperations.empty() && m_monitoredOperations.find(requestName) == m_monitoredOperations.end()) { + return; + } + + PerformanceMetricRecord record; + record.name = + Aws::Utils::StringUtils::ToLower(serviceName.c_str()) + "." + Aws::Utils::StringUtils::ToLower(requestName.c_str()) + ".latency"; + record.description = "Time to complete " + requestName + " operation"; + record.unit = "Milliseconds"; + record.date = Aws::Utils::DateTime::Now(); + record.measurements.emplace_back(durationMs); + + if (request) { + auto headers = request->GetHeaders(); + for (const auto& header : headers) { + if (header.first.find("test-dimension-") == 0) { + Aws::String const key = header.first.substr(15); + record.dimensions[key] = header.second; + } + } + } + + m_performanceRecords.push_back(record); +} + +void* JsonReportingMetrics::OnRequestStarted(const Aws::String&, const Aws::String&, + const std::shared_ptr&) const { + auto context = Aws::New("RequestContext"); + return context; +} + +void JsonReportingMetrics::OnRequestSucceeded(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request, + const Aws::Client::HttpResponseOutcome&, + const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void* context) const { + StoreLatencyInContext(serviceName, requestName, request, metricsFromCore, context); +} + +void JsonReportingMetrics::OnRequestFailed(const Aws::String& serviceName, const Aws::String& requestName, + const std::shared_ptr& request, + const Aws::Client::HttpResponseOutcome&, + const Aws::Monitoring::CoreMetricsCollection& metricsFromCore, void* context) const { + StoreLatencyInContext(serviceName, requestName, request, metricsFromCore, context); +} + +void JsonReportingMetrics::OnRequestRetry(const Aws::String&, const Aws::String&, const std::shared_ptr&, + void*) const {} + +void JsonReportingMetrics::OnFinish(const Aws::String&, const Aws::String&, const std::shared_ptr&, + void* context) const { + RequestContext* requestContext = static_cast(context); + + if (std::visit([](auto&& v) { return v > 0; }, requestContext->durationMs)) { + AddPerformanceRecord(requestContext->serviceName, requestContext->requestName, requestContext->request, requestContext->durationMs); + } + + Aws::Delete(requestContext); +} + +void JsonReportingMetrics::DumpJson() const { + Aws::Utils::Json::JsonValue root; + root.WithString("productId", m_productId); + root.WithString("sdkVersion", m_sdkVersion); + root.WithString("commitId", m_commitId); + + // If there is an invalid latency or there are no records, then use an empty results array + if (m_hasInvalidLatency || m_performanceRecords.empty()) { + root.WithArray("results", Aws::Utils::Array(0)); + WriteJsonToFile(root); + return; + } + + // Group performance records by name and dimensions + Aws::Map aggregatedRecords; + + for (const auto& record : m_performanceRecords) { + Aws::String key = record.name; + for (const auto& dim : record.dimensions) { + key += ":" + Aws::String(dim.first) + "=" + Aws::String(dim.second); + } + + if (aggregatedRecords.find(key) == aggregatedRecords.end()) { + aggregatedRecords[key] = record; + } else { + for (const auto& measurement : record.measurements) { + aggregatedRecords[key].measurements.push_back(measurement); + } + } + } + + Aws::Utils::Array results(aggregatedRecords.size()); + size_t index = 0; + + for (const auto& pair : aggregatedRecords) { + const auto& record = pair.second; + Aws::Utils::Json::JsonValue jsonMetric; + jsonMetric.WithString("name", record.name); + jsonMetric.WithString("description", record.description); + jsonMetric.WithString("unit", record.unit); + jsonMetric.WithInt64("date", record.date.Seconds()); + + if (!record.dimensions.empty()) { + Aws::Utils::Array dimensionsArray(record.dimensions.size()); + size_t dimensionIndex = 0; + for (const auto& dim : record.dimensions) { + Aws::Utils::Json::JsonValue dimension; + dimension.WithString("name", dim.first); + dimension.WithString("value", dim.second); + dimensionsArray[dimensionIndex++] = std::move(dimension); + } + jsonMetric.WithArray("dimensions", std::move(dimensionsArray)); + } + + Aws::Utils::Array measurementsArray(record.measurements.size()); + for (size_t measurementIndex = 0; measurementIndex < record.measurements.size(); ++measurementIndex) { + Aws::Utils::Json::JsonValue measurementValue; + if (std::holds_alternative(record.measurements[measurementIndex])) { + measurementValue.AsDouble(std::get(record.measurements[measurementIndex])); + } else { + measurementValue.AsInt64(std::get(record.measurements[measurementIndex])); + } + measurementsArray[measurementIndex] = std::move(measurementValue); + } + jsonMetric.WithArray("measurements", std::move(measurementsArray)); + + results[index++] = std::move(jsonMetric); + } + + root.WithArray("results", std::move(results)); + WriteJsonToFile(root); +} + +void JsonReportingMetrics::WriteJsonToFile(const Aws::Utils::Json::JsonValue& root) const { + std::ofstream outFile(m_outputFilename.c_str()); + if (outFile.is_open()) { + outFile << root.View().WriteReadable(); + } +} \ No newline at end of file