[BE] Add sharding data by default to metrics (#110035)

Extend metric library to allow setting global metrics on a process level which will always be emitted.

Current use case for them is to include shard information every time a metric is emitted by run_test.py

<!--
copilot:poem
-->
### <samp>🤖 Generated by Copilot at 0cae92c</samp>

> _`run_test` refactored_
> _Sharding metrics in Rockset_
> _Autumn of testing_
Pull Request resolved: https://github.com/pytorch/pytorch/pull/110035
Approved by: https://github.com/clee2000
This commit is contained in:
Zain Rizvi
2023-09-26 17:06:49 +00:00
committed by PyTorch MergeBot
parent d91492a7a4
commit 1277d0e834
3 changed files with 106 additions and 9 deletions

View File

@ -4,7 +4,7 @@ import unittest
from typing import Any, Dict
from unittest import mock
from tools.stats.upload_metrics import emit_metric
from tools.stats.upload_metrics import add_global_metric, emit_metric
from tools.stats.upload_stats_lib import BATCH_SIZE, upload_to_rockset
@ -85,6 +85,76 @@ class TestUploadStats(unittest.TestCase):
{**emit_should_include, **emitted_metric},
)
@mock.patch("boto3.Session.resource")
def test_when_global_metric_specified_then_it_emits_it(
self, mock_resource: Any
) -> None:
metric = {
"some_number": 123,
}
global_metric_name = "global_metric"
global_metric_value = "global_value"
add_global_metric(global_metric_name, global_metric_value)
emit_should_include = {
**metric,
global_metric_name: global_metric_value,
}
# Preserve the metric emitted
emitted_metric: Dict[str, Any] = {}
def mock_put_item(Item: Dict[str, Any]) -> None:
nonlocal emitted_metric
emitted_metric = Item
mock_resource.return_value.Table.return_value.put_item = mock_put_item
emit_metric("metric_name", metric)
self.assertEqual(
emitted_metric,
{**emitted_metric, **emit_should_include},
)
@mock.patch("boto3.Session.resource")
def test_when_local_and_global_metric_specified_then_global_is_overridden(
self, mock_resource: Any
) -> None:
global_metric_name = "global_metric"
global_metric_value = "global_value"
local_override = "local_override"
add_global_metric(global_metric_name, global_metric_value)
metric = {
"some_number": 123,
global_metric_name: local_override,
}
emit_should_include = {
**metric,
global_metric_name: local_override,
}
# Preserve the metric emitted
emitted_metric: Dict[str, Any] = {}
def mock_put_item(Item: Dict[str, Any]) -> None:
nonlocal emitted_metric
emitted_metric = Item
mock_resource.return_value.Table.return_value.put_item = mock_put_item
emit_metric("metric_name", metric)
self.assertEqual(
emitted_metric,
{**emitted_metric, **emit_should_include},
)
@mock.patch("boto3.Session.resource")
def test_when_optional_envvar_set_to_actual_value_then_emit_vars_emits_it(
self, mock_resource: Any