Использование понижающей выборки данных с помощью жизненного цикла потока данных
Это упрощенный пример, который позволяет быстро увидеть, как понижающая выборка данных, как часть жизненного цикла потока данных, уменьшает размер хранилища выборки метрик. Пример использует типичные данные мониторинга кластера Kubernetes. Чтобы протестировать понижающую выборку с жизненным циклом потока данных, выполните следующие шаги:
Предварительные условия
Обратитесь к предварительным условиям для потока данных временных рядов.
Создайте шаблон индекса с жизненным циклом потока данных
Это создает шаблон индекса для базового потока данных. Доступные параметры для шаблона индекса подробно описаны в Настройка потока данных временных рядов.
Для простоты во всех time_series_metric параметрах в отображении временных рядов установлено значение типа gauge, но также может использоваться тип метрики counter. Значения time_series_metric определяют тип статистических представлений, используемых при понижающей выборке.
Шаблон индекса включает набор статических измерений временных рядов: host, namespace, node и pod. Измерения временных рядов не изменяются при понижающей выборке.
Для включения понижающей выборки этот шаблон включает раздел lifecycle с объектом понижающей выборки downsampling. Параметр fixed_interval устанавливает интервал понижающей выборки, в котором вы хотите агрегировать исходные данные временных рядов. Параметр after определяет время после переключения индекса, которое должно пройти перед выполнением понижающей выборки.
resp = client.indices.put_index_template(
name="datastream_template",
index_patterns=[
"datastream*"
],
data_stream={},
template={
"lifecycle": {
"downsampling": [
{
"after": "1m",
"fixed_interval": "1h"
}
]
},
"settings": {
"index": {
"mode": "time_series"
}
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"kubernetes": {
"properties": {
"container": {
"properties": {
"cpu": {
"properties": {
"usage": {
"properties": {
"core": {
"properties": {
"ns": {
"type": "long"
}
}
},
"limit": {
"properties": {
"pct": {
"type": "float"
}
}
},
"nanocores": {
"type": "long",
"time_series_metric": "gauge"
},
"node": {
"properties": {
"pct": {
"type": "float"
}
}
}
}
}
}
},
"memory": {
"properties": {
"available": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
}
}
},
"majorpagefaults": {
"type": "long"
},
"pagefaults": {
"type": "long",
"time_series_metric": "gauge"
},
"rss": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
}
}
},
"usage": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
},
"limit": {
"properties": {
"pct": {
"type": "float"
}
}
},
"node": {
"properties": {
"pct": {
"type": "float"
}
}
}
}
},
"workingset": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
}
}
}
}
},
"name": {
"type": "keyword"
},
"start_time": {
"type": "date"
}
}
},
"host": {
"type": "keyword",
"time_series_dimension": True
},
"namespace": {
"type": "keyword",
"time_series_dimension": True
},
"node": {
"type": "keyword",
"time_series_dimension": True
},
"pod": {
"type": "keyword",
"time_series_dimension": True
}
}
}
}
}
},
)
print(resp) const response = await client.indices.putIndexTemplate({
name: "datastream_template",
index_patterns: ["datastream*"],
data_stream: {},
template: {
lifecycle: {
downsampling: [
{
after: "1m",
fixed_interval: "1h",
},
],
},
settings: {
index: {
mode: "time_series",
},
},
mappings: {
properties: {
"@timestamp": {
type: "date",
},
kubernetes: {
properties: {
container: {
properties: {
cpu: {
properties: {
usage: {
properties: {
core: {
properties: {
ns: {
type: "long",
},
},
},
limit: {
properties: {
pct: {
type: "float",
},
},
},
nanocores: {
type: "long",
time_series_metric: "gauge",
},
node: {
properties: {
pct: {
type: "float",
},
},
},
},
},
},
},
memory: {
properties: {
available: {
properties: {
bytes: {
type: "long",
time_series_metric: "gauge",
},
},
},
majorpagefaults: {
type: "long",
},
pagefaults: {
type: "long",
time_series_metric: "gauge",
},
rss: {
properties: {
bytes: {
type: "long",
time_series_metric: "gauge",
},
},
},
usage: {
properties: {
bytes: {
type: "long",
time_series_metric: "gauge",
},
limit: {
properties: {
pct: {
type: "float",
},
},
},
node: {
properties: {
pct: {
type: "float",
},
},
},
},
},
workingset: {
properties: {
bytes: {
type: "long",
time_series_metric: "gauge",
},
},
},
},
},
name: {
type: "keyword",
},
start_time: {
type: "date",
},
},
},
host: {
type: "keyword",
time_series_dimension: true,
},
namespace: {
type: "keyword",
time_series_dimension: true,
},
node: {
type: "keyword",
time_series_dimension: true,
},
pod: {
type: "keyword",
time_series_dimension: true,
},
},
},
},
},
},
});
console.log(response); PUT _index_template/datastream_template
{
"index_patterns": [
"datastream*"
],
"data_stream": {},
"template": {
"lifecycle": {
"downsampling": [
{
"after": "1m",
"fixed_interval": "1h"
}
]
},
"settings": {
"index": {
"mode": "time_series"
}
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"kubernetes": {
"properties": {
"container": {
"properties": {
"cpu": {
"properties": {
"usage": {
"properties": {
"core": {
"properties": {
"ns": {
"type": "long"
}
}
},
"limit": {
"properties": {
"pct": {
"type": "float"
}
}
},
"nanocores": {
"type": "long",
"time_series_metric": "gauge"
},
"node": {
"properties": {
"pct": {
"type": "float"
}
}
}
}
}
}
},
"memory": {
"properties": {
"available": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
}
}
},
"majorpagefaults": {
"type": "long"
},
"pagefaults": {
"type": "long",
"time_series_metric": "gauge"
},
"rss": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
}
}
},
"usage": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
},
"limit": {
"properties": {
"pct": {
"type": "float"
}
}
},
"node": {
"properties": {
"pct": {
"type": "float"
}
}
}
}
},
"workingset": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
}
}
}
}
},
"name": {
"type": "keyword"
},
"start_time": {
"type": "date"
}
}
},
"host": {
"type": "keyword",
"time_series_dimension": true
},
"namespace": {
"type": "keyword",
"time_series_dimension": true
},
"node": {
"type": "keyword",
"time_series_dimension": true
},
"pod": {
"type": "keyword",
"time_series_dimension": true
}
}
}
}
}
}
} Загрузите данные временных рядов
Используйте запрос API массовой загрузки для автоматической создания TSDS и индексирования набора из десяти документов.
Важно: Перед выполнением этого запроса массовой загрузки необходимо обновить временные метки до трех-пяти часов после текущего времени. То есть найдите 2022-06-21T15 и замените на текущую дату, а также отрегулируйте час до текущего времени плюс три часа.
resp = client.bulk(
index="datastream",
refresh=True,
operations=[
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:49:00Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 91153,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 463314616
},
"usage": {
"bytes": 307007078,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 585236
},
"rss": {
"bytes": 102728
},
"pagefaults": 120901,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:45:50Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 124501,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 982546514
},
"usage": {
"bytes": 360035574,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 1339884
},
"rss": {
"bytes": 381174
},
"pagefaults": 178473,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:44:50Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 38907,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 862723768
},
"usage": {
"bytes": 379572388,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 431227
},
"rss": {
"bytes": 386580
},
"pagefaults": 233166,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:44:40Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 86706,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 567160996
},
"usage": {
"bytes": 103266017,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 1724908
},
"rss": {
"bytes": 105431
},
"pagefaults": 233166,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:44:00Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 150069,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 639054643
},
"usage": {
"bytes": 265142477,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 1786511
},
"rss": {
"bytes": 189235
},
"pagefaults": 138172,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:42:40Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 82260,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 854735585
},
"usage": {
"bytes": 309798052,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 924058
},
"rss": {
"bytes": 110838
},
"pagefaults": 259073,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:42:10Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 153404,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 279586406
},
"usage": {
"bytes": 214904955,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 1047265
},
"rss": {
"bytes": 91914
},
"pagefaults": 302252,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:40:20Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 125613,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 822782853
},
"usage": {
"bytes": 100475044,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 2109932
},
"rss": {
"bytes": 278446
},
"pagefaults": 74843,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:40:10Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 100046,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 567160996
},
"usage": {
"bytes": 362826547,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 1986724
},
"rss": {
"bytes": 402801
},
"pagefaults": 296495,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:38:30Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 40018,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 1062428344
},
"usage": {
"bytes": 265142477,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 2294743
},
"rss": {
"bytes": 340623
},
"pagefaults": 224530,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
}
],
)
print(resp) response = client.bulk(
index: 'datastream',
refresh: true,
body: [
{
create: {}
},
{
"@timestamp": '2022-06-21T15:49:00Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 91_153,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 463_314_616
},
usage: {
bytes: 307_007_078,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 585_236
},
rss: {
bytes: 102_728
},
pagefaults: 120_901,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:45:50Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 124_501,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 982_546_514
},
usage: {
bytes: 360_035_574,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 1_339_884
},
rss: {
bytes: 381_174
},
pagefaults: 178_473,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:44:50Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 38_907,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 862_723_768
},
usage: {
bytes: 379_572_388,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 431_227
},
rss: {
bytes: 386_580
},
pagefaults: 233_166,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:44:40Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 86_706,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 567_160_996
},
usage: {
bytes: 103_266_017,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 1_724_908
},
rss: {
bytes: 105_431
},
pagefaults: 233_166,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:44:00Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 150_069,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 639_054_643
},
usage: {
bytes: 265_142_477,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 1_786_511
},
rss: {
bytes: 189_235
},
pagefaults: 138_172,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:42:40Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 82_260,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 854_735_585
},
usage: {
bytes: 309_798_052,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 924_058
},
rss: {
bytes: 110_838
},
pagefaults: 259_073,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:42:10Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 153_404,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 279_586_406
},
usage: {
bytes: 214_904_955,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 1_047_265
},
rss: {
bytes: 91_914
},
pagefaults: 302_252,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:40:20Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 125_613,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 822_782_853
},
usage: {
bytes: 100_475_044,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 2_109_932
},
rss: {
bytes: 278_446
},
pagefaults: 74_843,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:40:10Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 100_046,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 567_160_996
},
usage: {
bytes: 362_826_547,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 1_986_724
},
rss: {
bytes: 402_801
},
pagefaults: 296_495,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:38:30Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 40_018,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 1_062_428_344
},
usage: {
bytes: 265_142_477,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 2_294_743
},
rss: {
bytes: 340_623
},
pagefaults: 224_530,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
}
]
)
puts response const response = await client.bulk({
index: "datastream",
refresh: "true",
operations: [
{
create: {},
},
{
"@timestamp": "2022-06-21T15:49:00Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 91153,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 463314616,
},
usage: {
bytes: 307007078,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 585236,
},
rss: {
bytes: 102728,
},
pagefaults: 120901,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:45:50Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 124501,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 982546514,
},
usage: {
bytes: 360035574,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 1339884,
},
rss: {
bytes: 381174,
},
pagefaults: 178473,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:44:50Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 38907,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 862723768,
},
usage: {
bytes: 379572388,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 431227,
},
rss: {
bytes: 386580,
},
pagefaults: 233166,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:44:40Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 86706,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 567160996,
},
usage: {
bytes: 103266017,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 1724908,
},
rss: {
bytes: 105431,
},
pagefaults: 233166,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:44:00Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 150069,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 639054643,
},
usage: {
bytes: 265142477,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 1786511,
},
rss: {
bytes: 189235,
},
pagefaults: 138172,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:42:40Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 82260,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 854735585,
},
usage: {
bytes: 309798052,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 924058,
},
rss: {
bytes: 110838,
},
pagefaults: 259073,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:42:10Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 153404,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 279586406,
},
usage: {
bytes: 214904955,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 1047265,
},
rss: {
bytes: 91914,
},
pagefaults: 302252,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:40:20Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 125613,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 822782853,
},
usage: {
bytes: 100475044,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 2109932,
},
rss: {
bytes: 278446,
},
pagefaults: 74843,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:40:10Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 100046,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 567160996,
},
usage: {
bytes: 362826547,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 1986724,
},
rss: {
bytes: 402801,
},
pagefaults: 296495,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:38:30Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 40018,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 1062428344,
},
usage: {
bytes: 265142477,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 2294743,
},
rss: {
bytes: 340623,
},
pagefaults: 224530,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
],
});
console.log(response); PUT /datastream/_bulk?refresh
{"create": {}}
{"@timestamp":"2022-06-21T15:49:00Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":91153,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":463314616},"usage":{"bytes":307007078,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":585236},"rss":{"bytes":102728},"pagefaults":120901,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:45:50Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":124501,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":982546514},"usage":{"bytes":360035574,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1339884},"rss":{"bytes":381174},"pagefaults":178473,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:44:50Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":38907,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":862723768},"usage":{"bytes":379572388,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":431227},"rss":{"bytes":386580},"pagefaults":233166,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:44:40Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":86706,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":567160996},"usage":{"bytes":103266017,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1724908},"rss":{"bytes":105431},"pagefaults":233166,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:44:00Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":150069,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":639054643},"usage":{"bytes":265142477,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1786511},"rss":{"bytes":189235},"pagefaults":138172,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:42:40Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":82260,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":854735585},"usage":{"bytes":309798052,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":924058},"rss":{"bytes":110838},"pagefaults":259073,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:42:10Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":153404,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":279586406},"usage":{"bytes":214904955,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1047265},"rss":{"bytes":91914},"pagefaults":302252,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:40:20Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":125613,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":822782853},"usage":{"bytes":100475044,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":2109932},"rss":{"bytes":278446},"pagefaults":74843,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:40:10Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":100046,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":567160996},"usage":{"bytes":362826547,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1986724},"rss":{"bytes":402801},"pagefaults":296495,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:38:30Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":40018,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":1062428344},"usage":{"bytes":265142477,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":2294743},"rss":{"bytes":340623},"pagefaults":224530,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}} Просмотрите текущее состояние потока данных
После создания и добавления документов в поток данных проверьте текущее состояние нового индекса.
resp = client.indices.get_data_stream() print(resp)
response = client.indices.get_data_stream puts response
const response = await client.indices.getDataStream(); console.log(response);
GET _data_stream
Если политика жизненного цикла потока данных еще не применена, ваши результаты будут такими же, как и в следующем примере. Обратите внимание на исходный index_name: .ds-datastream-2024.04.29-000001.
{
"data_streams": [
{
"name": "datastream",
"timestamp_field": {
"name": "@timestamp"
},
"indices": [
{
"index_name": ".ds-datastream-2024.04.29-000001",
"index_uuid": "vUMNtCyXQhGdlo1BD-cGRw",
"managed_by": "Data stream lifecycle"
}
],
"generation": 1,
"status": "GREEN",
"template": "datastream_template",
"lifecycle": {
"enabled": true,
"downsampling": [
{
"after": "1m",
"fixed_interval": "1h"
}
]
},
"next_generation_managed_by": "Data stream lifecycle",
"hidden": false,
"system": false,
"allow_custom_routing": false,
"replicated": false,
"rollover_on_write": false,
"time_series": {
"temporal_ranges": [
{
"start": "2024-04-29T15:55:46.000Z",
"end": "2024-04-29T18:25:46.000Z"
}
]
}
}
]
} Далее выполните запрос поиска:
resp = client.search(
index="datastream",
)
print(resp) response = client.search( index: 'datastream' ) puts response
const response = await client.search({
index: "datastream",
});
console.log(response); GET datastream/_search
Запрос возвращает ваши десять недавно добавленных документов.
{
"took": 23,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 10,
"relation": "eq"
},
... Переключите поток данных
Жизненный цикл потока данных автоматически переключит поток данных и выполнит понижающую выборку. Этот шаг необходим только для того, чтобы увидеть результаты понижающей выборки в рамках этого учебника.
Переключите поток данных, используя API переключения:
resp = client.indices.rollover(
alias="datastream",
)
print(resp) const response = await client.indices.rollover({
alias: "datastream",
});
console.log(response); POST /datastream/_rollover/
Просмотр результатов понижающей выборки
По умолчанию действия жизненного цикла потока данных выполняются каждые пять минут. Понижающая выборка происходит после переключения индекса и истечения времени окончания временного ряда индекса, так как ожидается, что основной индекс будет получать большие записи до тех пор. Индекс теперь переключен после предыдущего шага, но время окончания его временного ряда, вероятно, все еще в будущем. После того, как временной ряд индекса станет прошлым, запустите запрос GET _data_stream снова.
resp = client.indices.get_data_stream() print(resp)
response = client.indices.get_data_stream puts response
const response = await client.indices.getDataStream(); console.log(response);
GET _data_stream
После выполнения действия жизненного цикла потока данных исходный индекс .ds-datastream-2024.04.29-000001 заменяется новым, пониженным индексом, в данном случае downsample-1h-.ds-datastream-2024.04.29-000001.
{
"data_streams": [
{
"name": "datastream",
"timestamp_field": {
"name": "@timestamp"
},
"indices": [
{
"index_name": "downsample-1h-.ds-datastream-2024.04.29-000001",
"index_uuid": "VqXuShP4T8ODAOnWFcqitg",
"managed_by": "Data stream lifecycle"
},
{
"index_name": ".ds-datastream-2024.04.29-000002",
"index_uuid": "8gCeSdjUSWG-o-PeEAJ0jA",
"managed_by": "Data stream lifecycle"
}
],
... Выполните запрос поиска в потоке данных (обратите внимание, что при запросе к индексам с пониженной выборкой есть некоторые нюансы).
resp = client.search(
index="datastream",
)
print(resp) response = client.search( index: 'datastream' ) puts response
const response = await client.search({
index: "datastream",
});
console.log(response); GET datastream/_search
Новый индекс с пониженной выборкой содержит только один документ, включающий статистику min, max, sum и value_count на основе исходных выборочных метрик.
{
"took": 26,
"timed_out": false,
"_shards": {
"total": 2,
"successful": 2,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 1,
"hits": [
{
"_index": "downsample-1h-.ds-datastream-2024.04.29-000001",
"_id": "0eL0wMf38sl_s5JnAAABjyrMjoA",
"_score": 1,
"_source": {
"@timestamp": "2024-04-29T17:00:00.000Z",
"_doc_count": 10,
"kubernetes": {
"container": {
"cpu": {
"usage": {
"core": {
"ns": 12828317850
},
"limit": {
"pct": 0.0000277905
},
"nanocores": {
"min": 38907,
"max": 153404,
"sum": 992677,
"value_count": 10
},
"node": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": {
"min": 279586406,
"max": 1062428344,
"sum": 7101494721,
"value_count": 10
}
},
"majorpagefaults": 0,
"pagefaults": {
"min": 74843,
"max": 302252,
"sum": 2061071,
"value_count": 10
},
"rss": {
"bytes": {
"min": 91914,
"max": 402801,
"sum": 2389770,
"value_count": 10
}
},
"usage": {
"bytes": {
"min": 100475044,
"max": 379572388,
"sum": 2668170609,
"value_count": 10
},
"limit": {
"pct": 0.00009923134
},
"node": {
"pct": 0.017700378
}
},
"workingset": {
"bytes": {
"min": 431227,
"max": 2294743,
"sum": 14230488,
"value_count": 10
}
}
},
"name": "container-name-44",
"start_time": "2021-03-30T07:59:06.000Z"
},
"host": "gke-apps-0",
"namespace": "namespace26",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0"
}
}
}
]
}
} Используйте API статистики потока данных, чтобы получить статистику потока данных, включая размер хранилища.
resp = client.indices.data_streams_stats(
name="datastream",
human=True,
)
print(resp) response = client.indices.data_streams_stats( name: 'datastream', human: true ) puts response
const response = await client.indices.dataStreamsStats({
name: "datastream",
human: "true",
});
console.log(response); GET /_data_stream/datastream/_stats?human=true
{
"_shards": {
"total": 4,
"successful": 4,
"failed": 0
},
"data_stream_count": 1,
"backing_indices": 2,
"total_store_size": "37.3kb",
"total_store_size_bytes": 38230,
"data_streams": [
{
"data_stream": "datastream",
"backing_indices": 2,
"store_size": "37.3kb",
"store_size_bytes": 38230,
"maximum_timestamp": 1714410000000
}
]
} Этот пример демонстрирует, как понижающая выборка работает как часть жизненного цикла потока данных для уменьшения размера хранилища данных метрик по мере того, как они становятся менее актуальными и менее часто запрашиваются.
© 2023-2025 Elasticsearch
As of September 2024, Elasticsearch is available under a choice of three licenses: the Server Side Public License (SSPL), the Elastic License, or the AGPLv3 (OSI approved).
Elasticsearch and the Elasticsearch logo are trademarks of Elasticsearch B.V., registered in the U.S. and in other countries.
https://www.elastic.co/guide/en/elasticsearch/reference/8.17/downsampling-dsl.html
