> ## Documentation Index
> Fetch the complete documentation index at: https://seilabs-docs-bridge-release-v6-6-0.mintlify.site/llms.txt
> Use this file to discover all available pages before exploring further.
# Sei Node Advanced Configuration & Monitoring
> Optimize your Sei node's performance with advanced system configurations, monitoring setup with Prometheus and Grafana, and effective alerting strategies for maintaining reliable node operations.
## Optimizing System Configuration
There are a virtually unlimited number of unique individual setups that cannot be covered in this document. As well, even similar builds and configurations can behave differently due to external factors, so your results may vary.
Here are some general guidelines to use as a starting point. Be cautious, make incremental changes, testing and observing before moving forward.
Always focus on only one specific area at a time - avoid making changes to memory, storage, and CPU configs all at once. Diagnosing potential problems becomes nearly impossible otherwise.
### Memory Management
The following settings in `/etc/sysctl.conf` can optimize memory usage and disk I/O patterns:
```bash theme={"dark"}
# Minimize swapping
vm.swappiness = 1
# Control disk write behavior
vm.dirty_background_ratio = 3
vm.dirty_ratio = 10
vm.dirty_expire_centisecs = 300
vm.dirty_writeback_centisecs = 100
```
Apply changes: `sudo sysctl -p`
### Network Stack
The following settings in `/etc/sysctl.conf` may improve network performance:
```bash theme={"dark"}
# Increase connection handling capacity
net.core.somaxconn = 32768
net.core.netdev_max_backlog = 32768
net.ipv4.tcp_max_syn_backlog = 16384
# Optimize buffer sizes
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_rmem = 4096 87380 16777216
net.ipv4.tcp_wmem = 4096 87380 16777216
```
### Storage Configuration
For NVMe drives, optimize I/O scheduling:
Storage Optimization Commands
```bash theme={"dark"}
# Set IO scheduler
echo "none" > /sys/block/nvme0n1/queue/scheduler
# Set read-ahead buffer
blockdev --setra 4096 /dev/nvme0n1
# Set IO priority in systemd service
sudo tee -a /etc/systemd/system/seid.service << EOF
[Service]
IOSchedulingClass=realtime
IOSchedulingPriority=2
EOF
# Configure disk mount options
sudo tee -a /etc/fstab << EOF
/dev/nvme0n1p1 /data ext4 defaults,noatime,nosuid,nodev,noexec,commit=60 0 0
EOF
```
## Infrastructure Monitoring
Monitoring is one of the most critical components of network infrastructure. performance tuning, and alerting configuration for Cosmos-SDK/Tendermint nodes.
### Prometheus Setup
First, install Prometheus:
```bash theme={"dark"}
wget https://github.com/prometheus/prometheus/releases/download/v2.42.0/prometheus-2.42.0.linux-amd64.tar.gz
tar xvf prometheus-2.42.0.linux-amd64.tar.gz
```
Example Prometheus configuration:
```yaml theme={"dark"}
global:
scrape_interval: 15s
evaluation_interval: 15s
scrape_configs:
- job_name: 'sei_node'
static_configs:
- targets: ['node1_ip:port']
metrics_path: /metrics
- job_name: 'node'
static_configs:
- targets: ['node2_ip:port']
```
### Grafana Integration
Install and configure Grafana:
```bash theme={"dark"}
sudo apt install -y apt-transport-https software-properties-common
sudo add-apt-repository "deb https://packages.grafana.com/oss/deb stable main"
sudo apt update && sudo apt-get install grafana
```
```json theme={"dark"}
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 1,
"links": [],
"panels": [
{
"alerting": {},
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"hiddenSeries": false,
"id": 2,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.2.0",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "tendermint_consensus_height",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Block Height",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"schemaVersion": 26,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Sei Node Metrics",
"uid": "sei_metrics",
"version": 1
}
```
### Alert Management
Install Alertmanager:
```bash theme={"dark"}
wget https://github.com/prometheus/alertmanager/releases/download/v0.25.0/alertmanager-0.25.0.linux-amd64.tar.gz
tar xvf alertmanager-0.25.0.linux-amd64.tar.gz
```
```yaml theme={"dark"}
groups:
- name: validator_alerts
rules:
- alert: NodeDown
expr: up == 0
for: 5m
labels:
severity: critical
annotations:
summary: 'Node {{ $labels.instance }} down'
- alert: BlockProductionSlow
expr: rate(tendermint_consensus_height[5m]) < 0.1
for: 5m
labels:
severity: warning
annotations:
summary: 'Block production is slow on {{ $labels.instance }}'
- alert: ValidatorMissedBlocks
expr: increase(tendermint_consensus_validator_missed_blocks[1h]) > 0
labels:
severity: critical
annotations:
summary: 'Validator missing blocks'
- alert: ValidatorJailed
expr: tendermint_consensus_validator_status == 0
labels:
severity: critical
annotations:
summary: 'Validator has been jailed'
- alert: ConsensusStalled
expr: tendermint_consensus_height_status == 0
for: 5m
labels:
severity: critical
annotations:
summary: 'Consensus has stalled'
```
## Log Management
### Loki Setup
Using Loki for log aggregation:
```bash theme={"dark"}
wget https://github.com/grafana/loki/releases/download/v2.8.0/loki-linux-amd64.zip
unzip loki-linux-amd64.zip
```
```yaml theme={"dark"}
server:
http_listen_port: 9080
positions:
filename: /tmp/positions.yaml
clients:
- url: http://localhost:3100/loki/api/v1/push
scrape_configs:
- job_name: sei_logs
static_configs:
- targets:
- localhost
labels:
job: seid_logs
__path__: /var/log/seid/*.log
```
### Log Rotation
Configure logrotate to manage log files:
```bash theme={"dark"}
sudo tee /etc/logrotate.d/sei << EOF
/var/log/sei/*.log {
daily
rotate 14
compress
delaycompress
notifempty
create 0640 sei sei
sharedscripts
postrotate
systemctl reload seid
endscript
}
EOF
```
## Security Configuration
### Network Security
UFW firewall configuration:
```bash theme={"dark"}
sudo ufw default deny incoming
sudo ufw default allow outgoing
sudo ufw allow 26656/tcp comment 'Sei P2P'
sudo ufw allow 26657/tcp comment 'Sei RPC'
sudo ufw allow 9090/tcp comment 'Sei gRPC'
sudo ufw enable
```
### Rate Limiting
```nginx theme={"dark"}
http {
limit_req_zone $binary_remote_addr zone=sei_rpc:10m rate=10r/s;
server {
listen 26657;
location / {
limit_req zone=sei_rpc burst=20 nodelay;
proxy_pass http://localhost:26657;
}
}
}
```
## Validator-Specific Monitoring
### Status Query
Query validator status through SDK:
```bash theme={"dark"}
seid query staking validator $(seid keys show --bech val -a )
```
Query through REST API:
```sh theme={"dark"}
curl -s "http://localhost:1317/cosmos/staking/v1beta1/validators/"
```
```sh theme={"dark"}
#!/bin/bash
MONIKER="$1"
API_URL="http://localhost:1317/cosmos/staking/v1beta1/validators?pagination.limit=500"
echo "Querying validators from $API_URL..."
VALIDATOR_DATA=$(curl -s "$API_URL" | jq -c --arg MONIKER "$MONIKER" '.validators[] | select(.description.moniker == $MONIKER)')
if [[ -z "$VALIDATOR_DATA" ]]; then
echo "❌ No validator found with moniker: $MONIKER"
exit 1
fi
echo "Validator details:"
echo "$VALIDATOR_DATA" | jq '.'
```
### Critical Metrics
Monitor these validator-specific metrics:
```bash theme={"dark"}
# Check signing status
seid query slashing signing-info $(seid tendermint show-validator)
# Check current delegations
seid query staking delegations-to $(seid keys show -a $VALIDATOR_KEY)
```
### ProposerPriority Divergence Detection
Sei nodes export two Prometheus gauges that let operators detect when a validator's `ProposerPriority` state diverges from the rest of the network. Divergence indicates corrupted consensus state and should be investigated immediately.
| Metric | Type | Description |
| ------------------------------------------------ | ----- | -------------------------------------------------------------------------------------------------------------------------------------------------- |
| `tendermint_state_proposer_priority_hash` | gauge | Encodes the first 8 bytes of the hash of the current validator set's proposer priorities, packed as a big-endian `uint64` and cast to a `float64`. |
| `tendermint_state_proposer_priority_hash_height` | gauge | The block height at which the most recent `tendermint_state_proposer_priority_hash` was computed. |
Both metrics are emitted together every 1024 heights (roughly every few minutes at Sei block times). Emitting the hash as a numeric value instead of a Prometheus label keeps series cardinality constant at one series per node, rather than creating a new time series on every priority change.
#### How to use these metrics
Compare `tendermint_state_proposer_priority_hash` across your validators, but **only compare samples taken at the same `tendermint_state_proposer_priority_hash_height`**. Since the hash is only meaningful at a shared height, always pair it with the height gauge before comparing:
* If every node reports the **same** hash value at the same height, their `ProposerPriority` state agrees.
* If a node reports a **different** hash value at the same height, its `ProposerPriority` state has diverged and likely indicates corrupted state on that node.
Example alert rule that flags divergence across scraped nodes at a shared height:
```yaml theme={"dark"}
groups:
- name: proposer_priority_alerts
rules:
- alert: ProposerPriorityDivergence
expr: >
count(
count by (tendermint_state_proposer_priority_hash) (
tendermint_state_proposer_priority_hash
* on(instance) group_left
(tendermint_state_proposer_priority_hash_height == scalar(max(tendermint_state_proposer_priority_hash_height)))
)
) > 1
for: 10m
labels:
severity: critical
annotations:
summary: 'ProposerPriority has diverged between validators'
```
Each checkpoint also writes a `proposer priority hash checkpoint` log line containing the full 32-byte hash and the packed value, which can be used for grep-based comparison across nodes.
## EVM RPC Metrics
Sei nodes emit OpenTelemetry-based metrics for the EVM JSON-RPC layer through the process-wide `MeterProvider` (for example, a Prometheus exporter). These `evmrpc_*` metrics are the recommended source for monitoring RPC performance and websocket activity.
The OpenTelemetry Prometheus exporter namespace is `sei_chain` (underscore). When these metrics are scraped through the Prometheus exporter, exported series are prefixed accordingly (for example, `sei_chain_evmrpc_request_latency_seconds`, `sei_chain_flatkv_commit_latency`, `sei_chain_app_abci_commit_duration_seconds`, `sei_chain_module_mid_block_duration`). The metric names documented on this page are the unprefixed instrument names; prepend the `sei_chain_` namespace when querying them in Prometheus/Grafana. This namespace was previously `sei-chain` (hyphen); because Prometheus normalizes the hyphen to an underscore anyway, existing dashboards and alert queries should continue to match, but confirm your PromQL uses the `sei_chain_` prefix after upgrading.
| Metric | Type | Description |
| ---------------------------------- | --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `evmrpc_request_latency_seconds` | histogram | RPC request latency in seconds, labeled by endpoint, connection, success, error class, and JSON-RPC code bucket. |
| `evmrpc_websocket_connects_total` | counter | Number of new websocket connections. |
| `evmrpc_redirected_requests_total` | counter | Number of EVM RPC requests forwarded (proxied) to another validator, labeled by endpoint and connection. Emitted when a request such as `eth_sendRawTransaction` or a pending `eth_getTransactionCount` is redirected to the validator that owns the sender's EVM address shard. |
\| `evmrpc_historical_debug_trace_attempts_total` | counter | Number of `debug_trace*` requests targeting historical blocks, labeled by endpoint and connection. Incremented whenever a `debug_trace*` request (`debug_traceTransaction`, `debug_traceTransactionProfile`, `debug_traceBlockByNumber`, `debug_traceBlockByHash`, `debug_traceCall`, or `debug_traceStateAccess`) targets a block older than the configured `maxBlockLookback`. Such requests are rejected with an error like `block number N is beyond max lookback of M`. |
### `evmrpc_historical_debug_trace_attempts_total` labels
| Label | Description |
| ------------ | --------------------------------------------------------------------------------------------------------------------------------------------- |
| `endpoint` | The `debug_trace*` method that attempted the historical block (e.g. `debug_traceTransaction`, `debug_traceBlockByNumber`, `debug_traceCall`). |
| `connection` | The connection type that received the request (e.g. `http`, `websocket`). |
### `evmrpc_redirected_requests_total` labels
| Label | Description |
| ------------ | ------------------------------------------------------------------------------------------ |
| `endpoint` | The RPC method being forwarded (e.g. `eth_sendRawTransaction`, `eth_getTransactionCount`). |
| `connection` | The connection type that received the original request (e.g. `http`, `websocket`). |
### `evmrpc_request_latency_seconds` labels
| Label | Description |
| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `endpoint` | The RPC method being served (e.g. `eth_call`, `eth_getBalance`). |
| `connection` | The connection type serving the request (e.g. `http`, `websocket`). |
| `success` | Boolean indicating whether the request completed without error or panic. |
| `error_class` | Low-cardinality classification of the failure. Empty for successful requests. Possible values: `panic`, `execution_reverted`, `evm_not_supported`, `sei_legacy_disabled`, `association_missing`, `jsonrpc_error`, `unknown`. |
| `jsonrpc_code` | Bucketed JSON-RPC error code. Empty when there is no code (success or an untyped error). Possible values: `spec` (predefined codes `-32700`..`-32600`), `server` (server-defined codes `-32099`..`-32000`), and `other`. |
The histogram uses the following explicit bucket boundaries (in seconds):
```
0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30
```
### Legacy metric deprecation
The legacy `sei_*` RPC metrics — including `sei_rpc_request_latency_ms` and `sei_websocket_connect` — are still emitted alongside the new `evmrpc_*` metrics for backward compatibility, but they are deprecated and scheduled for removal (PLT-326) once dashboards migrate to the OpenTelemetry `evmrpc_*` metrics. When building or updating dashboards and alerts, prefer the `evmrpc_request_latency_seconds` histogram and `evmrpc_websocket_connects_total` counter over their legacy counterparts.
## FlatKV State DB Metrics
Sei nodes emit OpenTelemetry-based metrics for the FlatKV state database through the process-wide `MeterProvider` (for example, a Prometheus exporter). These `flatkv_*` metrics let operators observe FlatKV performance and progress across commits, catchup, snapshots, rollbacks, and snapshot imports.
### Latency metrics
Each latency metric is a histogram reported in seconds (unit `s`). Unless otherwise noted, they carry a `success` boolean label indicating whether the operation completed without error.
| Metric | Type | Labels | Description |
| -------------------------------------- | --------- | ---------------------- | ------------------------------------------------------- |
| `flatkv_open_latency` | histogram | `success`, `read_only` | Time taken to open the FlatKV store (`LoadVersion`). |
| `flatkv_apply_changesets_latency` | histogram | `success` | Time taken to apply changesets to FlatKV. |
| `flatkv_commit_latency` | histogram | `success` | Time taken to commit FlatKV changes. |
| `flatkv_commit_batch_latency` | histogram | `success`, `db` | Time taken to commit a per-DB PebbleDB batch. |
| `flatkv_batch_read_old_values_latency` | histogram | `success` | Time taken to batch read old FlatKV values. |
| `flatkv_catchup_latency` | histogram | `success` | Time taken to replay FlatKV WAL entries during catchup. |
| `flatkv_snapshot_write_latency` | histogram | `success` | Time taken to write a FlatKV snapshot. |
| `flatkv_snapshot_prune_latency` | histogram | — | Time taken to prune old FlatKV snapshots. |
| `flatkv_rollback_latency` | histogram | `success` | Time taken to roll back FlatKV state. |
| `flatkv_import_latency` | histogram | `success` | Time taken to import FlatKV snapshot data. |
| `flatkv_import_worker_flush_latency` | histogram | `success`, `db` | Time taken to flush a FlatKV import worker batch. |
| `flatkv_flush_latency` | histogram | `success`, `db` | Time taken to flush a per-DB data DB. |
### Counter metrics
| Metric | Type | Labels | Description |
| ---------------------------------- | ------- | --------- | ----------------------------------------------------- |
| `flatkv_num_kv_pairs` | counter | `db` | Number of key-value pairs applied to FlatKV. |
| `flatkv_catchup_replay_num_blocks` | counter | — | Number of FlatKV WAL entries replayed during catchup. |
| `flatkv_snapshot_prune_attempts` | counter | `success` | Total number of FlatKV snapshot prune attempts. |
| `flatkv_import_kv_pairs` | counter | `db` | Number of key-value pairs imported into FlatKV. |
### Gauge metrics
| Metric | Type | Labels | Description |
| -------------------------------- | ----- | ------ | ---------------------------------------------------- |
| `flatkv_pending_writes` | gauge | `db` | Current number of pending FlatKV writes per data DB. |
| `flatkv_current_version` | gauge | — | Current committed FlatKV version. |
| `flatkv_current_snapshot_height` | gauge | — | Current FlatKV snapshot height. |
The `db` label identifies the underlying data DB (for example, the account, storage, code, or legacy data directory), letting operators break down applied writes, pending writes, batch commits, and flush latency per DB.
Emission of Pebble's own internal metrics is controlled separately by the FlatKV-level `EnablePebbleMetrics` configuration knob. When set, it overrides the per-DB `EnableMetrics` settings for all data DBs (account, code, storage, legacy, and metadata), so Pebble internal metrics are toggled uniformly rather than individually per DB.
Prometheus gauges are held in memory, so after a process restart these gauges reset to zero until the next emission at the following multiple of 1024 heights (up to roughly 8.5 minutes of stale or zero data). This is expected for a monitoring signal that is only consulted in response to incidents.
## App (ABCI) Metrics
Sei nodes emit OpenTelemetry-based metrics for the application (ABCI) layer through the process-wide `MeterProvider` (for example, a Prometheus exporter). These `app_*` metrics let operators observe ABCI phase durations, transaction throughput and gas usage, block processing, and light invariance checks. They are the recommended source for application-level observability.
A set of legacy telemetry metrics (for example, the `abci`/`tx`/`sei_lightinvariance_supply` series) are still emitted alongside the new `app_*` metrics for backward compatibility, but they are deprecated and scheduled for removal (PLT-327) once dashboards migrate to the OpenTelemetry `app_*` metrics. When building or updating dashboards and alerts, prefer the `app_*` metrics over their legacy counterparts.
### ABCI phase duration metrics
Each of these is a histogram reported in seconds (unit `s`), measuring the duration of the corresponding ABCI phase.
| Metric | Type | Description |
| -------------------------------------------- | --------- | ----------------------------------------------------------- |
| `app_abci_begin_block_duration_seconds` | histogram | Duration of ABCI `BeginBlock`. |
| `app_abci_end_block_duration_seconds` | histogram | Duration of ABCI `EndBlock`. |
| `app_abci_module_end_block_duration_seconds` | histogram | Duration of module `EndBlock` calls within ABCI `EndBlock`. |
| `app_abci_check_tx_duration_seconds` | histogram | Duration of ABCI `CheckTx`. |
| `app_abci_deliver_tx_duration_seconds` | histogram | Duration of ABCI `DeliverTx`. |
| `app_abci_deliver_batch_tx_duration_seconds` | histogram | Duration of ABCI `DeliverTxBatch`. |
| `app_abci_commit_duration_seconds` | histogram | Duration of ABCI `Commit` (state write to disk). |
### Block processing metric
| Metric | Type | Attributes | Description |
| ------------------------------------ | --------- | ---------- | ----------------------------------------------------------- |
| `app_block_process_duration_seconds` | histogram | `type` | Duration of block transaction processing by execution type. |
The `type` attribute identifies the execution path: `synchronous`, `synchronous_giga`, `optimistic_concurrency`, or `occ_giga`.
### Transaction counter metrics
| Metric | Type | Attributes | Description |
| --------------------------- | ------- | ---------- | ------------------------------------------------------------------------------------------------------------------- |
| `app_tx_count_total` | counter | `result` | Number of transactions delivered, labeled by result (for example, `successful`). |
| `app_tx_process_type_total` | counter | `type` | Transactions processed by execution type (`synchronous`, `synchronous_giga`, `optimistic_concurrency`, `occ_giga`). |
| `app_tx_gas_total` | counter | `type` | Cumulative transaction gas, where `type` is `gas_used` or `gas_wanted`. |
### App flow counter metrics
| Metric | Type | Attributes | Description |
| ----------------------------------------- | ------- | ---------- | ----------------------------------------------------------------------------------------------------------------------- |
| `app_optimistic_processing_total` | counter | `enabled` | Optimistic processing attempts; `enabled:true` means the optimistic result was used, `false` means it was discarded. |
| `app_failed_total_gas_wanted_check_total` | counter | `proposer` | Proposals rejected because total block gas wanted exceeded the maximum; `proposer` is the hex-encoded proposer address. |
| `app_giga_fallback_to_v2_total` | counter | — | Number of times the giga executor fell back to V2 processing. |
| `app_pending_nonce_total` | counter | `event` | Pending nonce events, where `event` is `added`, `expired`, `rejected`, or `accepted`. |
### Light invariance metrics
| Metric | Type | Attributes | Description |
| ---------------------------------------------------- | --------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ |
| `app_lightinvariance_supply_duration_seconds` | histogram | — | Duration of the light invariance total supply check. |
| `app_lightinvariance_supply_invalid_key_total` | counter | `type` | Invalid changed-pair keys detected during the invariance check (`type` is `sei` or `wei`). |
| `app_lightinvariance_supply_unmarshal_failure_total` | counter | `type`, `step` | Unmarshal failures during the invariance supply check (`type` is `usei`, `wei`, or `total_supply`; `step` is `pre_block` or `post_block`). |
### Build info metric
| Metric | Type | Attributes | Description |
| ---------------- | ----- | ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `app_build_info` | gauge | `seid_version`, `commit` | Running binary build info; the value is always `1`, with the running version and commit exposed as attributes. This observable gauge is populated by a scrape callback, so it reflects the currently running binary on every scrape. |
## Module & Governance Metrics
Sei nodes emit OpenTelemetry-based metrics for the SDK module lifecycle (mid-block, begin-blocker, and end-blocker execution) and for validator slash events through the process-wide `MeterProvider` (for example, a Prometheus exporter). These `module_*`, per-module `*_blocker_duration`, and `validator_slashed` metrics let operators observe where per-block module execution time is spent and track slashing activity.
These OpenTelemetry metrics are emitted alongside the existing legacy telemetry counterparts, which are retained pending verification (PLT-414). When building or updating dashboards and alerts, prefer these OpenTelemetry metrics over their legacy counterparts.
### Mid-block duration metrics
Each of these is a histogram reported in seconds (unit `s`), measuring the duration of module mid-block execution.
| Metric | Type | Attributes | Description |
| --------------------------------- | --------- | ---------- | ------------------------------------------------------------------------------ |
| `module_total_mid_block_duration` | histogram | — | Total duration of all modules' mid-block execution in seconds. |
| `module_mid_block_duration` | histogram | `module` | Duration of per-module mid-block execution in seconds, labeled by module name. |
### Per-module begin/end-blocker duration metrics
Each of these is a histogram reported in seconds (unit `s`), measuring the duration of the corresponding module's begin-blocker or end-blocker execution.
| Metric | Type | Description |
| ---------------------------------------- | --------- | ------------------------------------------------------------ |
| `capability_begin_blocker_duration` | histogram | Duration of capability begin-blocker execution in seconds. |
| `crisis_end_blocker_duration` | histogram | Duration of crisis end-blocker execution in seconds. |
| `crisis_init_genesis_unmarshal_duration` | histogram | Duration of crisis `InitGenesis` JSON unmarshal in seconds. |
| `distribution_begin_blocker_duration` | histogram | Duration of distribution begin-blocker execution in seconds. |
| `evidence_begin_blocker_duration` | histogram | Duration of evidence begin-blocker execution in seconds. |
| `gov_end_blocker_duration` | histogram | Duration of gov end-blocker execution in seconds. |
| `slashing_begin_blocker_duration` | histogram | Duration of slashing begin-blocker execution in seconds. |
| `staking_begin_blocker_duration` | histogram | Duration of staking begin-blocker execution in seconds. |
| `staking_end_blocker_duration` | histogram | Duration of staking end-blocker execution in seconds. |
### Validator slash counter
| Metric | Type | Attributes | Description |
| ------------------- | ------- | ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `validator_slashed` | counter | `type`, `validator` | Number of validator slash events, labeled by slash reason (`type` is `missing_signature` or `double_sign`) and the consensus address of the slashed validator. |
### Staking keeper metrics
The staking keeper delegation metrics are emitted with a `staking_keeper_` prefix to avoid collisions with other metric namespaces.
| Metric | Type | Description |
| --------------------------------------- | ------- | -------------------------------------------------------------------- |
| `staking_keeper_delegate` | counter | Number of delegation transactions. |
| `staking_keeper_last_delegate_amount` | gauge | Amount delegated in the last delegation transaction (in `usei`). |
| `staking_keeper_redelegate` | counter | Number of redelegation transactions. |
| `staking_keeper_last_redelegate_amount` | gauge | Amount redelegated in the last redelegation transaction (in `usei`). |
| `staking_keeper_undelegate` | counter | Number of undelegation transactions. |
| `staking_keeper_last_undelegate_amount` | gauge | Amount undelegated in the last undelegation transaction (in `usei`). |
The staking keeper metrics were previously emitted without the `staking_keeper_` prefix (for example, `delegate`, `last_delegate_amount`). Update any dashboards or alerts that reference the old unprefixed names.
The duration histograms use the following explicit bucket boundaries (in seconds):
```
0.000025, 0.000050, 0.0001, 0.0005, 0.001, 0.0025, 0.005, 0.010, 0.020, 0.050, 0.075, 0.1, 0.25, 0.5, 1, 10
```
## Consensus Validation Metrics
Sei nodes emit an OpenTelemetry-based counter through the process-wide `MeterProvider` (for example, a Prometheus exporter) that tracks halting consensus validation failures swallowed by non-default `ConsensusPolicy` builds.
| Metric | Type | Attributes | Description |
| ------------------------------------- | ------- | ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `sei_unsafe_validation_skipped_total` | counter | `validation_error` | Number of halting consensus validation failures that were swallowed (counted and continued instead of halting the chain) by a non-default `ConsensusPolicy`. The `validation_error` attribute identifies the specific validation failure that was skipped. |
In production (default) builds this counter is **always zero** — every validation failure halts the node as expected. A non-zero value is only ever emitted by special-purpose binaries built with the `mock_block_validation` or `mock_chain_validation` build tags (published as `sei-chain:mock_chain_validation-*` and `sei-chain:mock_chain_validation-nightly-*` Docker images). These binaries intentionally bypass halting validation and must never be run on mainnet or any node whose state you trust; they exist for forked-state replays and similar diagnostic scenarios.
Build-tag behavior:
* **default (production)** — no failures are swallowed; the counter stays at zero and every validation failure halts.
* **`mock_block_validation`** — swallows only `app_hash` and `data_hash` failures, preserving that tag's long-standing behavior; all other failures still halt.
* **`mock_chain_validation`** — swallows every swallow-eligible failure except `last_commit_verify` (which is excluded to avoid a downstream panic and therefore still halts). Intended for forked-state replays.
### `sei_unsafe_validation_skipped_total` `kind` values
| `kind` | Swallowed validation failure |
| ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
| `app_hash` | Block `AppHash` did not match the expected application hash. |
| `data_hash` | Block `DataHash` did not match the hash of the block data. |
| `last_results_hash` | Block `LastResultsHash` did not match the expected value. |
| `last_block_id` | Block `LastBlockID` did not match the expected previous block ID. |
| `consensus_hash` | Block `ConsensusHash` did not match the hash of the consensus params. |
| `validators_hash` | Block `ValidatorsHash` did not match the current validator set hash. |
| `next_validators_hash` | Block `NextValidatorsHash` did not match the next validator set hash. |
| `last_commit_verify` | `LastCommit` verification failed. Never swallowed by `mock_chain_validation` (excluded from the swallow set) — always halts. |
| `proposer_not_in_validator_set` | Block proposer address is not a member of the validator set. |
| `evidence_overflow` | Block evidence exceeded the maximum allowed byte size. |
| `last_commit_hash` | Block `LastCommitHash` did not match the expected value. |
| `evidence_hash` | Block `EvidenceHash` did not match the hash of the block evidence. |
| `per_evidence_validate_basic` | An individual evidence item failed `ValidateBasic`. |
## Backup Management
```bash theme={"dark"}
#!/bin/bash
BACKUP_DIR="/backup/sei"
DATE=$(date +%Y%m%d)
NODE_HOME="/root/.sei"
# Create backup directory
mkdir -p $BACKUP_DIR
# Stop service
systemctl stop seid
# Backup configuration
tar czf $BACKUP_DIR/sei-config-$DATE.tar.gz $NODE_HOME/config
# Backup data directory
tar czf $BACKUP_DIR/sei-data-$DATE.tar.gz $NODE_HOME/data
# Backup key files
tar czf $BACKUP_DIR/sei-keys-$DATE.tar.gz $NODE_HOME/keyring-file
# Start service
systemctl start seid
# Remove backups older than 7 days
find $BACKUP_DIR -type f -mtime +7 -name '*.tar.gz' -delete
# Log backup completion
echo "Backup completed successfully on $(date)" >> $BACKUP_DIR/backup.log
```
## Host System Monitoring
### Resource Usage Tracking
Install and configure node\_exporter:
```bash theme={"dark"}
wget https://github.com/prometheus/node_exporter/releases/download/v1.5.0/node_exporter-1.5.0.linux-amd64.tar.gz
tar xvf node_exporter-1.5.0.linux-amd64.tar.gz
```
Add to Prometheus configuration:
```yaml theme={"dark"}
scrape_configs:
- job_name: 'node'
static_configs:
- targets: ['localhost:9100']
```
## Performance Testing
```js theme={"dark"}
import { ethers } from 'ethers';
// Configuration
const EVM_RPC_URL = 'http://localhost:8545'; // EVM RPC endpoint to test
const CONTRACT_ADDRESS = '0x0000000000000000000000000000000000001002'; // replace with very active contract for best results
const INITIAL_BLOCK_RANGE = 50; // range of blocks to query using 'eth_getLogs'
const RANGE_INCREMENT = 10; // additional blocks to query each consecutive round
const MAX_TESTS = 50; // total number of rounds for testing
// Store metrics for final analysis
const metrics = [];
function getResponseSize(logs) {
return Buffer.byteLength(JSON.stringify(logs), 'utf8');
}
function formatBytes(bytes) {
if (bytes === 0) return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(2))} ${sizes[i]}`;
}
function padString(str, length) {
return String(str).padEnd(length);
}
function analyzeResults(metrics) {
console.log('\nPerformance Analysis');
console.log('='.repeat(50));
// Filter out queries with no logs for meaningful statistics
const queriesWithLogs = metrics.filter((m) => m.logsCount > 0);
const totalQueries = metrics.length;
console.log(`\nGeneral Statistics:`);
console.log(`Total Queries Run: ${totalQueries}`);
console.log(`Queries with Logs: ${queriesWithLogs.length}`);
console.log(`Empty Responses: ${totalQueries - queriesWithLogs.length}`);
if (queriesWithLogs.length > 0) {
const avgResponseTime = queriesWithLogs.reduce((acc, m) => acc + m.responseTime, 0) / queriesWithLogs.length;
const avgLogsPerQuery = queriesWithLogs.reduce((acc, m) => acc + m.logsCount, 0) / queriesWithLogs.length;
const maxLogs = Math.max(...queriesWithLogs.map((m) => m.logsCount));
const maxLogsQuery = queriesWithLogs.find((m) => m.logsCount === maxLogs);
console.log(`\nPerformance Metrics:`);
console.log(`Average Response Time (with logs): ${avgResponseTime.toFixed(2)}ms`);
console.log(`Average Logs per Query: ${avgLogsPerQuery.toFixed(2)}`);
console.log(`Maximum Logs in Single Query: ${maxLogs}`);
if (maxLogsQuery) {
console.log(`- At Range Size: ${maxLogsQuery.rangeSize} blocks`);
console.log(`- Response Time: ${maxLogsQuery.responseTime}ms`);
console.log(`- Efficiency: ${maxLogsQuery.logsPerMs.toFixed(3)} logs/ms`);
}
// Identify optimal range size based on logs/ms
const bestEfficiency = queriesWithLogs.reduce((best, m) => (m.logsPerMs > best.logsPerMs ? m : best));
console.log(`\nOptimal Performance:`);
console.log(`Best Efficiency: ${bestEfficiency.logsPerMs.toFixed(3)} logs/ms`);
console.log(`- At Range Size: ${bestEfficiency.rangeSize} blocks`);
console.log(`- Retrieved ${bestEfficiency.logsCount} logs in ${bestEfficiency.responseTime}ms`);
}
}
async function testEthGetLogs() {
const provider = new ethers.JsonRpcProvider(EVM_RPC_URL);
try {
const latestBlock = await provider.getBlockNumber();
console.log(`Latest block: ${latestBlock} (0x${latestBlock.toString(16)})`);
let currentToBlock = latestBlock;
let currentRange = INITIAL_BLOCK_RANGE;
let testCount = 0;
// Column headers with fixed widths
console.log('\nBlock Range Time Logs Size B/ms Logs/ms KB/Log Range');
console.log('='.repeat(80));
while (testCount < MAX_TESTS && currentToBlock > 0) {
const fromBlock = Math.max(0, currentToBlock - currentRange);
try {
const startTime = Date.now();
const filter = {
fromBlock: fromBlock,
toBlock: currentToBlock,
address: CONTRACT_ADDRESS
};
const logs = await provider.getLogs(filter);
const endTime = Date.now();
const responseTime = endTime - startTime;
const logsCount = logs.length;
const responseSize = getResponseSize(logs);
// Calculate metrics
const bytesPerMs = (responseSize / responseTime).toFixed(1);
const logsPerMs = (logsCount / responseTime).toFixed(3);
const kbPerLog = logsCount > 0 ? (responseSize / 1024 / logsCount).toFixed(2) : 'N/A';
// Store metrics for analysis
metrics.push({
rangeSize: currentRange,
responseTime,
logsCount,
responseSize,
bytesPerMs: parseFloat(bytesPerMs),
logsPerMs: parseFloat(logsPerMs),
kbPerLog: kbPerLog !== 'N/A' ? parseFloat(kbPerLog) : 0
});
// Format block range
const rangeDisplay = `${fromBlock.toString(16)}-${currentToBlock.toString(16)}`;
// Log with fixed column widths
console.log(padString(rangeDisplay, 17) + padString(responseTime, 6) + padString(logsCount, 8) + padString(formatBytes(responseSize), 9) + padString(bytesPerMs, 8) + padString(logsPerMs, 9) + padString(kbPerLog, 8) + currentRange);
if (logsCount === 10000) {
console.log(`\nWarning: Hit 10000 log limit at range ${currentRange}`);
}
currentToBlock = fromBlock - 1;
currentRange += RANGE_INCREMENT;
testCount++;
} catch (error) {
console.log(`Error at range ${currentRange}: ${error.message}`);
currentRange = Math.max(INITIAL_BLOCK_RANGE, currentRange - RANGE_INCREMENT);
currentToBlock = fromBlock - 1;
testCount++;
}
await new Promise((resolve) => setTimeout(resolve, 1000));
}
// Perform final analysis
analyzeResults(metrics);
} catch (error) {
console.error('Failed to initialize or get latest block:', error);
process.exit(1);
}
}
// Run the test
testEthGetLogs();
```
For specific customizations or additional metrics, consult the Sei
technical communities in [Telegram](https://t.me/+ZN-NcvOWStQwMzk0) or
[Discord](https://discord.gg/sei).