monitoring-tools►Prometheus

Prometheus监控cassandra

  • 1、将jmx jar包上传到cassandra安装目录的lib目录下

  • 2、在conf目录下新建cassandra-jmx.yml

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    lowercaseOutputName: true
    lowercaseOutputLabelNames: true
    whitelistObjectNames: [
    "org.apache.cassandra.metrics:type=ColumnFamily,name=RangeLatency,*",
    "org.apache.cassandra.metrics:type=ColumnFamily,name=LiveSSTableCount,*",
    "org.apache.cassandra.metrics:type=ColumnFamily,name=SSTablesPerReadHistogram,*",
    "org.apache.cassandra.metrics:type=ColumnFamily,name=SpeculativeRetries,*",
    "org.apache.cassandra.metrics:type=ColumnFamily,name=MemtableOnHeapSize,*",
    "org.apache.cassandra.metrics:type=ColumnFamily,name=MemtableSwitchCount,*",
    "org.apache.cassandra.metrics:type=ColumnFamily,name=MemtableLiveDataSize,*",
    "org.apache.cassandra.metrics:type=ColumnFamily,name=MemtableColumnsCount,*",
    "org.apache.cassandra.metrics:type=ColumnFamily,name=MemtableOffHeapSize,*",
    "org.apache.cassandra.metrics:type=ColumnFamily,name=BloomFilterFalsePositives,*",
    "org.apache.cassandra.metrics:type=ColumnFamily,name=BloomFilterFalseRatio,*",
    "org.apache.cassandra.metrics:type=ColumnFamily,name=BloomFilterDiskSpaceUsed,*",
    "org.apache.cassandra.metrics:type=ColumnFamily,name=BloomFilterOffHeapMemoryUsed,*",
    "org.apache.cassandra.metrics:type=ColumnFamily,name=SnapshotsSize,*",
    "org.apache.cassandra.metrics:type=ColumnFamily,name=TotalDiskSpaceUsed,*",
    "org.apache.cassandra.metrics:type=CQL,name=RegularStatementsExecuted,*",
    "org.apache.cassandra.metrics:type=CQL,name=PreparedStatementsExecuted,*",
    "org.apache.cassandra.metrics:type=Compaction,name=PendingTasks,*",
    "org.apache.cassandra.metrics:type=Compaction,name=CompletedTasks,*",
    "org.apache.cassandra.metrics:type=Compaction,name=BytesCompacted,*",
    "org.apache.cassandra.metrics:type=Compaction,name=TotalCompactionsCompleted,*",
    "org.apache.cassandra.metrics:type=ClientRequest,name=Latency,*",
    "org.apache.cassandra.metrics:type=ClientRequest,name=Unavailables,*",
    "org.apache.cassandra.metrics:type=ClientRequest,name=Timeouts,*",
    "org.apache.cassandra.metrics:type=Storage,name=Exceptions,*",
    "org.apache.cassandra.metrics:type=Storage,name=TotalHints,*",
    "org.apache.cassandra.metrics:type=Storage,name=TotalHintsInProgress,*",
    "org.apache.cassandra.metrics:type=Storage,name=Load,*",
    "org.apache.cassandra.metrics:type=Connection,name=TotalTimeouts,*",
    "org.apache.cassandra.metrics:type=ThreadPools,name=CompletedTasks,*",
    "org.apache.cassandra.metrics:type=ThreadPools,name=PendingTasks,*",
    "org.apache.cassandra.metrics:type=ThreadPools,name=ActiveTasks,*",
    "org.apache.cassandra.metrics:type=ThreadPools,name=TotalBlockedTasks,*",
    "org.apache.cassandra.metrics:type=ThreadPools,name=CurrentlyBlockedTasks,*",
    "org.apache.cassandra.metrics:type=DroppedMessage,name=Dropped,*",
    "org.apache.cassandra.metrics:type=Cache,scope=KeyCache,name=HitRate,*",
    "org.apache.cassandra.metrics:type=Cache,scope=KeyCache,name=Hits,*",
    "org.apache.cassandra.metrics:type=Cache,scope=KeyCache,name=Requests,*",
    "org.apache.cassandra.metrics:type=Cache,scope=KeyCache,name=Entries,*",
    "org.apache.cassandra.metrics:type=Cache,scope=KeyCache,name=Size,*",
    #"org.apache.cassandra.metrics:type=Streaming,name=TotalIncomingBytes,*",
    #"org.apache.cassandra.metrics:type=Streaming,name=TotalOutgoingBytes,*",
    "org.apache.cassandra.metrics:type=Client,name=connectedNativeClients,*",
    "org.apache.cassandra.metrics:type=Client,name=connectedThriftClients,*",
    "org.apache.cassandra.metrics:type=Table,name=WriteLatency,*",
    "org.apache.cassandra.metrics:type=Table,name=ReadLatency,*",
    "org.apache.cassandra.net:type=FailureDetector,*",
    ]
    #blacklistObjectNames: ["org.apache.cassandra.metrics:type=ColumnFamily,*"]
    rules:
    - pattern: org.apache.cassandra.metrics<type=(ConnectionStreaming), scope=(\S*), name=(\S*)><>(CountValue)
    name: cassandra_$1_$3
    labels:
      address: "$2"
    - pattern: org.apache.cassandra.metrics<type=(ColumnFamily), name=(RangeLatency)><>(Mean)
    name: cassandra_$1_$2_$3
    - pattern: org.apache.cassandra.net<type=(FailureDetector)><>(DownEndpointCount)
    name: cassandra_$1_$2
    - pattern: org.apache.cassandra.metrics<type=(Keyspace), keyspace=(\S*), name=(\S*)><>(CountMean95thPercentile)
    name: cassandra_$1_$3_$4
    labels:
      "$1": "$2"
    - pattern: org.apache.cassandra.metrics<type=(Table), keyspace=(\S*), scope=(\S*), name=(\S*)><>(CountMean95thPercentile)
    name: cassandra_$1_$4_$5
    labels:
      "keyspace": "$2"
      "table": "$3"
    - pattern: org.apache.cassandra.metrics<type=(ClientRequest), scope=(\S*), name=(\S*)><>(CountMean95thPercentile)
    name: cassandra_$1_$3_$4
    labels:
      "type": "$2"
    - pattern: org.apache.cassandra.metrics<type=(\S*)(?:, ((?!scope)\S*)=(\S*))?(?:, scope=(\S*))?,
      name=(\S*)><>(CountValue)
    name: cassandra_$1_$5
    labels:
      "$1": "$4"
      "$2": "$3"

  • 3、在conf目录下cassandra-env.sh增加javaagent

    1
    JVM_OPTS="$JVM_OPTS -javaagent:$CASSANDRA_HOME/lib/jamm-0.3.0.jar -javaagent:$CASSANDRA_HOME/lib/jmx_prometheus_javaagent-0.16.0.jar=7070:${CASSANDRA_HOME}/conf/cassandra-jmx.yml"

  • 4、etcd注册

    1
    etcdctl put /prometheus/discovery/cassandra/cassandra-v2-2 '{"name":"cassandra-v2-2","address":"10.10.21.177:7070","labels": [{"key":"c_type","val":"cassandra"},{"key":"ip","val":"10.10.21.177"}]}'