Projet

Général

Profil

Révision ef960abc

IDef960abcf9efcd8a35d3fd0c27c84e4de95f15cf
Parent 538cdc93
Enfant b9781d98

Ajouté par antonio il y a plus de 12 ans

Commit includes: fix slash in url params, examples update, fix issue #1 and #2, qps graph uses request instead of avgRequestPerSecond, new 'memory' graph, deprecation of availableram parameter

Voir les différences:

plugins/solr/solr4_
20 20
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 21
# DEALINGS IN THE SOFTWARE.
22 22
#
23
# Munin plugin for monitoring a multicore solr 4.* installation via mbean.
24
# It calls:
25
# > http://localhost:8080/solr/admin/cores?action=STATUS&wt=json
26
# and
27
# > http://localhost:8080/solr/corename/admin/mbeans?stats=true&wt=json
28
# for each core to retrieve cores and data. Verify those urls on your instance.
23
# Solr 4.* munin graph plugin
24
# Plugins configuration parameters:
29 25
#
30
# Configuration parameters:
31 26
# [solr_*]
32
#    host_port <host:port>
33
#    qpshandler_<handlerlabel> <handlerpath>
34
#    availableram <ramsize in bytes>
27
#    env.host_port <host:port>
28
#    env.url <default /solr>
29
#    env.qpshandler_<handlerlabel> <handlerpath>
35 30
#
36
# Example:
37
#    host_port solrhost:8080 
38
#    qpshandler_select /select
39
#    availableram 3221225472
31
#    ex:
32
#        env.host_port solrhost:8080 
33
#        env.url /solr
34
#        env.qpshandler_select /select
40 35
#
41
# Defined checks:
42
#    numdocs  
43
#    qps  
44
#    indexsize  
45
#    requesttimes  
46
#    documentcache  
47
#    fieldvaluecache  
48
#    filtercache  
49
#    queryresultcache  
50
#
51
# Installation example:
36
# Install plugins:
52 37
#    ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_numdocs_core_1
53 38
#    ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_requesttimes_select
54
#    ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_qps_core_1_select
39
#    ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_core_1_select
40
#    ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_indexsize
41
#    ln -s /usr/share/munin/plugins/solr_.py /etc/munin/plugins/solr_memory
55 42
#
56
# Source repo: https://github.com/averni/munin-solr
43

  
57 44

  
58 45
import sys
59 46
import os
......
72 59
        data = params['core'].rsplit('_', 1)
73 60
        handler = data.pop()
74 61
        params['params'] = {
75
                'handler': os.environ.get('qpshandler_%s' % handler, '/select')
62
                'handler': os.environ.get('qpshandler_%s' % handler, 'standard')
76 63
        }
77 64
        if not data:
78 65
            params['core'] = ''
......
90 77

  
91 78
class JSONReader:
92 79
    @classmethod
93
    def readValue(cls, struct, path):
80
    def readValue(cls, struct, path, convert = None):
94 81
        if not path[0] in struct:
95 82
            return -1
96 83
        obj = struct[path[0]]
......
98 85
            return -1
99 86
        for k in path[1:]:
100 87
            obj = obj[k]
88
        if convert:
89
            return convert(obj)
101 90
        return obj
102 91

  
103 92
class SolrCoresAdmin:
104
    def __init__(self, host):
93
    def __init__(self, host, solrurl):
105 94
        self.host = host
95
        self.solrurl = solrurl
106 96
        self.data = None
107 97

  
108 98
    def fetchcores(self):
109
        uri = "/solr/admin/cores?action=STATUS&wt=json"
99
        uri = os.path.join(self.solrurl, "admin/cores?action=STATUS&wt=json")
110 100
        conn = httplib.HTTPConnection(self.host)
111 101
        conn.request("GET", uri)
112 102
        res = conn.getresponse()
......
135 125
            return ret
136 126

  
137 127
class SolrCoreMBean:
138
    def __init__(self, host, core):
128
    def __init__(self, host, solrurl, core):
139 129
        self.host = host
140 130
        self.data = None
141 131
        self.core = core
132
        self.solrurl = solrurl
142 133

  
143 134
    def _fetch(self):
144
        uri = "/solr/%s/admin/mbeans?stats=true&wt=json" % self.core
135
        uri = os.path.join(self.solrurl, "%s/admin/mbeans?stats=true&wt=json" % self.core)
145 136
        conn = httplib.HTTPConnection(self.host)
146 137
        conn.request("GET", uri)
147 138
        res = conn.getresponse()
......
159 150
                data[key] = el
160 151
            else:
161 152
                key = el
153
        self._fetchSystem()
154

  
155
    def _fetchSystem(self):
156
        uri = os.path.join(self.solrurl, "%s/admin/system?stats=true&wt=json" % self.core)
157
        conn = httplib.HTTPConnection(self.host)
158
        conn.request("GET", uri)
159
        res = conn.getresponse()
160
        data = res.read()
161
        if res.status != 200:
162
            raise CheckException("System fetch failed: %s\n%s" %( str(res.status), res.read()))
163
        self.data['system'] = json.loads(data)
164

  
165

  
166
    def _readInt(self, path):
167
        return self._read(path, int)
168

  
169
    def _readFloat(self, path):
170
        return self._read(path, float)
162 171

  
163
    def _read(self, path):
172
    def _read(self, path, convert = None):
164 173
        if self.data is None:
165 174
            self._fetch()
166
        return JSONReader.readValue(self.data, path)
175
        return JSONReader.readValue(self.data, path, convert)
167 176

  
168 177
    def _readCache(self, cache):
169 178
        result = {}
170
        for key in ['lookups', 'hits', 'inserts', 'evictions', 'hitratio']:
179
        for key, ftype in [('lookups', int), ('hits', int), ('inserts', int), ('evictions', int), ('hitratio', float)]:
171 180
            path = ['solr-mbeans', 'CACHE', cache, 'stats', 'cumulative_%s' % key]
172
            result[key] = self._read(path)
173
        result['size'] = self._read(['solr-mbeans', 'CACHE', cache, 'stats', 'size'])
181
            result[key] = self._read(path, ftype)
182
        result['size'] = self._readInt(['solr-mbeans', 'CACHE', cache, 'stats', 'size'])
174 183
        return result
175 184

  
176 185
    def getCore(self):
177 186
        return self.core
178 187

  
188
    def requestcount(self, handler):
189
        path = ['solr-mbeans', 'QUERYHANDLER', handler, 'stats', 'requests']
190
        return self._readInt(path)
191

  
179 192
    def qps(self, handler):
180 193
        path = ['solr-mbeans', 'QUERYHANDLER', handler, 'stats', 'avgRequestsPerSecond']
181
        return self._read(path)
194
        return self._readFloat(path)
182 195

  
183 196
    def requesttimes(self, handler):
184 197
        times = {}
185 198
        path = ['solr-mbeans', 'QUERYHANDLER', handler, 'stats']
186 199
        for perc in ['avgTimePerRequest', '75thPcRequestTime', '99thPcRequestTime']:
187
            times[perc] = self._read(path + [perc])
200
            times[perc] = self._read(path + [perc], float)
188 201
        return times
189 202

  
190 203
    def numdocs(self):
191 204
        path = ['solr-mbeans', 'CORE', 'searcher', 'stats', 'numDocs']
192
        return self._read(path)
205
        return self._readInt(path)
193 206

  
194 207
    def documentcache(self):
195 208
        return self._readCache('documentCache')
......
203 216
    def queryresultcache(self):
204 217
        return self._readCache('queryResultCache')
205 218

  
219
    def memory(self):
220
        data = self._read(['system', 'jvm', 'memory', 'raw'])
221
        del data['used%']
222
        for k in data.keys():
223
            data[k] = int(data[k])
224
        return data
225

  
206 226
#############################################################################
207 227
# Graph Templates
208 228

  
......
240 260

  
241 261
"""
242 262

  
243
QPSMAIN_GRAPH_TPL = """graph_title Solr {core} {handler} Request per second"
244
graph_args -l 0
263
QPSMAIN_GRAPH_TPL = """graph_title Solr {core} {handler} Request per second
264
graph_args --base 1000 -r --lower-limit 0
265
graph_scale no
245 266
graph_vlabel request / second
246 267
graph_category solr
268
graph_period second
269
graph_order {gorder}
247 270
{cores_qps_graphs}"""
248 271

  
249 272
QPSCORE_GRAPH_TPL = """qps_{core}.label {core} Request per second
250
qps_{core}.type LINESTACK1
273
qps_{core}.draw {gtype}
274
qps_{core}.type DERIVE
275
qps_{core}.min 0
251 276
qps_{core}.graph yes"""
252 277

  
253 278
REQUESTTIMES_GRAPH_TPL = """multigraph {core}_requesttimes
......
256 281
graph_vlabel millis
257 282
graph_category solr
258 283
savgtimeperrequest_{core}.label {core} Avg time per request
259
savgtimeperrequest_{core}.type gauge
284
savgtimeperrequest_{core}.type GAUGE
260 285
savgtimeperrequest_{core}.graph yes
261 286
s75thpcrequesttime_{core}.label {core} 75th perc
262
s75thpcrequesttime_{core}.type gauge
287
s75thpcrequesttime_{core}.type GAUGE
263 288
s75thpcrequesttime_{core}.graph yes
264 289
s99thpcrequesttime_{core}.label {core} 99th perc
265
s99thpcrequesttime_{core}.type gauge
290
s99thpcrequesttime_{core}.type GAUGE
266 291
s99thpcrequesttime_{core}.graph yes
267

  
268 292
"""
269 293

  
270 294
NUMDOCS_GRAPH_TPL = """graph_title Solr Docs %s
......
272 296
docs.label Docs
273 297
graph_category solr"""
274 298

  
275
INDEXSIZE_GRAPH_TPL = """graph_args --base 1024 -l 0 --upper-limit {availableram}
299
INDEXSIZE_GRAPH_TPL = """graph_args --base 1024 -l 0 
276 300
graph_vlabel Bytes
277 301
graph_title Index Size
278 302
graph_category solr
279
graph_info Solr Index Memory Usage.
303
graph_info Solr Index Size.
280 304
graph_order {cores}
281 305
{cores_config}
306
xmx.label Xmx
307
xmx.colour ff0000
282 308
"""
283 309

  
284 310
INDEXSIZECORE_GRAPH_TPL = """{core}.label {core}
285 311
{core}.draw STACK""" 
286 312

  
313
MEMORYUSAGE_GRAPH_TPL = """graph_args --base 1024 -l 0 --upper-limit {availableram}
314
graph_vlabel Bytes
315
graph_title Solr memory usage
316
graph_category solr
317
graph_info Solr Memory Usage.
318
used.label Used
319
max.label Max
320
max.colour ff0000
321
"""
322

  
287 323
#############################################################################
288 324
# Graph managment
289
CHECKS_DEFINED = [
290
    'numdocs',
291
    'qps',
292
    'indexsize',
293
    'requesttimes',
294
    'documentcache',
295
    'fieldvaluecache',
296
    'filtercache',
297
    'queryresultcache'
298
]
299 325

  
300 326
class SolrMuninGraph:
301
    def __init__(self, hostport, solrmbean):
302
        self.solrcoresadmin = SolrCoresAdmin(hostport)
327
    def __init__(self, hostport, solrurl, params):
328
        self.solrcoresadmin = SolrCoresAdmin(hostport, solrurl)
303 329
        self.hostport = hostport
330
        self.solrurl = solrurl
304 331
        self.params = params
305 332

  
306 333
    def _getMBean(self, core):
307
        return SolrCoreMBean(self.hostport, core)
334
        return SolrCoreMBean(self.hostport, self.solrurl, core)
308 335

  
309 336
    def _cacheConfig(self, cacheType, cacheName):
310 337
        return CACHE_GRAPH_TPL.format(core=self.params['core'], cacheType=cacheType, cacheName=cacheName)
......
318 345
        data = getattr(solrmbean, cacheType)()
319 346
        results.append('multigraph solr_{core}_{cacheType}_hit_rates'.format(core=self.params['core'], cacheType=cacheType))
320 347
        for label in hits_fields:
321
            results.append("%s.value %s" % (label, data[label]))
348
            results.append("%s.value %.8f" % (label, data[label]))
322 349
        results.append('multigraph solr_{core}_{cacheType}_size'.format(core=self.params['core'], cacheType=cacheType))
323 350
        for label in size_fields:
324
            results.append("%s.value %s" % (label, data[label]))
351
            results.append("%s.value %d" % (label, data[label]))
325 352
        return "\n".join(results)
326 353

  
327 354
    def config(self, mtype):
328
        if not mtype:
329
            raise CheckException("""Check missing. Available checks: \n\t%s""" % '\n\t'.join(CHECKS_DEFINED))
330
        if not hasattr(self, '%sConfig' % mtype):
355
        if not mtype or not hasattr(self, '%sConfig' % mtype):
331 356
            raise CheckException("Unknown check %s" % mtype)
332 357
        return getattr(self, '%sConfig' % mtype)()
333 358

  
......
345 370

  
346 371
    def qpsConfig(self):
347 372
        cores = self._getCores()
348
        graph = [QPSCORE_GRAPH_TPL.format(core=c) for c in cores ]
373
        graph = [QPSCORE_GRAPH_TPL.format(core=c, gtype='LINESTACK1') for pos,c in enumerate(cores) ]
349 374
        return QPSMAIN_GRAPH_TPL.format(
350 375
            cores_qps_graphs='\n'.join(graph), 
351 376
            handler=self.params['params']['handler'], 
352 377
            core=self.params['core'], 
353
            cores_qps_cdefs='%s,%s' % (','.join(map(lambda x: 'qps_%s' % x, cores)),','.join(['+']*(len(cores)-1)))
378
            cores_qps_cdefs='%s,%s' % (','.join(map(lambda x: 'qps_%s' % x, cores)),','.join(['+']*(len(cores)-1))), 
379
            gorder=','.join(cores)
354 380
        )
355 381

  
356 382
    def qps(self):
......
358 384
        cores = self._getCores()
359 385
        for c in cores:
360 386
            mbean = self._getMBean(c)
361
            results.append('qps_%s.value %s' % (c, mbean.qps(self.params['params']['handler'])))
387
            results.append('qps_%s.value %d' % (c, mbean.requestcount(self.params['params']['handler'])))
362 388
        return '\n'.join(results)
363 389

  
364 390
    def requesttimesConfig(self):
......
373 399
            mbean = self._getMBean(c)
374 400
            results.append('multigraph {core}_requesttimes'.format(core=c))
375 401
            for k, time in mbean.requesttimes(self.params['params']['handler']).items():
376
                results.append('s%s_%s.value %s' % (k.lower(), c, time))
402
                results.append('s%s_%s.value %.5f' % (k.lower(), c, time))
377 403
        return '\n'.join(results)
378 404

  
379 405
    def numdocsConfig(self):
......
381 407

  
382 408
    def numdocs(self):
383 409
        mbean = self._getMBean(self.params['core'])
384
        return 'docs.value %s' % mbean.numdocs(**self.params['params'])
410
        return 'docs.value %d' % mbean.numdocs(**self.params['params'])
385 411

  
386 412
    def indexsizeConfig(self):
387 413
        cores = self._getCores()
388
        availableram = os.environ.get('availableram', 16868532224)
389 414
        graph = [ INDEXSIZECORE_GRAPH_TPL.format(core=c) for c in cores]
390
        return INDEXSIZE_GRAPH_TPL.format(cores=" ".join(cores), cores_config="\n".join(graph), availableram=availableram)
415
        return INDEXSIZE_GRAPH_TPL.format(cores=" ".join(cores), cores_config="\n".join(graph))
391 416

  
392 417
    def indexsize(self):
393 418
        results = []
394 419
        for c, size in self.solrcoresadmin.indexsize(**self.params['params']).items():
395
            results.append("%s.value %s" % (c, size))
420
            results.append("%s.value %d" % (c, size))
421
        cores = self._getCores()
422
        mbean = self._getMBean(cores[0])
423
        memory = mbean.memory()
424
        results.append('xmx.value %d' % memory['max'])
396 425
        return "\n".join(results)
397 426

  
427
    def memoryConfig(self):
428
        cores = self._getCores()
429
        mbean = self._getMBean(cores[0])
430
        memory = mbean.memory()
431
        return MEMORYUSAGE_GRAPH_TPL.format(availableram=memory['max'] * 1.05)
432

  
433
    def memory(self):
434
        results = []
435
        cores = self._getCores()
436
        mbean = self._getMBean(cores[0])
437
        memory = mbean.memory()
438
        return '\n'.join(['used.value %d' % memory['used'], 'max.value %d' % memory['max']])
439

  
398 440
    def documentcacheConfig(self):
399 441
        return self._cacheConfig('documentcache', 'Document Cache')
400 442

  
......
422 464
if __name__ == '__main__':
423 465
    params = parse_params()
424 466
    SOLR_HOST_PORT = os.environ.get('host_port', 'localhost:8080').replace('http://', '')
425
    mb = SolrMuninGraph(SOLR_HOST_PORT, params)
426
    try:
427
        if hasattr(mb, params['op']):
428
            print getattr(mb,  params['op'])(params['type'])
429
    except Exception, ex:
430
        print "ERROR: %s" % ex
431
        exit(1)
467
    SOLR_URL  = os.environ.get('url', '/solr')
468
    if SOLR_URL[0] != '/':
469
        SOLR_URL = '/' + SOLR_URL 
470
    mb = SolrMuninGraph(SOLR_HOST_PORT, SOLR_URL, params)
471
    if hasattr(mb, params['op']):
472
        print getattr(mb,  params['op'])(params['type'])
473

  

Formats disponibles : Unified diff