Projet

Général

Profil

Révision 5f9e882b

ID5f9e882bced3a34c167c0bb5603a05f74e8cf3e9
Parent 13bd1599
Enfant 734da6b9

Ajouté par Lars Kruse il y a presque 7 ans

nginx_upstream_multi_: fix flake8 issues

Voir les différences:

plugins/nginx/nginx_upstream_multi_
1 1
#!/usr/bin/env python3
2 2
#
3
# Munin plugin to monitor requests number, cache statuses, http status codes and average request times of
4
# specified nginx upstreams.
3
# Munin plugin to monitor requests number, cache statuses, http status codes and average request
4
# times of specified nginx upstreams.
5 5
#
6 6
# Copyright Igor Borodikhin
7 7
#
8 8
# License : GPLv3
9 9
#
10 10
# Configuration parameters:
11
# env.graphs - which graphs to produce (optional, list of graphs separated by spaces, default - cache http time request)
11
# env.graphs - which graphs to produce (optional, list of graphs separated by spaces, default -
12
#              cache http time request)
12 13
# env.log - log file path (mandatory, ex.: /var/log/nginx/upstream.log)
13
# env.upstream - list of upstreams to monitor (mandatory, including port numbers separated by space, ex.: 10.0.0.1:80 10.0.0.2:8080)
14
# env.statuses - list of http status codes to monitor (optional, default - all statuses, ex.: 200 403 404 410 500 502)
15
# env.percentiles - which percentiles to draw on time graphs (optional, list of percentiles separated by spaces, default - 80)
14
# env.upstream - list of upstreams to monitor (mandatory, including port numbers separated by
15
#                space, e.g.: 10.0.0.1:80 10.0.0.2:8080)
16
# env.statuses - list of http status codes to monitor (optional, default - all statuses,
17
#                e.g.: 200 403 404 410 500 502)
18
# env.percentiles - which percentiles to draw on time graphs (optional, list of percentiles
19
#                   separated by spaces, default - 80)
16 20
#
17 21
# ## Installation
18
# Copy file to directory /usr/share/munin/pligins/ and create symbolic link(s) for each log file you wish to monitor.
22
# Copy file to directory /usr/share/munin/pligins/ and create symbolic link(s) for each log file
23
# you wish to monitor.
19 24
#
20 25
# Specify log_format at /etc/nginx/conf.d/upstream.conf:
21
# log_format upstream "ua=[$upstream_addr] ut=[$upstream_response_time] us=[$upstream_status] cs=[$upstream_cache_status]"
26
# log_format upstream "ua=[$upstream_addr] ut=[$upstream_response_time] us=[$upstream_status] \
27
#     cs=[$upstream_cache_status]"
22 28
#
23 29
# Use it in your site configuration (/etc/nginx/sites-enabled/anything.conf):
24 30
# access_log /var/log/nginx/upstream.log upstream;
25 31
#
26
# Attention! Because munin-node does not have read permission for nginx log files we need to run it as root.
32
# Attention! Since the default user (nobody) does not have read permission for nginx log files we
33
# need to run it as root.
27 34
#
28 35
# And specify some options in /etc/munin/plugin-conf.d/munin-node:
29 36
#
......
35 42
#     env.statuses 200 403 404 410 500 502
36 43
#     env.percentiles 50 80
37 44
#
38
#%# family=contrib
45
#  #%# family=contrib
39 46

  
40 47
import copy
41 48
import math
......
66 73
    logPath = "/var/log/nginx/access.log"
67 74

  
68 75
# Http statuses list
69
httpStatusString = ("100:Continue;101:Switching protocols;102:Processing;200:OK;201:Created;202:Accepted;"
70
"203:Non-Authoritative Information;204:No content;205:Reset content;206:Partial content;207:Multi-status;"
71
"226:IM used;300:Multiple choices;301:Moved permanently;302:Moved temporarily;303:See other;304:Not modified;"
72
"305:Use proxy;307:Temporary redirect;400:Bad request;401:Unauthorized;402:Payment required;403:Forbidden;"
73
"404:Not found;405:Method not allowed;406:Not acceptable;407:Proxy Authentication Required;408:Request timeout;"
74
"409:Conflict;410:Gone;411:Length required;412:Precondition failed;413:Request entity too large;"
75
"414:Request URI too large;415:Usupported media type;416:Request range not satisfiable;417:Expectation failed;"
76
"422:Unprocessable entity;423:Locked;424:Failed dependency;425:Unordered collection;426:Upgrade required;"
77
"449:Retry with;456:Unrecoverable error;500:Internal server error;501:Not implemented;502:Bad gateway;"
78
"503:Service unavailable;504:Gateway timeout;505:HTTP version not supported;506:Variant also negotiates;"
79
"507:Insufficient storage;508:Loop detected;509:Bandwidth limit exceeded;510:Not extended")
76
httpStatusString = (
77
    "100:Continue;101:Switching protocols;102:Processing;200:OK;201:Created;202:Accepted;"
78
    "203:Non-Authoritative Information;204:No content;205:Reset content;206:Partial content;"
79
    "207:Multi-status;226:IM used;300:Multiple choices;301:Moved permanently;"
80
    "302:Moved temporarily;303:See other;304:Not modified;305:Use proxy;307:Temporary redirect;"
81
    "400:Bad request;401:Unauthorized;402:Payment required;403:Forbidden;404:Not found;"
82
    "405:Method not allowed;406:Not acceptable;407:Proxy Authentication Required;"
83
    "408:Request timeout;409:Conflict;410:Gone;411:Length required;412:Precondition failed;"
84
    "413:Request entity too large;414:Request URI too large;415:Usupported media type;"
85
    "416:Request range not satisfiable;417:Expectation failed;422:Unprocessable entity;"
86
    "423:Locked;424:Failed dependency;425:Unordered collection;426:Upgrade required;"
87
    "449:Retry with;456:Unrecoverable error;500:Internal server error;501:Not implemented;"
88
    "502:Bad gateway;503:Service unavailable;504:Gateway timeout;505:HTTP version not supported;"
89
    "506:Variant also negotiates;507:Insufficient storage;508:Loop detected;"
90
    "509:Bandwidth limit exceeded;510:Not extended")
80 91

  
81 92
if "statuses" in os.environ:
82 93
    statuses = os.environ["statuses"].split()
......
88 99
    [code, title] = statusString.split(":")
89 100
    if len(statuses) > 0 and code in statuses or len(statuses) == 0:
90 101
        httpStatusList[code] = {
91
            "title" : title,
92
            "requests" : 0
102
            "title": title,
103
            "requests": 0
93 104
        }
94 105

  
95
cacheStatusList = { "MISS" : 0, "BYPASS" : 0, "EXPIRED" : 0, "UPDATING" : 0, "STALE" : 0, "HIT" : 0 }
106
cacheStatusList = {"MISS": 0, "BYPASS": 0, "EXPIRED": 0, "UPDATING": 0, "STALE": 0, "HIT": 0}
96 107

  
97 108
# Parse upstreams
98 109
upstreams = {}
......
101 112
    upstreamList = upstreamString.split()
102 113
    for upstream in upstreamList:
103 114
        upstreams[upstream] = {
104
            "requests" : 0,
105
            "time"     : 0,
106
            "times"    : [],
107
            "cache"    : copy.deepcopy(cacheStatusList),
108
            "http"     : copy.deepcopy(httpStatusList)
115
            "requests": 0,
116
            "time": 0,
117
            "times": [],
118
            "cache": copy.deepcopy(cacheStatusList),
119
            "http": copy.deepcopy(httpStatusList)
109 120
        }
110 121
else:
111 122
    raise Exception("No upstreams specified")
......
132 143
def sanitize(string):
133 144
    return string.replace(".", "_").replace(":", "_").replace("/", "_").replace("-", "_")
134 145

  
146

  
135 147
if len(sys.argv) == 2 and sys.argv[1] == "config":
136 148
    # Parent graph declaration
137 149
    print("multigraph nginx_upstream_multi_%s" % siteName.replace(".", "_"))
......
145 157
    if "request" in graphs_enabled:
146 158
        for upstream in upstreams.keys():
147 159
            print()
148
            print("multigraph nginx_upstream_multi_%s.%s_requests" % (sanitize(siteName), sanitize(upstream)))
160
            print("multigraph nginx_upstream_multi_%s.%s_requests"
161
                  % (sanitize(siteName), sanitize(upstream)))
149 162
            print("graph_title Requests number - %s" % upstream)
150 163
            print("graph_vlabel rps")
151 164
            print("graph_category webserver")
......
156 169
    if "time" in graphs_enabled:
157 170
        for upstream in upstreams.keys():
158 171
            print()
159
            print("multigraph nginx_upstream_multi_%s.%s_times" % (sanitize(siteName), sanitize(upstream)))
172
            print("multigraph nginx_upstream_multi_%s.%s_times"
173
                  % (sanitize(siteName), sanitize(upstream)))
160 174
            print("graph_title Request time - %s" % upstream)
161 175
            print("graph_vlabel sec.")
162 176
            print("graph_category webserver")
163 177
            print("us%s_times.label average" % (sanitize(upstream)))
164 178
            for percentile in percentiles:
165
                print("us%s_times_percentile_%s.label %s-percentile" % (sanitize(upstream), percentile, percentile))
179
                print("us%s_times_percentile_%s.label %s-percentile"
180
                      % (sanitize(upstream), percentile, percentile))
166 181
            print()
167 182

  
168 183
    # HTTP Status codes graph declaration
169 184
    if "http" in graphs_enabled:
170 185
        for upstream in upstreams.keys():
171 186
            print()
172
            print("multigraph nginx_upstream_multi_%s.%s_statuses" % (sanitize(siteName), sanitize(upstream)))
187
            print("multigraph nginx_upstream_multi_%s.%s_statuses"
188
                  % (sanitize(siteName), sanitize(upstream)))
173 189
            print("graph_title HTTP - %s" % upstream)
174 190
            print("graph_vlabel rps")
175 191
            print("graph_category webserver")
176 192
            for status in sorted(httpStatusList.keys()):
177
                print("http%s_%s_status.label %s - %s" % (status, sanitize(upstream), status, httpStatusList[status]["title"]))
193
                print("http%s_%s_status.label %s - %s"
194
                      % (status, sanitize(upstream), status, httpStatusList[status]["title"]))
178 195
            print()
179 196

  
180 197
    # Cache status graph declaration
181 198
    if "cache" in graphs_enabled:
182 199
        for upstream in upstreams.keys():
183 200
            print()
184
            print("multigraph nginx_upstream_multi_%s.%s_cache" % (sanitize(siteName), sanitize(upstream)))
201
            print("multigraph nginx_upstream_multi_%s.%s_cache"
202
                  % (sanitize(siteName), sanitize(upstream)))
185 203
            print("graph_title Cache - %s" % upstream)
186 204
            print("graph_vlabel rps")
187 205
            print("graph_category webserver")
......
199 217
    except Exception:
200 218
        lastByte = 0
201 219

  
202
    if lastByteHandle != None:
220
    if lastByteHandle is not None:
203 221
        lastByteHandle.close()
204 222

  
205 223
    try:
......
224 242
        if (match):
225 243
            # Extract data
226 244
            address = match.group(1)
227
            time    = match.group(2)
228
            status  = match.group(3)
229
            cache   = match.group(4)
245
            time = match.group(2)
246
            status = match.group(3)
247
            cache = match.group(4)
230 248

  
231 249
            # Replace separators by space
232 250
            address = address.replace(",", " ")
233 251
            address = address.replace(" : ", " ")
234
            address = re.sub("\s+", " ", address)
252
            address = re.sub(r"\s+", " ", address)
235 253

  
236
            time    = time.replace(",", " ")
237
            time    = time.replace(" : ", " ")
238
            time    = re.sub("\s+", " ", time)
254
            time = time.replace(",", " ")
255
            time = time.replace(" : ", " ")
256
            time = re.sub(r"\s+", " ", time)
239 257

  
240
            status  = status.replace(",", " ")
241
            status  = status.replace(" : ", " ")
242
            status  = re.sub("\s+", " ", status)
258
            status = status.replace(",", " ")
259
            status = status.replace(" : ", " ")
260
            status = re.sub(r"\s+", " ", status)
243 261

  
244
            cache   = cache.replace(",", " ")
245
            cache   = cache.replace(" : ", " ")
246
            cache   = re.sub("\s+", " ", cache)
262
            cache = cache.replace(",", " ")
263
            cache = cache.replace(" : ", " ")
264
            cache = re.sub(r"\s+", " ", cache)
247 265

  
248 266
            addresses = address.split()
249
            times     = time.split()
250
            statuses  = status.split()
251
            caches    = cache.split()
267
            times = time.split()
268
            statuses = status.split()
269
            caches = cache.split()
252 270

  
253 271
            index = 0
254 272
            for uAddress in addresses:
255 273
                if uAddress in upstreams.keys():
256 274
                    try:
257
                        uTime    = float(times[index])
275
                        uTime = float(times[index])
258 276
                    except ValueError:
259
                        uTime    = 0
277
                        uTime = 0
260 278

  
261 279
                    if index < len(statuses):
262
                        uStatus  = statuses[index]
280
                        uStatus = statuses[index]
263 281
                    else:
264 282
                        uStatus = "-"
265 283

  
266 284
                    if index < len(caches):
267
                        uCache   = caches[index]
285
                        uCache = caches[index]
268 286
                    else:
269 287
                        uCache = "-"
270 288

  
271 289
                    if uAddress != "-":
272
                        upstreams[uAddress]["requests"]                  += 1
290
                        upstreams[uAddress]["requests"] += 1
273 291
                    if uTime != "-":
274
                        upstreams[uAddress]["time"]                      += uTime
292
                        upstreams[uAddress]["time"] += uTime
275 293
                        upstreams[uAddress]["times"].append(uTime)
276 294
                    if uStatus != "-" and uStatus in upstreams[uAddress]["http"].keys():
277 295
                        upstreams[uAddress]["http"][uStatus]["requests"] += 1
278 296
                    if uCache != "-":
279
                        upstreams[uAddress]["cache"][uCache]             += 1
297
                        upstreams[uAddress]["cache"][uCache] += 1
280 298
                index += 1
281 299

  
282 300
    try:
......
301 319
    if "request" in graphs_enabled:
302 320
        for upstream in upstreams.keys():
303 321
            print()
304
            print("multigraph nginx_upstream_multi_%s.%s_requests" % (sanitize(siteName), sanitize(upstream)))
305

  
322
            print("multigraph nginx_upstream_multi_%s.%s_requests"
323
                  % (sanitize(siteName), sanitize(upstream)))
306 324
            value = 0
307 325
            if timeElapsed > 0:
308 326
                value = upstreams[upstream]["requests"] / timeElapsed
309

  
310 327
            print("us%s_requests.value %s" % (sanitize(upstream), value))
311 328
            print()
312 329

  
......
318 335
                uTime = upstreams[upstream]["time"] / upstreams[upstream]["requests"]
319 336
                upstreams[upstream]["times"].sort()
320 337
            print()
321
            print("multigraph nginx_upstream_multi_%s.%s_times" % (sanitize(siteName), sanitize(upstream)))
338
            print("multigraph nginx_upstream_multi_%s.%s_times"
339
                  % (sanitize(siteName), sanitize(upstream)))
322 340
            print("us%s_times.value %s" % (sanitize(upstream), uTime))
323 341
            for percentile in percentiles:
324 342
                percentileValue = 0
325 343
                if upstreams[upstream]["requests"] > 0:
326 344
                    uTime = upstreams[upstream]["time"] / upstreams[upstream]["requests"]
327 345
                    percentileKey = int(percentile) * len(upstreams[upstream]["times"]) / 100
328
                    if len(upstreams[upstream]["times"])%2 > 0:
346
                    if len(upstreams[upstream]["times"]) % 2 > 0:
329 347
                        low = int(math.floor(percentileKey))
330 348
                        high = int(math.ceil(percentileKey))
331
                        percentileValue = (upstreams[upstream]["times"][low] + upstreams[upstream]["times"][high]) / 2
349
                        percentileValue = (upstreams[upstream]["times"][low]
350
                                           + upstreams[upstream]["times"][high]) / 2
332 351
                    else:
333 352
                        percentileValue = upstreams[upstream]["times"][int(percentileKey)]
334
                print("us%s_times_percentile_%s.value %s" % (sanitize(upstream), percentile, percentileValue))
353
                print("us%s_times_percentile_%s.value %s"
354
                      % (sanitize(upstream), percentile, percentileValue))
335 355
            print()
336 356

  
337 357
    # HTTP Status codes graph data
338 358
    if "http" in graphs_enabled:
339 359
        for upstream in upstreams.keys():
340 360
            print()
341
            print("multigraph nginx_upstream_multi_%s.%s_statuses" % (sanitize(siteName), sanitize(upstream)))
361
            print("multigraph nginx_upstream_multi_%s.%s_statuses"
362
                  % (sanitize(siteName), sanitize(upstream)))
342 363
            for status in sorted(httpStatusList.keys()):
343 364
                value = 0
344 365
                if timeElapsed > 0:
......
351 372
    if "cache" in graphs_enabled:
352 373
        for upstream in upstreams.keys():
353 374
            print()
354
            print("multigraph nginx_upstream_multi_%s.%s_cache" % (sanitize(siteName), sanitize(upstream)))
375
            print("multigraph nginx_upstream_multi_%s.%s_cache"
376
                  % (sanitize(siteName), sanitize(upstream)))
355 377
            for status in cacheStatusList:
356 378
                value = 0
357 379
                if timeElapsed > 0:

Formats disponibles : Unified diff