Révision b17436fe
Rework to tail_open log files
| plugins/znc/znc_logs.py | ||
|---|---|---|
| 11 | 11 |
[znc_logs] |
| 12 | 12 |
user znc # or any other user/group that can read the znclog-folder |
| 13 | 13 |
group znc |
| 14 |
env.logdir /var/lib/znc/moddata/log/ # path to the log-folder with a "/" at the end |
|
| 14 |
env.logdir /var/lib/znc/moddata/log/ # path to the GLOBAL log-folder with a "/" at the end |
|
| 15 |
env.expire 0 # Keep channel names forever - OR - |
|
| 16 |
env.expire 1 # Forget channel names from last run |
|
| 15 | 17 |
|
| 16 | 18 |
=head1 COPYRIGHT |
| 17 | 19 |
GPL VERSION 3 |
| ... | ... | |
| 19 | 21 |
=head1 AUTHOR |
| 20 | 22 |
Thor77 <thor77[at]thor77.org> |
| 21 | 23 |
''' |
| 22 |
from sys import argv |
|
| 23 |
from time import strftime |
|
| 24 |
from os import environ, listdir |
|
| 24 |
import json |
|
| 25 |
import os, sys, time |
|
| 26 |
import re |
|
| 27 |
import stat |
|
| 28 |
import traceback |
|
| 25 | 29 |
|
| 26 |
logdir = environ.get('logdir')
|
|
| 30 |
logdir = os.environ.get('logdir')
|
|
| 31 |
expire = os.environ.get('expire', 0)
|
|
| 27 | 32 |
|
| 28 | 33 |
if not logdir: |
| 29 | 34 |
raise Exception('You have to set the logdir with env.logdir <path to log> in the plugin-conf!')
|
| 30 | 35 |
|
| 31 |
date = strftime('%Y%m%d')
|
|
| 32 |
last_values_file = environ['MUNIN_PLUGSTATE'] + '/last_values'
|
|
| 33 |
|
|
| 36 |
date = time.strftime('%Y%m%d')
|
|
| 37 |
longdate = time.strftime('%Y-%m-%d')
|
|
| 38 |
last_values_file = os.environ['MUNIN_PLUGSTATE'] + '/znc_logs_last' |
|
| 34 | 39 |
|
| 35 | 40 |
def get_last(): |
| 36 | 41 |
try: |
| 37 | 42 |
d = {}
|
| 38 | 43 |
with open(last_values_file, 'r') as f: |
| 39 |
for line in f: |
|
| 40 |
line = line[:-1] |
|
| 41 |
key, value = line.split(':')
|
|
| 42 |
d[key] = float(value) |
|
| 44 |
d = json.load(f) |
|
| 43 | 45 |
return d |
| 44 | 46 |
except FileNotFoundError: |
| 45 | 47 |
return {}
|
| 46 | 48 |
|
| 49 |
def tail_open(filename, position=0): |
|
| 50 |
# Based on tail_open from perls' Munin::Plugin |
|
| 51 |
filereset = 0 |
|
| 52 |
size = os.stat(filename)[stat.ST_SIZE] |
|
| 53 |
if size is None: |
|
| 54 |
return (undef, undef) |
|
| 55 |
f = open(filename, 'r', encoding='utf-8', errors='replace') |
|
| 56 |
if position > size: |
|
| 57 |
filereset = 1 |
|
| 58 |
else: |
|
| 59 |
f.seek(position, 0) |
|
| 60 |
newpos = f.tell() |
|
| 61 |
if newpos != position: |
|
| 62 |
raise Exception |
|
| 63 |
return (f, filereset) |
|
| 47 | 64 |
|
| 48 |
def data(): |
|
| 49 |
last = get_last() |
|
| 50 |
current = {}
|
|
| 51 |
for filename in listdir(logdir): |
|
| 52 |
filename_ = filename.replace('.log', '')
|
|
| 53 |
network, channel, file_date = filename_.split('_')
|
|
| 54 |
network_channel = '{}_{}'.format(network, channel)
|
|
| 55 |
# check if log is from today and it is a channel |
|
| 56 |
if file_date == date and channel.startswith('#'):
|
|
| 57 |
# current lines in the file |
|
| 58 |
current_value = sum(1 for i in open(logdir + filename, 'r', encoding='utf-8', errors='replace')) |
|
| 59 |
|
|
| 60 |
if network_channel not in last: |
|
| 61 |
value = 0 |
|
| 62 |
else: |
|
| 63 |
last_value = last[network_channel] |
|
| 64 |
# what munin gets |
|
| 65 |
value = (current_value - last_value) / 5 # subtrate last from current and divide through 5 to get new lines / minute |
|
| 66 |
if value < 0: |
|
| 67 |
value = 0 |
|
| 68 |
# save it to the states-file |
|
| 69 |
current[network_channel] = current_value |
|
| 65 |
def tail_close(fh): |
|
| 66 |
position = fh.tell() |
|
| 67 |
fh.close() |
|
| 68 |
return position |
|
| 70 | 69 |
|
| 71 |
# print things to munin |
|
| 72 |
network_channel = network_channel.replace('.', '').replace('#', '')
|
|
| 73 |
print('{network_channel}.label {channel}@{network}'.format(network_channel=network_channel, channel=channel, network=network))
|
|
| 74 |
print('{network_channel}.value {value}'.format(network_channel=network_channel, value=value))
|
|
| 75 |
with open(last_values_file, 'w') as f: |
|
| 76 |
for k in current: |
|
| 77 |
f.write('{}:{}\n'.format(k, current[k]))
|
|
| 78 | 70 |
|
| 71 |
last = get_last() |
|
| 72 |
if "users" in last: |
|
| 73 |
user_list = last["users"] |
|
| 74 |
else: |
|
| 75 |
user_list = {}
|
|
| 76 |
if "channels" in last: |
|
| 77 |
channel_list = last["channels"] |
|
| 78 |
else: |
|
| 79 |
channel_list = {}
|
|
| 80 |
if "log_pos" in last: |
|
| 81 |
log_pos = last["log_pos"] |
|
| 82 |
else: |
|
| 83 |
log_pos = {}
|
|
| 84 |
|
|
| 85 |
channel_stats = {}
|
|
| 86 |
user_stats = {}
|
|
| 87 |
|
|
| 88 |
def read_data(savestate=True): |
|
| 89 |
# Version 1.6 will change to directory-based filing, so walk recursively |
|
| 90 |
for (dirpath, dirnames, filenames) in os.walk(logdir): |
|
| 91 |
for filename in filenames: |
|
| 92 |
filename_ = filename.replace('.log', '')
|
|
| 93 |
|
|
| 94 |
user, network, channel, file_date = (None, None, None, None) |
|
| 79 | 95 |
|
| 80 |
if len(argv) > 1 and argv[1] == 'config': |
|
| 96 |
try: |
|
| 97 |
if len(dirpath) > len(logdir): |
|
| 98 |
# We're below the log path, so this is a 1.6-style log |
|
| 99 |
reldir = dirpath.replace(logdir + "/", '', 1) |
|
| 100 |
try: |
|
| 101 |
network, channel = reldir.split(os.sep) |
|
| 102 |
except ValueError as e: |
|
| 103 |
user, network, channel = reldir.split(os.sep) |
|
| 104 |
file_date = filename_ |
|
| 105 |
else: |
|
| 106 |
try: |
|
| 107 |
network, channel, file_date = filename_.split('_')
|
|
| 108 |
except ValueError as e: |
|
| 109 |
user, network, channel, file_date = filename_.split('_')
|
|
| 110 |
except ValueError as e: |
|
| 111 |
continue |
|
| 112 |
network_channel = '{}@{}'.format(channel, network)
|
|
| 113 |
if network.lower() not in channel_list: |
|
| 114 |
channel_list[network.lower()] = {}
|
|
| 115 |
if channel.startswith('#'):
|
|
| 116 |
channel_list[network.lower()][channel.lower()] = network_channel |
|
| 117 |
user_list[user.lower()] = user |
|
| 118 |
# check if log is from today |
|
| 119 |
if (file_date == date or file_date == longdate): |
|
| 120 |
# current lines in the file |
|
| 121 |
(fh, r) = tail_open(os.path.join(dirpath,filename), log_pos.get(os.path.join(dirpath, filename), 0)) |
|
| 122 |
current_value = 0 |
|
| 123 |
while True: |
|
| 124 |
where = fh.tell() |
|
| 125 |
line = fh.readline() |
|
| 126 |
if line.endswith('\n'):
|
|
| 127 |
current_value += 1 |
|
| 128 |
else: |
|
| 129 |
# Incomplete last line |
|
| 130 |
fh.seek(where, 0) |
|
| 131 |
log_pos[os.path.join(dirpath, filename)] = tail_close(fh) |
|
| 132 |
break |
|
| 133 |
|
|
| 134 |
if network_channel.lower() in channel_stats and channel.startswith('#'):
|
|
| 135 |
channel_stats[network_channel.lower()] += current_value |
|
| 136 |
else: |
|
| 137 |
channel_stats[network_channel.lower()] = current_value |
|
| 138 |
|
|
| 139 |
if user is not None and user.lower() in user_stats: |
|
| 140 |
user_stats[user.lower()] += current_value |
|
| 141 |
else: |
|
| 142 |
user_stats[user.lower()] = current_value |
|
| 143 |
if savestate: |
|
| 144 |
savedata = {}
|
|
| 145 |
if int(expire) == 0: |
|
| 146 |
savedata["users"] = user_list |
|
| 147 |
savedata["channels"] = channel_list |
|
| 148 |
savedata["log_pos"] = log_pos |
|
| 149 |
with open(last_values_file, 'w') as f: |
|
| 150 |
json.dump(savedata,f) |
|
| 151 |
|
|
| 152 |
|
|
| 153 |
def emit_config(): |
|
| 81 | 154 |
print('graph_title Lines in the ZNC-log')
|
| 82 | 155 |
print('graph_category znc')
|
| 83 |
print('graph_vlabel lines/minute')
|
|
| 156 |
print('graph_vlabel lines / ${graph_period}')
|
|
| 84 | 157 |
print('graph_scale no')
|
| 85 |
data() |
|
| 158 |
print('graph_args --base 1000 --lower-limit 0')
|
|
| 159 |
print('graph_period minute')
|
|
| 160 |
graph_order = [] |
|
| 161 |
|
|
| 162 |
if 'MUNIN_CAP_DIRTYCONFIG' in os.environ and os.environ['MUNIN_CAP_DIRTYCONFIG'] == 1: |
|
| 163 |
read_data(1) |
|
| 164 |
else: |
|
| 165 |
read_data(0) |
|
| 166 |
|
|
| 167 |
for network in channel_list.keys(): |
|
| 168 |
for channel in channel_list[network].keys(): |
|
| 169 |
|
|
| 170 |
# print things to munin |
|
| 171 |
network_channel = "{}_{}".format(network,channel).replace('.', '').replace('#', '').replace('@','_')
|
|
| 172 |
print('{network_channel}.label {label}'.format(network_channel=network_channel, label=channel_list[network][channel]))
|
|
| 173 |
print('{network_channel}.type ABSOLUTE'.format(network_channel=network_channel))
|
|
| 174 |
print('{network_channel}.min 0'.format(network_channel=network_channel))
|
|
| 175 |
print('{network_channel}.draw AREASTACK'.format(network_channel=network_channel))
|
|
| 176 |
|
|
| 177 |
graph_order.append(network_channel) |
|
| 178 |
for user in user_list.keys(): |
|
| 179 |
fuser = re.sub(r'^[^A-Za-z_]', '_', user) |
|
| 180 |
fuser = re.sub(r'[^A-Za-z0-9_]', '_', fuser) |
|
| 181 |
print('{fuser}.label User {user}'.format(fuser=fuser, user=user))
|
|
| 182 |
print('{fuser}.type ABSOLUTE'.format(fuser=fuser))
|
|
| 183 |
print('{fuser}.min 0'.format(fuser=fuser))
|
|
| 184 |
print('{fuser}.draw LINE1'.format(fuser=fuser))
|
|
| 185 |
|
|
| 186 |
print('graph_order {}'.format(" ".join(sorted(graph_order, key=str.lower))))
|
|
| 187 |
|
|
| 188 |
def emit_values(): |
|
| 189 |
read_data(1) |
|
| 190 |
for network in channel_list.keys(): |
|
| 191 |
for channel in channel_list[network].keys(): |
|
| 192 |
|
|
| 193 |
# print things to munin |
|
| 194 |
key = channel_list[network][channel] |
|
| 195 |
network_channel = "{}_{}".format(network,channel).replace('.', '').replace('#', '').replace('@','_')
|
|
| 196 |
if key.lower() in channel_stats: |
|
| 197 |
print('{network_channel}.value {value}'.format(network_channel=network_channel, value=channel_stats[key.lower()]))
|
|
| 198 |
else: |
|
| 199 |
print('{network_channel}.value U'.format(network_channel=network_channel))
|
|
| 200 |
for user in user_list.keys(): |
|
| 201 |
fuser = re.sub(r'^[^A-Za-z_]', '_', user) |
|
| 202 |
fuser = re.sub(r'[^A-Za-z0-9_]', '_', fuser) |
|
| 203 |
if user.lower() in user_stats: |
|
| 204 |
print('{fuser}.value {value}'.format(fuser=fuser, value=user_stats[user.lower()]))
|
|
| 205 |
else: |
|
| 206 |
print('{fuser}.value U'.format(fuser=fuser))
|
|
| 207 |
|
|
| 208 |
|
|
| 209 |
if len(sys.argv) > 1 and sys.argv[1] == 'config': |
|
| 210 |
emit_config() |
|
| 211 |
sys.exit(0) |
|
| 212 |
|
|
| 213 |
emit_values() |
|
Formats disponibles : Unified diff