Projet

Général

Profil

Révision b17436fe

IDb17436fe80cbc6f79fa8e108f640253453949e82
Parent 7f94d4d6
Enfant 6c4efbff

Ajouté par Paul Saunders il y a plus de 10 ans

Rework to tail_open log files

Voir les différences:

plugins/znc/znc_logs.py
11 11
[znc_logs]
12 12
user znc # or any other user/group that can read the znclog-folder
13 13
group znc
14
env.logdir /var/lib/znc/moddata/log/ # path to the log-folder with a "/" at the end
14
env.logdir /var/lib/znc/moddata/log/ # path to the GLOBAL log-folder with a "/" at the end
15
env.expire 0    # Keep channel names forever  - OR -
16
env.expire 1    # Forget channel names from last run
15 17

  
16 18
=head1 COPYRIGHT
17 19
GPL VERSION 3
......
19 21
=head1 AUTHOR
20 22
Thor77 <thor77[at]thor77.org>
21 23
'''
22
from sys import argv
23
from time import strftime
24
from os import environ, listdir
24
import json
25
import os, sys, time
26
import re
27
import stat
28
import traceback
25 29

  
26
logdir = environ.get('logdir')
30
logdir = os.environ.get('logdir')
31
expire = os.environ.get('expire', 0)
27 32

  
28 33
if not logdir:
29 34
    raise Exception('You have to set the logdir with env.logdir <path to log> in the plugin-conf!')
30 35

  
31
date = strftime('%Y%m%d')
32
last_values_file = environ['MUNIN_PLUGSTATE'] + '/last_values'
33

  
36
date = time.strftime('%Y%m%d')
37
longdate = time.strftime('%Y-%m-%d')
38
last_values_file = os.environ['MUNIN_PLUGSTATE'] + '/znc_logs_last'
34 39

  
35 40
def get_last():
36 41
    try:
37 42
        d = {}
38 43
        with open(last_values_file, 'r') as f:
39
            for line in f:
40
                line = line[:-1]
41
                key, value = line.split(':')
42
                d[key] = float(value)
44
            d = json.load(f)
43 45
        return d
44 46
    except FileNotFoundError:
45 47
        return {}
46 48

  
49
def tail_open(filename, position=0):
50
    # Based on tail_open from perls' Munin::Plugin
51
    filereset = 0
52
    size = os.stat(filename)[stat.ST_SIZE]
53
    if size is None:
54
        return (undef, undef)
55
    f = open(filename, 'r', encoding='utf-8', errors='replace')
56
    if position > size:
57
        filereset = 1
58
    else:
59
        f.seek(position, 0)
60
        newpos = f.tell()
61
        if newpos != position:
62
            raise Exception
63
    return (f, filereset)
47 64

  
48
def data():
49
    last = get_last()
50
    current = {}
51
    for filename in listdir(logdir):
52
        filename_ = filename.replace('.log', '')
53
        network, channel, file_date = filename_.split('_')
54
        network_channel = '{}_{}'.format(network, channel)
55
        # check if log is from today and it is a channel
56
        if file_date == date and channel.startswith('#'):
57
            # current lines in the file
58
            current_value = sum(1 for i in open(logdir + filename, 'r', encoding='utf-8', errors='replace'))
59

  
60
            if network_channel not in last:
61
                value = 0
62
            else:
63
                last_value = last[network_channel]
64
                # what munin gets
65
                value = (current_value - last_value) / 5  # subtrate last from current and divide through 5 to get new lines / minute
66
                if value < 0:
67
                    value = 0
68
            # save it to the states-file
69
            current[network_channel] = current_value
65
def tail_close(fh):
66
    position = fh.tell()
67
    fh.close()
68
    return position
70 69

  
71
            # print things to munin
72
            network_channel = network_channel.replace('.', '').replace('#', '')
73
            print('{network_channel}.label {channel}@{network}'.format(network_channel=network_channel, channel=channel, network=network))
74
            print('{network_channel}.value {value}'.format(network_channel=network_channel, value=value))
75
    with open(last_values_file, 'w') as f:
76
        for k in current:
77
            f.write('{}:{}\n'.format(k, current[k]))
78 70

  
71
last = get_last()
72
if "users" in last:
73
    user_list = last["users"]
74
else:
75
    user_list = {}
76
if "channels" in last:
77
    channel_list = last["channels"]
78
else:
79
    channel_list = {}
80
if "log_pos" in last:
81
    log_pos = last["log_pos"]
82
else:
83
    log_pos = {}
84

  
85
channel_stats = {}
86
user_stats = {}
87

  
88
def read_data(savestate=True):
89
    # Version 1.6 will change to directory-based filing, so walk recursively
90
    for (dirpath, dirnames, filenames) in os.walk(logdir):
91
        for filename in filenames:
92
            filename_ = filename.replace('.log', '')
93

  
94
            user, network, channel, file_date = (None, None, None, None)
79 95

  
80
if len(argv) > 1 and argv[1] == 'config':
96
            try:
97
                if len(dirpath) > len(logdir):
98
                    # We're below the log path, so this is a 1.6-style log
99
                    reldir = dirpath.replace(logdir + "/", '', 1)
100
                    try:
101
                        network, channel = reldir.split(os.sep)
102
                    except ValueError as e:
103
                        user, network, channel = reldir.split(os.sep)
104
                    file_date = filename_
105
                else:
106
                    try:
107
                        network, channel, file_date = filename_.split('_')
108
                    except ValueError as e:
109
                        user, network, channel, file_date = filename_.split('_')
110
            except ValueError as e:
111
                continue
112
            network_channel = '{}@{}'.format(channel, network)
113
            if network.lower() not in channel_list:
114
                channel_list[network.lower()] = {}
115
            if channel.startswith('#'):
116
                channel_list[network.lower()][channel.lower()] = network_channel
117
            user_list[user.lower()] = user
118
            # check if log is from today
119
            if (file_date == date or file_date == longdate):
120
                # current lines in the file
121
                (fh, r) = tail_open(os.path.join(dirpath,filename), log_pos.get(os.path.join(dirpath, filename), 0))
122
                current_value = 0
123
                while True:
124
                    where = fh.tell()
125
                    line = fh.readline()
126
                    if line.endswith('\n'):
127
                        current_value += 1
128
                    else:
129
                        # Incomplete last line
130
                        fh.seek(where, 0)
131
                        log_pos[os.path.join(dirpath, filename)] = tail_close(fh)
132
                        break
133

  
134
                if network_channel.lower() in channel_stats and channel.startswith('#'):
135
                    channel_stats[network_channel.lower()] += current_value
136
                else:
137
                    channel_stats[network_channel.lower()] = current_value
138

  
139
                if user is not None and user.lower() in user_stats:
140
                    user_stats[user.lower()] += current_value
141
                else:
142
                    user_stats[user.lower()] = current_value
143
    if savestate:
144
        savedata = {}
145
        if int(expire) == 0:
146
            savedata["users"] = user_list
147
            savedata["channels"] = channel_list
148
        savedata["log_pos"] = log_pos
149
        with open(last_values_file, 'w') as f:
150
            json.dump(savedata,f)
151

  
152

  
153
def emit_config():
81 154
    print('graph_title Lines in the ZNC-log')
82 155
    print('graph_category znc')
83
    print('graph_vlabel lines/minute')
156
    print('graph_vlabel lines / ${graph_period}')
84 157
    print('graph_scale no')
85
data()
158
    print('graph_args --base 1000 --lower-limit 0')
159
    print('graph_period minute')
160
    graph_order = []
161

  
162
    if 'MUNIN_CAP_DIRTYCONFIG' in os.environ and os.environ['MUNIN_CAP_DIRTYCONFIG'] == 1:
163
        read_data(1)
164
    else:
165
        read_data(0)
166

  
167
    for network in channel_list.keys():
168
        for channel in channel_list[network].keys():
169

  
170
            # print things to munin
171
            network_channel = "{}_{}".format(network,channel).replace('.', '').replace('#', '').replace('@','_')
172
            print('{network_channel}.label {label}'.format(network_channel=network_channel, label=channel_list[network][channel]))
173
            print('{network_channel}.type  ABSOLUTE'.format(network_channel=network_channel))
174
            print('{network_channel}.min   0'.format(network_channel=network_channel))
175
            print('{network_channel}.draw  AREASTACK'.format(network_channel=network_channel))
176

  
177
            graph_order.append(network_channel)
178
    for user in user_list.keys():
179
            fuser = re.sub(r'^[^A-Za-z_]', '_', user)
180
            fuser = re.sub(r'[^A-Za-z0-9_]', '_', fuser)
181
            print('{fuser}.label User {user}'.format(fuser=fuser, user=user))
182
            print('{fuser}.type  ABSOLUTE'.format(fuser=fuser))
183
            print('{fuser}.min   0'.format(fuser=fuser))
184
            print('{fuser}.draw  LINE1'.format(fuser=fuser))
185

  
186
    print('graph_order {}'.format(" ".join(sorted(graph_order, key=str.lower))))
187

  
188
def emit_values():
189
    read_data(1)
190
    for network in channel_list.keys():
191
        for channel in channel_list[network].keys():
192

  
193
            # print things to munin
194
            key = channel_list[network][channel]
195
            network_channel = "{}_{}".format(network,channel).replace('.', '').replace('#', '').replace('@','_')
196
            if key.lower() in channel_stats:
197
                print('{network_channel}.value {value}'.format(network_channel=network_channel, value=channel_stats[key.lower()]))
198
            else:
199
                print('{network_channel}.value U'.format(network_channel=network_channel))
200
    for user in user_list.keys():
201
        fuser = re.sub(r'^[^A-Za-z_]', '_', user)
202
        fuser = re.sub(r'[^A-Za-z0-9_]', '_', fuser)
203
        if user.lower() in user_stats:
204
            print('{fuser}.value {value}'.format(fuser=fuser, value=user_stats[user.lower()]))
205
        else:
206
            print('{fuser}.value U'.format(fuser=fuser))
207

  
208

  
209
if len(sys.argv) > 1 and sys.argv[1] == 'config':
210
    emit_config()
211
    sys.exit(0)
212

  
213
emit_values()

Formats disponibles : Unified diff