python-zulip-api/zulip/integrations/log2zulip/log2zulip
Tim Abbott 297b62efc6 log2zulip: Use simpler code for accessing production venv.
The only problem with this resulting code is that setup_path_on_import
only works if the Python versions are the same, so you need to run
this with Python 3 in that case.

We try to resolve that for use on Zulip servers with
zulip/zulip:47c5aae5b242fb6d2f5e860602e0fc0af68419bb; since that's the
main case where the code path runs, that should be good enough.
2017-10-06 16:37:56 -07:00

128 lines
4.3 KiB
Python
Executable file

#!/usr/bin/env python
from __future__ import print_function
import argparse
import errno
import os
import platform
import re
import sys
import shutil
import subprocess
import tempfile
import traceback
try:
# Use the Zulip virtualenv if available
sys.path.append("/home/zulip/deployments/current")
import scripts.lib.setup_path_on_import
except ImportError:
pass
import json
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../"))
import zulip
from typing import List
temp_dir = "/var/tmp/" if os.name == "posix" else tempfile.gettempdir()
def mkdir_p(path):
# type: (str) -> None
# Python doesn't have an analog to `mkdir -p` < Python 3.2.
try:
os.makedirs(path)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
def send_log_zulip(file_name, count, lines, extra=""):
# type: (str, int, List[str], str) -> None
content = "%s new errors%s:\n```\n%s\n```" % (count, extra, "\n".join(lines))
zulip_client.send_message({
"type": "stream",
"to": "logs",
"subject": "%s on %s" % (file_name, platform.node()),
"content": content,
})
def process_lines(raw_lines, file_name):
# type: (List[str], str) -> None
lines = []
for line in raw_lines:
# Add any filtering or modification code here
if re.match(".*upstream timed out.*while reading upstream.*", line):
continue
lines.append(line)
if len(lines) == 0:
return
elif len(lines) > 10:
send_log_zulip(file_name, len(lines), lines[0:3], extra=", examples include")
else:
send_log_zulip(file_name, len(lines), lines)
def process_logs():
# type: () -> None
for filename in log_files:
data_file_path = os.path.join(temp_dir, "log2zulip.state")
mkdir_p(os.path.dirname(data_file_path))
if not os.path.exists(data_file_path):
open(data_file_path, "w").write("{}")
last_data = json.loads(open(data_file_path).read())
new_data = {}
for log_file in log_files:
file_data = last_data.get(log_file, {})
if not os.path.exists(log_file):
# If the file doesn't exist, log an error and then move on to the next file
print("Log file does not exist or could not stat log file: %s" % (log_file,))
continue
length = int(subprocess.check_output(["wc", "-l", log_file]).split()[0])
if file_data.get("last") is None:
file_data["last"] = 1
if length + 1 < file_data["last"]:
# The log file was rotated, restart from empty. Note that
# because we don't actually store the log file content, if
# a log file ends up at the same line length as before
# immediately after rotation, this tool won't notice.
file_data["last"] = 1
output = subprocess.check_output(["tail", "-n+%s" % (file_data["last"],), log_file])
new_lines = output.decode('utf-8', errors='replace').split('\n')[:-1]
if len(new_lines) > 0:
process_lines(new_lines, filename)
file_data["last"] += len(new_lines)
new_data[log_file] = file_data
open(data_file_path, "w").write(json.dumps(new_data))
if __name__ == "__main__":
parser = zulip.add_default_arguments(argparse.ArgumentParser()) # type: argparse.ArgumentParser3
parser.add_argument("--control-path", default="/etc/log2zulip.conf")
args = parser.parse_args()
lock_path = os.path.join(temp_dir, "log2zulip.lock")
if os.path.exists(lock_path):
# This locking code is here to protect against log2zulip,
# running in a cron job, ending up with multiple copies
# running at the same time.
print("Log2zulip lock held; not doing anything")
sys.exit(0)
# TODO: Convert this locking code to use a standard context manager.
try:
open(lock_path, "w").write("1")
zulip_client = zulip.init_from_options(args)
try:
log_files = json.loads(open(args.control_path, "r").read())
except (json.JSONDecodeError, IOError):
print("Could not load control data from %s" % (args.control_path,))
traceback.print_exc()
sys.exit(1)
process_logs()
finally:
try:
os.remove(lock_path)
except OSError as IOError:
pass