This is the second day of my participation in the August More text Challenge. For details, see:August is more challenging

Recently, we found that the service was very unstable and many errors were reported. We wanted to analyze and review the causes of the errors according to the logs we kept on Elasticsearch, so we did the following operations according to the logs we saved on Elasticsearch

Email sending method:

from email.mime.text import MIMEText from smtplib import SMTP import requests def send_mail(to, subject, msg, smtpserver, username, password, _subtype='plain'): msg=MIMEText(msg, _subtype=_subtype, _charset='utf-8') msg['Subject'], msg['From'], msg['To']=subject, username, ";" .join(to) try: smtp=SMTP() smtp.connect(smtpserver) smtp.login(username, password) smtp.sendmail(username, to, msg.as_string()) smtp.close() except Exception as e: return eCopy the code

According to the es operation, logs are filtered and mail is sent

Def monitor (self, user_list = [']) : try: es = Elasticsearch ([{' host ':' 127.0.0.1 ', 'port' : '9200'}], http_auth = (' admin ', '123456'), a timeout = 3600) a_dict = {" 1 ":" test APP ", "2" : "test PC", "3" : "test APP1", "4" : "Test PC2"} p_dict = {" sakjldk ":" test ", "hasdja" : "test 2", "LLJL" : "test 3} l_dict = {" dhjsjkda" : "Beijing room", "opwq" : "Aws," "dkjashdj" : "ali cloud", "kdhka" : "hangzhou room", "llys" : "guangzhou room"} t_dict = {" ooiui ":" [online], "" KKK" : "online 】 【 aws," "KJKJHG" : "[advance online]", "iuuiui" : "[online]," "HHHMGG" : "" test", "} index = 'logs' p_url = {" jjii ": Body = {"query": {"bool": {"must": {"match": {"level": "ERROR"}}, {"range": {"@timestamp": {"gt": {"range": {"@timestamp": {"gt": "Now - 5 m"}}}], "must_not" : [error filter don't want to see the request {" match_phrase ": {" message" : QueryErrorData = es.search(index=index, body=body, scroll='1m', ) except Exception as e: Send_mail (user_list, 'log alarm {} service request error '.format(" es "), "ES link failure {}".format(e), 'smtp.exmail.qq.com', '[email protected]', 'jjdasda') content = "" title = "" repeat = [] if queryErrorData['hits']['hits']: for row in queryErrorData['hits']['hits']: if row['_source']['logger_name'] not in repeat: repeat.append(row['_source']['logger_name']) title += "|" + log_tag_name_dict[row['_source']['logger_name']] + row['_source']['logger_name'] b = re.findall("'me': '(. *?) ", the row [" _source "] [' message']) # according to the condition to filter a = re. The.findall (" 'app' : '(.*?)'", row['_source']['message']) p = re.findall("'p': '(.*?)'", row['_source']['message']) body = re.findall("Body((.*?))]-", row['_source']['message']) if a: A_content = ":" + a + "" + a_dict [0] [a [0]] +" \ n "else: a_content =" "if p: P_content = "client:" + p [0] + "" + p_dict [p [0]] +" \ n "else. P_content = "" user_content =" {} \n".format(info) body = "request information body :" + body[0] + "\n\n" if body else '' if b and a: url = p_url[row['_source']['logger_name']] + s_url params = {"aList": [a[0]]} headers = {"Content-Type": "application/json"} s_list = requests.post(url=url, json=params, headers=headers).json() s_dict_list = {str(row['id']): A row for the row in s_list [' data ']} if s_dict_list. Get (STR [0] (b)) : m_id = "ID:" + b [0] + "| name: " + s_dict_list.get(str(b[0]))[ 'name'] + "\n" else: m_id = '' else: M_id = '# processing log time General + 8 hours eta_temp = row [" _source "] [' timestamp] fd = datetime. Datetime. Strptime (eta_temp, "%Y-%m-%d %H:%M:%S") eta = (fd + datetime.timedelta(hours=8)).strftime("%Y-%m-%d %H:%M:%S") eta_date = (fd + Datetime.timedelta (hours=8).strftime("%Y-%m-%d") timestamp = "Request time: ">" + + "\ n" the content + = "\ n" * 3 + "-" * 200 + "\ n" + "service name:  " + row['_source']['logger_name'] + " {}".format( log_name_dict.get(row['_source']['logger_name'], '')) + "\n" + timestamp + menu_id + user_content + application_content + portal_content + body + "\n\n".join( row['_source']['message'].split(')-')) if content: Send_mail (user_list, 'log alarm {} service request error '. Format (title), content, 'smtp.exmail.qq.com', '[email protected]', 'hjhhj') return queryErrorDataCopy the code