Python日志处理:串行与并行执行优化及Logstash JSON格式配置
本文介绍Python中串行与并行执行的差异,重点讲解如何通过Python生成JSON格式日志以优化Logstash性能。通过对比直接发送与文件Tail两种方案,以及python-logstash模块的AMQP支持,帮助开发者解决日志字段解析问题,提升ELK栈处理效率。
代码示例如下:
Python串行与并行执行示例
#coding=utf-8
import threading
import time
import cx_Oracle
from pprint import pprint
import csv
print time.asctime()
table_name = "dbtest.csv"
f = open(table_name + ".csv", "w")
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
def query01():
tname = threading.current_thread()
aa=10
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
def query02():
tname = threading.current_thread()
aa=20
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
def query03():
tname = threading.current_thread()
aa=30
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
def query04():
tname = threading.current_thread()
aa=40
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
def query05():
tname = threading.current_thread()
aa=50
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
def query06():
tname = threading.current_thread()
aa=60
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
def query07():
tname = threading.current_thread()
aa=70
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
return row
cursor.close()
def query08():
tname = threading.current_thread()
aa=80
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
def query09():
tname = threading.current_thread()
aa=90
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
def query10():
tname = threading.current_thread()
aa=100
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
threads = []
t1 = threading.Thread(target=query01, name='query01')
threads.append(t1)
t2 = threading.Thread(target=query02, name='query02')
threads.append(t2)
t2 = threading.Thread(target=query03, name='query03')
threads.append(t2)
t2 = threading.Thread(target=query04, name='query04')
threads.append(t2)
t2 = threading.Thread(target=query05, name='query05')
threads.append(t2)
t2 = threading.Thread(target=query06, name='query06')
threads.append(t2)
t2 = threading.Thread(target=query07, name='query07')
threads.append(t2)
t2 = threading.Thread(target=query08, name='query08')
threads.append(t2)
t2 = threading.Thread(target=query09, name='query09')
threads.append(t2)
t2 = threading.Thread(target=query10, name='query10')
threads.append(t2)
if __name__ == '__main__':
for t in threads:
# t.setDaemon(True)
t.start()
# t.run()
# t.start()
# print '3333333'
print threading.current_thread()
# print t.is_alive()
# print '3333333'
t.join()
print "all over "
print time.asctime()
C:\Python27\python.exe C:/Users/tlcb/PycharmProjects/untitled/a2.py
Wed Mar 28 11:08:19 2018
<_MainThread(MainThread, started 18744)>
[(10, '10boobook10', '10aaaaaaaaaaaa10', '10bbbbbbbbbbbbbbbbb10'), (10, '10sssssssss10', 'tlcb', 'tlcb'), (10, '10boobook10', '10aaaaaaaaaaaa10', '10bbbbbbbbbbbbbbbbb10')]
<_MainThread(MainThread, started 18744)>
[(20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20'), (20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20'), (20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20')]
<_MainThread(MainThread, started 18744)>
[(30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30'), (30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30'), (30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30')]
<_MainThread(MainThread, started 18744)>
[(40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40'), (40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40'), (40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40')]
<_MainThread(MainThread, started 18744)>
[(50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50'), (50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50'), (50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50')]
<_MainThread(MainThread, started 18744)>
[(60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60'), (60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60'), (60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60')]
<_MainThread(MainThread, started 18744)>
<_MainThread(MainThread, started 18744)>
[(80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80'), (80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80'), (80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80')]
<_MainThread(MainThread, started 18744)>
[(90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90'), (90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90'), (90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90')]
<_MainThread(MainThread, started 18744)>
[(100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100'), (100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100'), (100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100')]
all over
Wed Mar 28 11:08:34 2018
Process finished with exit code 0
这个时候是串行 花费了15秒
多线程跑:
#coding=utf-8
import threading
import time
import cx_Oracle
from pprint import pprint
import csv
print time.asctime()
table_name = "dbtest.csv"
f = open(table_name + ".csv", "w")
def query01():
tname = threading.current_thread()
aa=10
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
def query02():
tname = threading.current_thread()
aa=20
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
def query03():
tname = threading.current_thread()
aa=30
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
def query04():
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
tname = threading.current_thread()
aa=40
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
def query05():
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
tname = threading.current_thread()
aa=50
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
def query06():
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
tname = threading.current_thread()
aa=60
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
def query07():
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
tname = threading.current_thread()
aa=70
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
return row
cursor.close()
conn.close()
def query08():
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
tname = threading.current_thread()
aa=80
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
def query09():
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
tname = threading.current_thread()
aa=90
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
def query10():
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
tname = threading.current_thread()
aa=100
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
threads = []
t1 = threading.Thread(target=query01, name='query01')
threads.append(t1)
t2 = threading.Thread(target=query02, name='query02')
threads.append(t2)
t2 = threading.Thread(target=query03, name='query03')
threads.append(t2)
t2 = threading.Thread(target=query04, name='query04')
threads.append(t2)
t2 = threading.Thread(target=query05, name='query05')
threads.append(t2)
t2 = threading.Thread(target=query06, name='query06')
threads.append(t2)
t2 = threading.Thread(target=query07, name='query07')
threads.append(t2)
t2 = threading.Thread(target=query08, name='query08')
threads.append(t2)
t2 = threading.Thread(target=query09, name='query09')
threads.append(t2)
t2 = threading.Thread(target=query10, name='query10')
threads.append(t2)
if __name__ == '__main__':
for t in threads:
# t.setDaemon(True)
t.start()
# t.run()
# t.start()
# print '3333333'
print threading.current_thread()
# print t.is_alive()
# print '3333333'
t.join()
print "all over "
print time.asctime()
C:\Python27\python.exe C:/Users/tlcb/PycharmProjects/untitled/a2.py
Wed Mar 28 11:12:47 2018
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
[(40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40'), (40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40'), (40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40')]
[(60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60'), (60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60'), (60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60')]
[(80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80'), (80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80'), (80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80')]
[(50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50'), (50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50'), (50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50')]
[(10, '10boobook10', '10aaaaaaaaaaaa10', '10bbbbbbbbbbbbbbbbb10'), (10, '10sssssssss10', 'tlcb', 'tlcb'), (10, '10boobook10', '10aaaaaaaaaaaa10', '10bbbbbbbbbbbbbbbbb10')]
[(20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20'), (20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20'), (20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20')]
[(30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30'), (30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30'), (30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30')]
[(100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100'), (100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100'), (100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100')]
[(90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90'), (90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90'), (90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90')]
all over
Wed Mar 28 11:12:55 2018
Process finished with exit code 0
此时花了 8秒
补充知识:Python日志定制Logstash JSON格式
近期完成了日志收集系统的调试工作,本文旨在分享一种优化Logstash处理Python日志的方案。
众所周知,Logstash使用Grok插件解析日志时会消耗大量CPU资源,因为正则匹配过程较为耗时。
基于Logstash的性能调优建议,我们可以预先将日志生成为JSON格式。针对Python程序,具体实现方式如下:
主要有两种实现路径:第一种是在生成JSON后,直接发送至Logstash监听端口;第二种是将JSON写入文件,利用Logstash的Tail插件读取文件,直接解析单行JSON日志数据。
Python内置的logging模块可以方便地调整为输出JSON格式。此外,已有社区开发的Python Logstash模块满足了这一需求。
Logstash JSON格式日志配置
import logging
import logstash
import sys
host = 'localhost'
test_logger = logging.getLogger('python-logstash-logger')
test_logger.setLevel(logging.INFO)
test_logger.addHandler(logstash.LogstashHandler(host, 5959, version=1))
# test_logger.addHandler(logstash.TCPLogstashHandler(host, 5959, version=1))
test_logger.error('python-logstash: test logstash error message.')
test_logger.info('python-logstash: test logstash info message.')
test_logger.warning('python-logstash: test logstash warning message.')
# add extra field to logstash message
extra = {
'test_string': 'python version: ' + repr(sys.version_info),
'test_boolean': True,
'test_dict': {'a': 1, 'b': 'c'},
'test_float': 1.23,
'test_integer': 123,
'test_list': [1, 2, '3'],
}
test_logger.info('python-logstash: test extra fields', extra=extra)
python-logstash库支持AMQP协议方案。
AMQP协议支持方案
import logging
import logstash
# AMQP parameters
host = 'localhost'
username = 'guest'
password= 'guest'
exchange = 'logstash.py'
# get a logger and set logging level
test_logger = logging.getLogger('python-logstash-logger')
test_logger.setLevel(logging.INFO)
# add the handler
test_logger.addHandler(logstash.AMQPLogstashHandler(version=1,
host=host,
durable=True,
username=username,
password=password,
exchange=exchange))
# log
test_logger.error('python-logstash: test logstash error message.')
test_logger.info('python-logstash: test logstash info message.')
test_logger.warning('python-logstash: test logstash warning message.')
try:
1/0
except:
test_logger.exception('python-logstash: test logstash exception with stack trace')
无论采用何种方式,最终生成的日志格式符合规范即可。
JSON日志输出格式示例
{
"@source"=>"unknown",
"@type"=>"nginx",
"@tags"=>[],
"@fields"=>{
"remote_addr"=>"192.168.0.1",
"remote_user"=>"-",
"body_bytes_sent"=>"13988",
"request_time"=>"0.122",
"status"=>"200",
"request"=>"GET /some/url HTTP/1.1",
"request_method"=>"GET",
"http_referrer"=>"http://www.example.org/some/url",
"http_user_agent"=>"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.79 Safari/537.1"
},
"@timestamp"=>"2012-08-23T10:49:14+02:00"
}
此处简要说明,该模块的实际使用体验并不理想。我在Python中将日志格式化为JSON字符串,原本期望在Elasticsearch中像Grok解析那样将日志拆解为独立字段,但实际情况是整条日志仍被包裹在message字段中。相信读者已理解我的意思,这导致在Kibana中进行搜索变得十分困难。
在Kibana搜索时,我常使用类似 source:xxx AND level:INFO 的查询语句。结果正如前述,整条日志内容都堆积在@message字段内,无法直接利用结构化字段进行高效检索。
以上关于Python串行与并行执行实例及日志优化的内容分享完毕,希望能为读者提供参考,欢迎继续支持本站。
本站声明
本文内容仅供参考,不代表本站观点。转载或引用请注明出处。如发现内容问题,请通过联系我们页面反馈。













