JSON日志通过python处理并导入Hive代码,Python代码库,德仔网

代码语言
代码分类
【Python】 JSON日志通过python处理并导入Hive
作者: / 发布于2016/11/23/ 683
[spark@Master Py_logproc]$ pwd
/home/spark/opt/Log_Data/Py_logproc
[spark@Master Py_logproc]$ cat json2hive_python_recordasarray_basic.py
# -*- encoding:utf-8 -*-
#!/usr/bin/env python
import sys
sys.path.append('/home/spark/opt/hive-1.2.1/lib/py')
from hive_service import ThriftHive
from hive_service.ttypes import HiveServerException
from thrift import Thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
import json
import warnings
warnings.filterwarnings("ignore")
 
 
 
 
def hiveExe(sql):
   try:
      transport = TSocket.TSocket('127.0.0.1', 10000)
      transport = TTransport.TBufferedTransport(transport)
      protocol = TBinaryProtocol.TBinaryProtocol(transport)
      client = ThriftHive.Client(protocol)
      transport.open()
      client.execute(sql)
      transport.close()
   except Thrift.TException, tx:
      print '%s' % (tx.message)
 
 
 
 
if __name__=="__main__":
  import sys
  reload(sys)
  sys.setdefaultencoding( "utf-8" )
  
  if len(sys.argv)==1: 
     print "need argv" 
  else: 
     print sys.argv 
 
 
 
 
for json_array in open('/home/spark/opt/Log_Data/Py_logproc/log_tmpdir/yemaopythonlog'):
  yemao_array = json.loads(json_array)
  for yemao in yemao_array:
   print yemao['time']
   if not yemao.has_key('_reason'):
     
    id              =   yemao['id']
    time            =   yemao['time']
    url_from        =   yemao['url_from']
    url_current     =   yemao['url_current']
    url_to          =   yemao['url_to']
    options         =   yemao['options']
    ip              =   yemao['ip']
    uid             =   yemao['uid']
    new_visitor     =   yemao['new_visitor']
    province        =   yemao['province']
    city            =   yemao['city']
    site            =   yemao['site']
    device          =   yemao['device']
    browser         =   yemao['browser']
    phone           =   yemao['phone']
    token           =   yemao['token']
    dorm            =   yemao['dorm']
    order_phone     =   yemao['order_phone']
    order_dormitory =   yemao['order_dormitory']
    order_amount    =   yemao['order_amount']
    order_id        =   yemao['order_id']
    uname           =   yemao['uname']
    site_id         =   yemao['site_id']
    address         =   yemao['address']
    dorm_id         =   yemao['dorm_id']
    dormentry_id    =   yemao['dormentry_id']
    tag             =   yemao['tag']
    rid             =   yemao['rid']
    cart_quantity   =   yemao['cart_quantity']
    response        =   yemao['response']
    paytype         =   yemao['paytype']
     
    if yemao.has_key('data'):
        data = yemao['data']
    else:
        data = '0'
 
 
 
 
    data = '"'+str(data)+'"'
 
 
 
 
    if yemao.has_key('info'):
        info = yemao['info']
    else:
        info = '0'
 
 
 
 
    if yemao.has_key('status'):
        status = yemao['status']
    else:
        status = '0'
    log_date        =   int(sys.argv[1])
 
 
 
 
   if __name__ == '__main__':
    insert_sql="insert into yemao_logpy(id,time,url_from,url_to,url_current,ip,dorm_id,browser,log_date) values ('%s',  '%s',  '%s','%s','%s','%s','%s','%s', %d)" % (id,time,url_from,url_current,url_to,ip,dorm_id,browser,log_date)
    print insert_sql
    hiveExe(insert_sql)
 
 
 
 
print 'yemao_array_python2hive done'
[spark@Master Py_logproc]$
试试其它关键字
同语言下
可能有用的
贡献的其它代码Label