视频1 视频21 视频41 视频61 视频文章1 视频文章21 视频文章41 视频文章61 推荐1 推荐3 推荐5 推荐7 推荐9 推荐11 推荐13 推荐15 推荐17 推荐19 推荐21 推荐23 推荐25 推荐27 推荐29 推荐31 推荐33 推荐35 推荐37 推荐39 推荐41 推荐43 推荐45 推荐47 推荐49 关键词1 关键词101 关键词201 关键词301 关键词401 关键词501 关键词601 关键词701 关键词801 关键词901 关键词1001 关键词1101 关键词1201 关键词1301 关键词1401 关键词1501 关键词1601 关键词1701 关键词1801 关键词1901 视频扩展1 视频扩展6 视频扩展11 视频扩展16 文章1 文章201 文章401 文章601 文章801 文章1001 资讯1 资讯501 资讯1001 资讯1501 标签1 标签501 标签1001 关键词1 关键词501 关键词1001 关键词1501 专题2001
Mysql到Mongodb的数据转换程序_MySQL
2020-11-09 19:43:23 责编:小采
文档


以前写的Mysql到Mongodb的数据转换程序,翻了出来,在数据量不大的情况情况完全够用。
# -*- coding: utf-8 -*-

import sys, os
import multiprocessing
import logging
import random
import time, datetime
import MySQLdb
from MySQLdb import cursors
from pymongo import MongoClient

class Config:
 tables = ['hs_card', 'hs_hero', 'hs_set', 'hs_skill', 'hs_level', 'hs_pack', 'hs_salesevent']
 #tables = ['hs_card', 'hs_hero', 'hs_set', 'hs_skill', 'hs_level']
 index = {
 'hs_card': ['name'],
 }

class Mysql2Mongo(object):
 mysql_host = '50.23.4.2'
 mysql_port = 3306
 mysql_user = "root"
 mysql_pass = "stm123"
 mysql_db = "ccg_alpha"

 mongo_host = '50.23.4.2'
 mongo_port = 27017

 conn = None
 cursor = None
 mongo = None
 mongodb = None

 def __init__(self, logger):
	self.logger = logger

	self.conn = self.getMysqlConn()
	self.cursor = self.conn.cursor()

	self.mongo = MongoClient(host=self.mongo_host, port=self.mongo_port)
	self.mongodb = self.mongo['ccg_alpha']



 def getMysqlConn(self):
 return MySQLdb.connect(host=self.mysql_host, port=self.mysql_port, user=self.mysql_user, \
 passwd=self.mysql_pass, db=self.mysql_db, cursorclass=MySQLdb.cursors.SSCursor)


 def setMongoCollectionDocument(self, table, data):
	if(isinstance(data, dict) == False):
	return False
	else:
	self.mongodb[table].insert(data)


 def getMysqlTableDesc(self, table):
	sql = """desc %s""" % (table)
	n = self.cursor.execute(sql)
	data = self.cursor.fetchall()
	keys = []
	types = []
	for row in data:
	key = str(row[0])
	if(row[1].find('int') >= 0):
	type = 1
	
	elif (row[1].find('char') >= 0):
	type = 2
	elif (row[1].find('text') >= 0):
	type = 2
	elif(row[1].find('decimal') >= 0):
	type = 3
	else:
	type = 2
	keys.append(key)
	types.append(type)
	return keys, types

 def mysql2Mongo(self, table):
 self.mongodb[table].drop()
 keys, types = self.getMysqlTableDesc(table)

 sql = """select * from %s order by id asc""" % (table)
 n = self.cursor.execute(sql)
 data = self.cursor.fetchall()
 #print table, keys, types
 for row in data:
	ret = {}
	for k, key in enumerate(keys):
	if key == 'id':
	key = '_id'
	#ret[key] = int(row[k])
	if(types[k] == 1):
	if row[k]==None:
	ret[key]= 0
	continue
	#print k, key, row
	ret[key] = int(row[k])
	elif(types[k] == 2):
	if row[k]==None:
	ret[key]= ''
	continue
	ret[key] = str(row[k])
	elif(types[k] == 3):
	if row[k]==None:
	ret[key]= ''
	continue
	ret[key] = float(row[k])
	else:
	if row[k]==None:
	ret[key]= ''
	continue
	ret[key] = str(row[k])
	#if(table== 'hs_card') or (table== 'hs_hero'):
	#ret['rand'] = random.random()
	print ret
	self.setMongoCollectionDocument(table, ret)



 def __del__(self):
	self.mongo.close()
	self.cursor.close()
	self.conn.close()

if __name__ == '__main__':
 multiprocessing.log_to_stderr()
 logger = multiprocessing.get_logger()
 logger.setLevel(logging.INFO)

 # args = sys.argv
 t1 = time.time()
 cls = Mysql2Mongo(logger)
 for tb in Config.tables:
	cls.mysql2Mongo(tb)
	
 #index 
 for t, f in Config.index.items():
 pass
 
 print time.time() - t1
 logger.info("done")

下载本文
显示全文
专题