视频1 视频21 视频41 视频61 视频文章1 视频文章21 视频文章41 视频文章61 推荐1 推荐3 推荐5 推荐7 推荐9 推荐11 推荐13 推荐15 推荐17 推荐19 推荐21 推荐23 推荐25 推荐27 推荐29 推荐31 推荐33 推荐35 推荐37 推荐39 推荐41 推荐43 推荐45 推荐47 推荐49 关键词1 关键词101 关键词201 关键词301 关键词401 关键词501 关键词601 关键词701 关键词801 关键词901 关键词1001 关键词1101 关键词1201 关键词1301 关键词1401 关键词1501 关键词1601 关键词1701 关键词1801 关键词1901 视频扩展1 视频扩展6 视频扩展11 视频扩展16 文章1 文章201 文章401 文章601 文章801 文章1001 资讯1 资讯501 资讯1001 资讯1501 标签1 标签501 标签1001 关键词1 关键词501 关键词1001 关键词1501 专题2001
FastDFS加Redis实现自定义文件名存储海量文件
2020-11-27 14:28:29 责编:小采
文档


FastDFS非常适合存储大量的小文件,遗憾的是本身不支持自定义文件名,文件名是存储成功以后根据存储位置生成的一个file_id。很多应用场景不得不使用自定义文件名,在不修改其源码的情况下,可以在存储客户端fdfs_client增加一个用来存储自定义文件名和fastdfs的file_id之间的映射关系的数据库间接实现自定义文件名的存取和访问,在这里我们选用了reids。顺便说一下,淘宝也有一个类似于FastDFS的文件存储系统TFS,对于自定义文件名,它是用mysql来存储映射关系的,我认为在高并发访问下mysql本身就是瓶颈,因此在这个方案中采用了redis。

准备工作:

fastdfs环境安装...略...(官方:https://code.google.com/p/fastdfs/)

redis环境安装...略...(官方:http://redis.io/)

用python实现,因此需要安装fastdfs的python客户端(下载:https://fastdfs.googlecode.com/files/fdfs_client-py-1.2.6.tar.gz)

python的redis客户端,到https://pypi.python.org/pypi/redis下载

# -*- coding: utf-8 -*-
import setting
from fdfs_client.client import *
from fdfs_client.exceptions import *
 
from fdfs_client.connection import *
 
import redis
import time
import logging
import random
 
logging.basicConfig(format='[%(levelname)s]: %(message)s', level=logging.DEBUG)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
 
 
class RedisError(Exception):
 def __init__(self, value):
 self.value = value
 def __str__(self):
 return repr(self.value)
 
class fastdfsClient(Fdfs_client):
 def __init__(self):
 self.tracker_pool = ConnectionPool(**setting.fdfs_tracker)
 self.timeout = setting.fdfs_tracker['timeout']
 return None
 
 def __del__(self):
 try:
 self.pool.destroy()
 self.pool = None
 except:
 pass
 
class fastdfs(object):
 def __init__(self):
 '''
 conf_file:配置文件
 '''
 self.fdfs_client = fastdfsClient()
 self.fdfs_redis = []
 for i in setting.fdfs_redis_dbs:
 self.fdfs_redis.append(redis.Redis(host=i[0], port=i[1], db=i[2]))
 
 def store_by_buffer(self,buf,filename=None,file_ext_name = None):
 '''
 buffer存储文件
 参数:
 filename:自定义文件名,如果不指定,将远程file_id作为文件名
 file_ext_name:文件扩展名(可选),如果不指定,将根据自定义文件名智能判断
 返回值:
 {
 'group':组名,
 'file_id':不含组名的文件ID,
 'size':文件尺寸,
 'upload_time':上传时间
 }
 '''
 if filename and random.choice(self.fdfs_redis).exists(filename):
 logger.info('File(%s) exists.'%filename)
 return random.choice(self.fdfs_redis).hgetall(filename)
 t1 = time.time()
# try:
 ret_dict = self.fdfs_client.upload_by_buffer(buf,file_ext_name)
# except Exception,e:
# logger.error('Error occurred while uploading: %s'%e.message)
# return None
 t2 = time.time()
 logger.info('Upload file(%s) by buffer, time consume: %fs' % (filename,(t2 - t1)))
 for key in ret_dict:
 logger.debug('[+] %s : %s' % (key, ret_dict[key]))
 stored_filename = ret_dict['Remote file_id']
 stored_filename_without_group = stored_filename[stored_filename.index('/')+1:]
 if not filename:
 filename =stored_filename_without_group
 vmp = {'group':ret_dict['Group name'],'file_id':stored_filename_without_group,'size':ret_dict['Uploaded size'],'upload_time':int(time.time()*1000)}
 try:
 for i in self.fdfs_redis:
 if not i.hmset(filename,vmp):
 raise RedisError('Save Failure')
 logger.info('Store file(%s) by buffer successful' % filename)
 except Exception,e:
 logger.error('Save info to Redis failure. rollback...')
 try:
 ret_dict = self.fdfs_client.delete_file(stored_filename)
 except Exception,e:
 logger.error('Error occurred while deleting: %s'%e.message)
 return None
 return vmp
 
 def remove(self,filename):
 '''
 删除文件,
 filename是用户自定义文件名
 return True|False
 '''
 fileinfo = random.choice(self.fdfs_redis).hgetall(filename)
 stored_filename = '%s/%s'%(fileinfo['group'],fileinfo['file_id'])
 try:
 ret_dict = self.fdfs_client.delete_file(stored_filename)
 logger.info('Remove stored file successful')
 except Exception,e:
 logger.error('Error occurred while deleting: %s'%e.message)
 return False
 for i in self.fdfs_redis:
 if not i.delete(filename):
 logger.error('Remove fileinfo in redis failure')
 logger.info('%s removed.'%filename)
 return True
 
 def download(self,filename):
 '''
 下载文件
 返回二进制
 '''
 finfo = self.getInfo(filename)
 if finfo:
 ret = self.fdfs_client.download_to_buffer('%s/%s'%(finfo['group'],finfo['file_id']))
 return ret['Content']
 else:
 logger.debug('%s is not exists'%filename)
 return None
 
 def list(self,pattern='*'):
 '''
 列出文件列表
 '''
 return random.choice(self.fdfs_redis).keys(pattern)
 
 def getInfo(self,filename):
 '''
 获得文件信息
 return:{
 'group':组名,
 'file_id':不含组名的文件ID,
 'size':文件尺寸,
 'upload_time':上传时间
 }
 '''
 return random.choice(self.fdfs_redis).hgetall(filename)

配置:

# -*- coding: utf-8 -*-
#fastdfs tracker, multiple tracker supported
fdfs_tracker = {
'host_tuple':('192.168.2.233','192.168.2.234'),
'port':22122,
'timeout':30,
'name':'Tracker Pool'
}
#fastdfs meta db, multiple redisdb supported
fdfs_redis_dbs = (
 ('192.168.2.233',6379,0),
 ('192.168.2.233',6379,1)
)

下载本文
显示全文
专题