fastdfs之sync源码分析

1. sync流程简介

sync行为是由storage向tracker report的时候，响应信息触发为组内每一个storage服务器创建一个同步线程（仅仅一个，单线程同步），并且响应信息还更新线程中storage的状态。同步线程根据storage状态判断工作状态，根据binlog改变实现文件同步。

sync被触发的流程如下，从下而上。

2. storage sync相关源码解析 

2.1 sync 触发

sync线程由tracker_merge_servers函数触发。其功能合并storage服务器的状态，触发sync线程，向tracker上报状态不同的storage。

storage/tracker_client_thread.c

static int tracker_merge_servers(ConnectionInfo *pTrackerServer, \
		FDFSStorageBrief *briefServers, const int server_count)
{
	FDFSStorageBrief *pServer;
	FDFSStorageBrief *pEnd;
	FDFSStorageServer *pInsertedServer;
	FDFSStorageServer **ppFound;
	FDFSStorageServer **ppGlobalServer;
	FDFSStorageServer **ppGlobalEnd;
	FDFSStorageServer targetServer;
	FDFSStorageServer *pTargetServer;
	FDFSStorageBrief diffServers[FDFS_MAX_SERVERS_EACH_GROUP];
	FDFSStorageBrief *pDiffServer;
	int res;
	int result;
	int nDeletedCount;

	memset(&targetServer, 0, sizeof(targetServer));
	pTargetServer = &targetServer;

	nDeletedCount = 0;
	pDiffServer = diffServers;
	pEnd = briefServers + server_count;
	for (pServer=briefServers; pServerserver.ip_addr, pServer->ip_addr);
			}

			/*
			//logInfo("ip_addr=%s, local status: %d, " \
				"tracker status: %d", pServer->ip_addr, \
				(*ppFound)->server.status, pServer->status);
			*/
			if ((*ppFound)->server.status == pServer->status)
			{ // 如果状态没改变，判断下一个。
				continue;
			}

			// 当输入状态改变为：离线
			if (pServer->status == FDFS_STORAGE_STATUS_OFFLINE)
			{
				if ((*ppFound)->server.status == \
						FDFS_STORAGE_STATUS_ACTIVE
				 || (*ppFound)->server.status == \
						FDFS_STORAGE_STATUS_ONLINE)
				{ // 本地状态为：活动或者在线，则改变为离线
					(*ppFound)->server.status = \
					FDFS_STORAGE_STATUS_OFFLINE;
				}
				else if ((*ppFound)->server.status != \
						FDFS_STORAGE_STATUS_NONE
				     && (*ppFound)->server.status != \
						FDFS_STORAGE_STATUS_INIT)
				{ // 本地状态为：无或者初始化，加入不同状态服务列表，准备告知tracker
					// add differ server
					memcpy(pDiffServer++, \
						&((*ppFound)->server), \
						sizeof(FDFSStorageBrief));
				}
			}
			// 当输入状态不为离线
			else if ((*ppFound)->server.status == \
					FDFS_STORAGE_STATUS_OFFLINE)
			{ // 本地状态为：离线，更新本地状态
				(*ppFound)->server.status = pServer->status;
			}
			else if ((*ppFound)->server.status == \
					FDFS_STORAGE_STATUS_NONE)
			{ // 本地状态：无
				if (pServer->status == \
					FDFS_STORAGE_STATUS_DELETED \
				 || pServer->status == \
					FDFS_STORAGE_STATUS_IP_CHANGED)
				{ //ignore
					// 输入状态：删除，ip改变，直接忽略。
				}
				else
				{
// *** 本地状态为无，变了状态，说明该storage工作了，启动同步线程。
					// 备注：storage与storage之间同步线程，是单线程的。
                    //        仅仅启动一个storage-storage sync线程。
                    // 注意：函数输入参数为FDFSStorageBrief，指向全局变量g_sorted_storages之中
					(*ppFound)->server.status = \
							pServer->status;
					// start sync threads [ppFound is none, and then change]
					if ((result=tracker_start_sync_threads(\
						&((*ppFound)->server))) != 0)
					{
						return result;
					}
				}
			}
			else if (((pServer->status == \
					FDFS_STORAGE_STATUS_WAIT_SYNC) || \
				(pServer->status == \
					FDFS_STORAGE_STATUS_SYNCING)) && \
				((*ppFound)->server.status > pServer->status))
			{
                *(pServer->ip_addr + IP_ADDRESS_SIZE - 1) = '\0';
				if (is_local_host_ip(pServer->ip_addr) && \
					buff2int(pServer->port) == g_server_port)
				{
					need_rejoin_tracker = true;
					logWarning("file: "__FILE__", line: %d, " \
						"tracker response status: %d, " \
						"local status: %d, need rejoin " \
						"tracker server: %s:%d", \
						__LINE__, pServer->status, \
						(*ppFound)->server.status, \
						pTrackerServer->ip_addr,
						pTrackerServer->port);
				}
				// pxxian: add differ server
				memcpy(pDiffServer++, &((*ppFound)->server), \
					sizeof(FDFSStorageBrief));
			}
			else
			{
				(*ppFound)->server.status = pServer->status;
			}
		} // pxxian: end of if (ppFound != NULL)
		else if (pServer->status == FDFS_STORAGE_STATUS_DELETED
		      || pServer->status == FDFS_STORAGE_STATUS_IP_CHANGED)
		{   //ignore
			nDeletedCount++;
		}
		else
		{
			/*
			//logInfo("ip_addr=%s, tracker status: %d", 
				pServer->ip_addr, pServer->status);
			*/

			if ((res=pthread_mutex_lock( \
				 &reporter_thread_lock)) != 0)
			{
				logError("file: "__FILE__", line: %d, "\
					"call pthread_mutex_lock fail,"\
					" errno: %d, error info: %s", \
					__LINE__, res, STRERROR(res));
			}

			// pxxian: add storage server to g_storage_servers and g_sorted_storages
			if (g_storage_count < FDFS_MAX_SERVERS_EACH_GROUP)
			{
				pInsertedServer = g_storage_servers + \
						g_storage_count;
				memcpy(&(pInsertedServer->server), \
					pServer, sizeof(FDFSStorageBrief));
				// pxxian: insert storage into g_sorted_storages
				if (tracker_insert_into_sorted_servers( \
						pInsertedServer))
				{
					g_storage_count++;

// *** 在本地没找到该storage信息，该storage为加入的服务器，则启动一个sync线程
                    // 注意：函数输入参数为FDFSStorageBrief，指向全局变量g_sorted_storages之中
					// start sync threads [ppFound is NULL]
					result = tracker_start_sync_threads( \
						&(pInsertedServer->server));
				}
				else
				{
					result = 0;
				}
			}
			else
			{
				logError("file: "__FILE__", line: %d, " \
					"tracker server %s:%d, " \
					"storage servers of group \"%s\" " \
					"exceeds max: %d", \
					__LINE__, pTrackerServer->ip_addr, \
					pTrackerServer->port, g_group_name, \
					FDFS_MAX_SERVERS_EACH_GROUP);
				result = ENOSPC;
			}

			if ((res=pthread_mutex_unlock( \
				&reporter_thread_lock)) != 0)
			{
				logError("file: "__FILE__", line: %d, "\
				"call pthread_mutex_unlock fail, " \
				"errno: %d, error info: %s", \
				__LINE__, res, STRERROR(res));
			}

			if (result != 0)
			{
				return result;
			}
		} // pxxian: end of else
	} // pxxian: end of for

	if (g_storage_count + nDeletedCount == server_count)
	{
		if (pDiffServer - diffServers > 0)
		{
			return tracker_sync_diff_servers(pTrackerServer, \
				diffServers, pDiffServer - diffServers);
		}

		return 0;
	}

	ppGlobalServer = g_sorted_storages;
	ppGlobalEnd = g_sorted_storages + g_storage_count;
	pServer = briefServers;
	while (pServer < pEnd && ppGlobalServer < ppGlobalEnd)
	{
		if ((*ppGlobalServer)->server.status == FDFS_STORAGE_STATUS_NONE)
		{
			ppGlobalServer++;
			continue;
		}

		res = strcmp(pServer->id, (*ppGlobalServer)->server.id);
		if (res < 0)
		{
			if (pServer->status != FDFS_STORAGE_STATUS_DELETED
			 && pServer->status != FDFS_STORAGE_STATUS_IP_CHANGED)
			{
				logError("file: "__FILE__", line: %d, " \
					"tracker server %s:%d, " \
					"group \"%s\", " \
					"enter impossible statement branch", \
					__LINE__, pTrackerServer->ip_addr, \
					pTrackerServer->port, g_group_name);
			}

			pServer++;
		}
		else if (res == 0)
		{
			pServer++;
			ppGlobalServer++;
		}
		else
		{
			// pxxian: add differ server
			memcpy(pDiffServer++, &((*ppGlobalServer)->server), \
				sizeof(FDFSStorageBrief));
			ppGlobalServer++;
		}
	}

	while (ppGlobalServer < ppGlobalEnd)
	{
		if ((*ppGlobalServer)->server.status == FDFS_STORAGE_STATUS_NONE)
		{
			ppGlobalServer++;
			continue;
		}
		// pxxian: add differ server
		memcpy(pDiffServer++, &((*ppGlobalServer)->server), \
			sizeof(FDFSStorageBrief));
		ppGlobalServer++;
	}

	return tracker_sync_diff_servers(pTrackerServer, \
			diffServers, pDiffServer - diffServers);
}

 2.2 启动sync线程

sync线程的启动由tracker_start_sync_threads[storage/tracker_client_thread.c]，调用storage_sync_thread_start[storage/sync.c]创建同步sync线程。

storage/tracker_client_thread.c

static int tracker_start_sync_threads(const FDFSStorageBrief *pStorage)
{
	int result;

    // 判断是否是当前storage，是则跳过
	if (strcmp(pStorage->id, g_my_server_id_str) == 0)
	{
		return 0;
	}

    // 调用创建sync线程函数
	result = storage_sync_thread_start(pStorage);
	if (result == 0)
	{
		if (g_if_trunker_self) // 小文件合并存储同步【当下不做分析】
		{
			result = trunk_sync_thread_start(pStorage);
		}
	}

	return result;
}

 storage/sync.c

int storage_sync_thread_start(const FDFSStorageBrief *pStorage)
{
	int result;
	pthread_attr_t pattr;
	pthread_t tid;

    // 判断storage是否合适
	if (pStorage->status == FDFS_STORAGE_STATUS_DELETED || \
	    pStorage->status == FDFS_STORAGE_STATUS_IP_CHANGED || \
	    pStorage->status == FDFS_STORAGE_STATUS_NONE)
	{
		logWarning("file: "__FILE__", line: %d, " \
			"storage id: %s 's status: %d is invalid, " \
			"can't start sync thread!", __LINE__, \
			pStorage->id, pStorage->status);
		return 0;
	}

    // 判断是否是storage本身
	if (storage_server_is_myself(pStorage) || \
		is_local_host_ip(pStorage->ip_addr)) //can't self sync to self
	{
		logWarning("file: "__FILE__", line: %d, " \
			"storage id: %s is myself, can't start sync thread!", \
			__LINE__, pStorage->id);
		return 0;
	}
    // 初始化线程属性
	if ((result=init_pthread_attr(&pattr, g_thread_stack_size)) != 0)
	{
		return result;
	}

	/*
	//printf("start storage ip_addr: %s, g_storage_sync_thread_count=%d\n", 
			pStorage->ip_addr, g_storage_sync_thread_count);
	*/

// *** 创建同步线程
	if ((result=pthread_create(&tid, &pattr, storage_sync_thread_entrance, \
		(void *)pStorage)) != 0)
	{
		logError("file: "__FILE__", line: %d, " \
			"create thread failed, errno: %d, " \
			"error info: %s", \
			__LINE__, result, STRERROR(result));

		pthread_attr_destroy(&pattr);
		return result;
	}

	if ((result=pthread_mutex_lock(&sync_thread_lock)) != 0)
	{
		logError("file: "__FILE__", line: %d, " \
			"call pthread_mutex_lock fail, " \
			"errno: %d, error info: %s", \
			__LINE__, result, STRERROR(result));
	}
    // 更新storage全局变量
	g_storage_sync_thread_count++;
	sync_tids = (pthread_t *)realloc(sync_tids, sizeof(pthread_t) * \
					g_storage_sync_thread_count);
	if (sync_tids == NULL)
	{
		logError("file: "__FILE__", line: %d, " \
			"malloc %d bytes fail, " \
			"errno: %d, error info: %s", \
			__LINE__, (int)sizeof(pthread_t) * \
			g_storage_sync_thread_count, \
			errno, STRERROR(errno));
	}
	else
	{
		sync_tids[g_storage_sync_thread_count - 1] = tid;
	}

	if ((result=pthread_mutex_unlock(&sync_thread_lock)) != 0)
	{
		logError("file: "__FILE__", line: %d, " \
			"call pthread_mutex_unlock fail, " \
			"errno: %d, error info: %s", \
			__LINE__, result, STRERROR(result));
	}

	pthread_attr_destroy(&pattr);

	return 0;
}

2.3 sync线程入口

同步线程入口函数为storage_sync_thread_entrance[storage/storage_sync.c]，同步线程在此函数内循环，实现storage-storage之间文件同步。

storage/storage_sync.c

static void* storage_sync_thread_entrance(void* arg)
{
	FDFSStorageBrief *pStorage;
	StorageBinLogReader reader;
	StorageBinLogRecord record;
	ConnectionInfo storage_server;
	char local_ip_addr[IP_ADDRESS_SIZE];
	int read_result;
	int sync_result;
	int conn_result;
	int result;
	int record_len;
	int previousCode;
	int nContinuousFail;
	time_t current_time;
	time_t start_time;
	time_t end_time;
	time_t last_keep_alive_time;
	

	memset(local_ip_addr, 0, sizeof(local_ip_addr));
	memset(&reader, 0, sizeof(reader));
	reader.mark_fd = -1;
	reader.binlog_fd = -1;

	current_time =  g_current_time;
	last_keep_alive_time = 0;
	start_time = 0;
	end_time = 0;

    // 前面有介绍，pStorage指向全局变量g_sorted_storages中的一个，指向同组非本身的storage服务器。
    // 其状态的改变都在tracker_merge_servers函数完成，同步线程根据状态、binlog的改变完成同步工作
	pStorage = (FDFSStorageBrief *)arg;

	strcpy(storage_server.ip_addr, pStorage->ip_addr);
	storage_server.port = g_server_port;
	storage_server.sock = -1;

    // 同步线程最外一层while循环
	logDebug("file: "__FILE__", line: %d, " \
		"sync thread to storage server %s:%d started", \
		__LINE__, storage_server.ip_addr, storage_server.port);
	// sync thread to storage start
	while (g_continue_flag && \
		pStorage->status != FDFS_STORAGE_STATUS_DELETED && \
		pStorage->status != FDFS_STORAGE_STATUS_IP_CHANGED && \
		pStorage->status != FDFS_STORAGE_STATUS_NONE)
	{ // 当本storage服务器继续运行，其同组的目的storage状态不为删除、ip变化、无
		while (g_continue_flag && \
			(pStorage->status == FDFS_STORAGE_STATUS_INIT ||
			 pStorage->status == FDFS_STORAGE_STATUS_OFFLINE ||
			 pStorage->status == FDFS_STORAGE_STATUS_ONLINE))
		{ // pStorage未准备好，继续等待。
			sleep(1);
		}

        // 先判断pStorage是否退出了。
		if ((!g_continue_flag) ||
			pStorage->status == FDFS_STORAGE_STATUS_DELETED || \
			pStorage->status == FDFS_STORAGE_STATUS_IP_CHANGED || \
			pStorage->status == FDFS_STORAGE_STATUS_NONE)
		{
			break;
		}

        // 根据storage配置文件storage.conf定义的同步时间段sync_start_time~sync_end_time判断
        // 是否存在不同步时间，使能g_sync_part_time，存在则为true，则在这段时间不进行同步行为。
		if (g_sync_part_time)
		{
			current_time = g_current_time;
			storage_sync_get_start_end_times(current_time, \
				&g_sync_end_time, &g_sync_start_time, \
				&start_time, &end_time);
			start_time += 60;
			end_time -= 60;
			while (g_continue_flag && (current_time >= start_time \
					&& current_time <= end_time))
			{
				current_time = g_current_time;
				sleep(1);
			}
		}

		previousCode = 0;
		nContinuousFail = 0;
		conn_result = 0;

		// connect to the storage
		while (g_continue_flag && \
			pStorage->status != FDFS_STORAGE_STATUS_DELETED && \
			pStorage->status != FDFS_STORAGE_STATUS_IP_CHANGED && \
			pStorage->status != FDFS_STORAGE_STATUS_NONE)
		{
			strcpy(storage_server.ip_addr, pStorage->ip_addr);
			storage_server.sock = \
				socket(AF_INET, SOCK_STREAM, 0);
			if(storage_server.sock < 0)
			{
				logCrit("file: "__FILE__", line: %d," \
					" socket create fail, " \
					"errno: %d, error info: %s. " \
					"program exit!", __LINE__, \
					errno, STRERROR(errno));
				g_continue_flag = false;
				break;
			}

			if (g_client_bind_addr && *g_bind_addr != '\0')
			{
				socketBind(storage_server.sock, g_bind_addr, 0);
			}

			if (tcpsetnonblockopt(storage_server.sock) != 0)
			{
				nContinuousFail++;
				close(storage_server.sock);
				storage_server.sock = -1;
				sleep(1);

				continue;
			}

			if ((conn_result=connectserverbyip_nb(storage_server.sock,\
				pStorage->ip_addr, g_server_port, \
				g_fdfs_connect_timeout)) == 0)
			{
				char szFailPrompt[64];
				if (nContinuousFail == 0)
				{
					*szFailPrompt = '\0';
				}
				else
				{
					sprintf(szFailPrompt, \
						", continuous fail count: %d", \
						nContinuousFail);
				}
				logInfo("file: "__FILE__", line: %d, " \
					"successfully connect to " \
					"storage server %s:%d%s", __LINE__, \
					pStorage->ip_addr, \
					g_server_port, szFailPrompt);
				nContinuousFail = 0;
				break;
			}

			if (previousCode != conn_result)
			{
				logError("file: "__FILE__", line: %d, " \
					"connect to storage server %s:%d fail" \
					", errno: %d, error info: %s", \
					__LINE__, \
					pStorage->ip_addr, g_server_port, \
					conn_result, STRERROR(conn_result));
				previousCode = conn_result;
			}

			nContinuousFail++;
			close(storage_server.sock);
			storage_server.sock = -1;

			if (!g_continue_flag)
			{
				break;
			}

			sleep(1);
		} // end of while connect

		if (nContinuousFail > 0)
		{
			logError("file: "__FILE__", line: %d, " \
				"connect to storage server %s:%d fail, " \
				"try count: %d, errno: %d, error info: %s", \
				__LINE__, pStorage->ip_addr, \
				g_server_port, nContinuousFail, \
				conn_result, STRERROR(conn_result));
		}

        // 判断是否该退出
		if ((!g_continue_flag) ||
			pStorage->status == FDFS_STORAGE_STATUS_DELETED || \
			pStorage->status == FDFS_STORAGE_STATUS_IP_CHANGED || \
			pStorage->status == FDFS_STORAGE_STATUS_NONE)
		{
			break;
		}

		if (pStorage->status != FDFS_STORAGE_STATUS_ACTIVE && \
			pStorage->status != FDFS_STORAGE_STATUS_WAIT_SYNC && \
			pStorage->status != FDFS_STORAGE_STATUS_SYNCING)
		{ // 如果不是活动，等待同步，同步中的状态，等待，重新连接storage
			close(storage_server.sock);
			sleep(5);
			continue;
		}

		// read the binlog records which need to sync
		if ((result=storage_reader_init(pStorage, &reader)) != 0)
		{
			logCrit("file: "__FILE__", line: %d, " \
				"storage_reader_init fail, errno=%d, " \
				"program exit!", \
				__LINE__, result);
			g_continue_flag = false;
			break;
		}

		if (!reader.need_sync_old)
		{ // 上次同步已完成
			while (g_continue_flag && \
			(pStorage->status != FDFS_STORAGE_STATUS_ACTIVE && \
			 pStorage->status != FDFS_STORAGE_STATUS_DELETED && \
			 pStorage->status != FDFS_STORAGE_STATUS_IP_CHANGED && \
			 pStorage->status != FDFS_STORAGE_STATUS_NONE))
			{ // 等待状态变为活动，或者其他应该要退出的状态：删除，ip改变，无
				sleep(1);
			}

			if (pStorage->status != FDFS_STORAGE_STATUS_ACTIVE)
			{ // 仅pStorage为活动，才往下走
				close(storage_server.sock);
				storage_reader_destroy(&reader);
				continue;
			}
		}

		getSockIpaddr(storage_server.sock, \
			local_ip_addr, IP_ADDRESS_SIZE);
		insert_into_local_host_ip(local_ip_addr);

		/*
		//printf("file: "__FILE__", line: %d, " \
			"storage_server.ip_addr=%s, " \
			"local_ip_addr: %s\n", \
			__LINE__, pStorage->ip_addr, local_ip_addr);
		*/
        // 根据ip地址判断是否是本身，是则退出
		if (is_local_host_ip(pStorage->ip_addr))
		{  //can't self sync to self
			logError("file: "__FILE__", line: %d, " \
				"ip_addr %s belong to the local host," \
				" sync thread exit.", \
				__LINE__, pStorage->ip_addr);
			fdfs_quit(&storage_server);
			close(storage_server.sock);
			break;
		}

		// report this storage server id to the storage server
		if (storage_report_my_server_id(&storage_server) != 0)
		{
			close(storage_server.sock);
			storage_reader_destroy(&reader);
			sleep(1);
			continue;
		}

		// change the storage status.
		if (pStorage->status == FDFS_STORAGE_STATUS_WAIT_SYNC)
		{
			pStorage->status = FDFS_STORAGE_STATUS_SYNCING;
			storage_report_storage_status(pStorage->id, \
				pStorage->ip_addr, pStorage->status);
		}

		if (pStorage->status == FDFS_STORAGE_STATUS_SYNCING)
		{
			if (reader.need_sync_old && reader.sync_old_done)
			{ // 如果当前状态为同步中，并且现在已经完成同步，改变pStorage状态为离线（fdfs规则）
				pStorage->status = FDFS_STORAGE_STATUS_OFFLINE;
				storage_report_storage_status(pStorage->id, \
					pStorage->ip_addr, \
					pStorage->status);
			}
		}

		if (g_sync_part_time)
		{
			current_time = g_current_time;
			storage_sync_get_start_end_times(current_time, \
				&g_sync_start_time, &g_sync_end_time, \
				&start_time, &end_time);
		}

// *** 同步文件循环
		sync_result = 0;
		while (g_continue_flag && (!g_sync_part_time || \
			(current_time >= start_time && \
			current_time <= end_time)) && \
			(pStorage->status == FDFS_STORAGE_STATUS_ACTIVE || \
			pStorage->status == FDFS_STORAGE_STATUS_SYNCING))
		{
			// read a binlog record
			read_result = storage_binlog_read(&reader, \
					&record, &record_len);

			if (read_result == ENOENT)
			{ // 同步结束
				// sync old done
				if (reader.need_sync_old && \
					!reader.sync_old_done)
				{
					reader.sync_old_done = true;
					if (storage_write_to_mark_file(&reader) != 0)
					{
						logCrit("file: "__FILE__", line: %d, " \
							"storage_write_to_mark_file " \
							"fail, program exit!", \
							__LINE__);
						g_continue_flag = false;
						break;
					}

					// report sync done
					if (pStorage->status == \
						FDFS_STORAGE_STATUS_SYNCING)
					{
						pStorage->status = \
							FDFS_STORAGE_STATUS_OFFLINE;
						storage_report_storage_status( \
							pStorage->id, \
							pStorage->ip_addr, \
							pStorage->status);
					}
				}


				if (reader.last_scan_rows!=reader.scan_row_count)
				{
					if (storage_write_to_mark_file(&reader)!=0)
					{
						logCrit("file: "__FILE__", line: %d, " \
							"storage_write_to_mark_file fail, " \
							"program exit!", __LINE__);
						g_continue_flag = false;
						break;
					}
				}

                // 判断pStorage是否存活，无同步文件时，常在此循环
				current_time = g_current_time;
				if (current_time - last_keep_alive_time >= \
					g_heart_beat_interval)
				{
					if (fdfs_active_test(&storage_server)!=0)
					{
						break;
					}

					last_keep_alive_time = current_time;
				}
                // 等待，重新去读取binlog记录
				usleep(g_sync_wait_usec);
				continue;
			}

			if (g_sync_part_time)
			{
				current_time = g_current_time;
			}

			/*
			logDebug("file: "__FILE__", line: %d, " \
						"sync logic file: %s", \
						__LINE__, record.filename);
			*/
			if (read_result != 0)
			{ // 读取binlog记录，出错
				if (read_result == EINVAL && \
					g_file_sync_skip_invalid_record)
				{
					logWarning("file: "__FILE__", line: %d, " \
						"skip invalid record, binlog index: " \
						"%d, offset: %"PRId64, \
						__LINE__, reader.binlog_index, \
						reader.binlog_offset);
				}
				else
				{
					sleep(5);
					break;
				}
			}
// *** deal with sync data，进行storage-storage之间文件同步
			else if ((sync_result=storage_sync_data(&reader, \
				&storage_server, &record)) != 0)
			{
				logDebug("file: "__FILE__", line: %d, " \
					"binlog index: %d, current record " \
					"offset: %"PRId64", next " \
					"record offset: %"PRId64, \
					__LINE__, reader.binlog_index, \
					reader.binlog_offset, \
					reader.binlog_offset + record_len);
				if (rewind_to_prev_rec_end(&reader) != 0)
				{
					logCrit("file: "__FILE__", line: %d, " \
						"rewind_to_prev_rec_end fail, "\
						"program exit!", __LINE__);
					g_continue_flag = false;
				}

				break;
			}

			reader.binlog_offset += record_len;
			reader.scan_row_count++;

			if (g_sync_interval > 0)
			{
				usleep(g_sync_interval);
			}
		} // end of while

		if (reader.last_scan_rows != reader.scan_row_count)
		{
			if (storage_write_to_mark_file(&reader) != 0)
			{
				logCrit("file: "__FILE__", line: %d, " \
					"storage_write_to_mark_file fail, " \
					"program exit!", __LINE__);
				g_continue_flag = false;
				break;
			}
		}

		close(storage_server.sock);
		storage_server.sock = -1;
		storage_reader_destroy(&reader);

		if (!g_continue_flag)
		{
			break;
		}

		if (!(sync_result == ENOTCONN || sync_result == EIO))
		{
			sleep(1);
		}
	} // end of while. 退出最外一层

	if (storage_server.sock >= 0)
	{
		close(storage_server.sock);
	}
	storage_reader_destroy(&reader);

	if (pStorage->status == FDFS_STORAGE_STATUS_DELETED
	 || pStorage->status == FDFS_STORAGE_STATUS_IP_CHANGED)
	{
		storage_changelog_req();
		sleep(2 * g_heart_beat_interval + 1);
		pStorage->status = FDFS_STORAGE_STATUS_NONE;
	}

	// clean this pthread_t tid
	storage_sync_thread_exit(&storage_server);

	return NULL;
}

2.4 sync数据同步

sync线程文件同步是主动分发的，先判断自身的binlog是否存在“client-storage直接操作的”而又未同步给其他storage的记录，调用storage_sync_data[storage/storage_sync.c]实现文件同步。

storage/storage_sync.c

static int storage_sync_data(StorageBinLogReader *pReader, \
			ConnectionInfo *pStorageServer, \
			StorageBinLogRecord *pRecord)
{
	int result;

	// deal with data sync
	switch(pRecord->op_type)
	{
		case STORAGE_OP_TYPE_SOURCE_CREATE_FILE:
			result = storage_sync_copy_file(pStorageServer, \
					pReader, pRecord, \
					STORAGE_PROTO_CMD_SYNC_CREATE_FILE);
			break;
		case STORAGE_OP_TYPE_SOURCE_DELETE_FILE:
			result = storage_sync_delete_file( \
					pStorageServer, pRecord);
			break;
		case STORAGE_OP_TYPE_SOURCE_UPDATE_FILE:
			result = storage_sync_copy_file(pStorageServer, \
					pReader, pRecord, \
					STORAGE_PROTO_CMD_SYNC_UPDATE_FILE);
			break;
		case STORAGE_OP_TYPE_SOURCE_APPEND_FILE:
			result = storage_sync_modify_file(pStorageServer, \
					pReader, pRecord, \
					STORAGE_PROTO_CMD_SYNC_APPEND_FILE);
			if (result == ENOENT)  //resync appender file
			{
			result = storage_sync_copy_file(pStorageServer, \
					pReader, pRecord, \
					STORAGE_PROTO_CMD_SYNC_UPDATE_FILE);
			}
			break;
		case STORAGE_OP_TYPE_SOURCE_MODIFY_FILE:
			result = storage_sync_modify_file(pStorageServer, \
					pReader, pRecord, \
					STORAGE_PROTO_CMD_SYNC_MODIFY_FILE);
			if (result == ENOENT)  //resync appender file
			{
			result = storage_sync_copy_file(pStorageServer, \
					pReader, pRecord, \
					STORAGE_PROTO_CMD_SYNC_UPDATE_FILE);
			}
			break;
		case STORAGE_OP_TYPE_SOURCE_TRUNCATE_FILE:
			result = storage_sync_truncate_file(pStorageServer, \
					pReader, pRecord);
			break;
		case STORAGE_OP_TYPE_SOURCE_CREATE_LINK:
			result = storage_sync_link_file(pStorageServer, \
					pRecord);
			break;
		case STORAGE_OP_TYPE_REPLICA_CREATE_FILE:
			STARAGE_CHECK_IF_NEED_SYNC_OLD(pReader, pRecord)
			result = storage_sync_copy_file(pStorageServer, \
					pReader, pRecord, \
					STORAGE_PROTO_CMD_SYNC_CREATE_FILE);
			break;
		case STORAGE_OP_TYPE_REPLICA_DELETE_FILE:
			STARAGE_CHECK_IF_NEED_SYNC_OLD(pReader, pRecord)
			result = storage_sync_delete_file( \
					pStorageServer, pRecord);
			break;
		case STORAGE_OP_TYPE_REPLICA_UPDATE_FILE:
			STARAGE_CHECK_IF_NEED_SYNC_OLD(pReader, pRecord)
			result = storage_sync_copy_file(pStorageServer, \
					pReader, pRecord, \
					STORAGE_PROTO_CMD_SYNC_UPDATE_FILE);
			break;
		case STORAGE_OP_TYPE_REPLICA_CREATE_LINK:
			STARAGE_CHECK_IF_NEED_SYNC_OLD(pReader, pRecord)
			result = storage_sync_link_file(pStorageServer, \
					pRecord);
			break;
		case STORAGE_OP_TYPE_REPLICA_APPEND_FILE:
			return 0;
		case STORAGE_OP_TYPE_REPLICA_MODIFY_FILE:
			return 0;
		case STORAGE_OP_TYPE_REPLICA_TRUNCATE_FILE:
			return 0;
		default:
			logError("file: "__FILE__", line: %d, " \
				"invalid file operation type: %d", \
				__LINE__, pRecord->op_type);
			return EINVAL;
	}

	// renew message
	if (result == 0)
	{
		pReader->sync_row_count++;

		if (pReader->sync_row_count - pReader->last_sync_rows >= \
			g_write_mark_file_freq)
		{
			if ((result=storage_write_to_mark_file(pReader)) != 0)
			{
				logCrit("file: "__FILE__", line: %d, " \
					"storage_write_to_mark_file " \
					"fail, program exit!", __LINE__);
				g_continue_flag = false;
				return result;
			}
			
		}
	}

	return result;
}