Redis(5.0.0)持久化AOF和 RDB 结合源码分析

主要是挖个坑。候补（代码还没看完。。）

https://github.com/antirez/redis/tree/5.0

一、Redis保存持久化文件

二、Redis启动加载持久化文件

src/server.c loadDataFromDisk函数 AOF 和 RDB 通过不同的方式加载

 /* Function called at startup to load RDB or AOF file in memory. */
 void loadDataFromDisk(void) {
     long long start = ustime();
     if (server.aof_state == AOF_ON) {
         if (loadAppendOnlyFile(server.aof_filename) == C_OK)
             serverLog(LL_NOTICE,"DB loaded from append only file: %.3f seconds",(float)(ustime()-start)/);
     } else {
         rdbSaveInfo rsi = RDB_SAVE_INFO_INIT;
         if (rdbLoad(server.rdb_filename,&rsi) == C_OK) {
             serverLog(LL_NOTICE,"DB loaded from disk: %.3f seconds",
                 (float)(ustime()-start)/);

             /* Restore the replication ID / offset from the RDB file. */
             if (server.masterhost &&
                 rsi.repl_id_is_set &&
                 rsi.repl_offset != - &&
                 /* Note that older implementations may save a repl_stream_db
                  * of -1 inside the RDB file in a wrong way, see more information
                  * in function rdbPopulateSaveInfo. */
                 rsi.repl_stream_db != -)
             {
                 memcpy(server.replid,rsi.repl_id,sizeof(server.replid));
                 server.master_repl_offset = rsi.repl_offset;
                 /* If we are a slave, create a cached master from this
                  * information, in order to allow partial resynchronizations
                  * with masters. */
                 replicationCacheMasterUsingMyself();
                 selectDb(server.cached_master,rsi.repl_stream_db);
             }
         } else if (errno != ENOENT) {
             serverLog(LL_WARNING,"Fatal error loading the DB: %s. Exiting.",strerror(errno));
             exit();
         }
     }
 }

loadDataFromDisk

AOF路线

src/aof.c loadAppendOnlyFile 主要通过检查是否含有RDB前缀文件，如果有则加载，创建一个FakeClient一条条的执行AOF文件中的指令 。

特别地，事务会单独加载。

 int loadAppendOnlyFile(char *filename) {
     struct client *fakeClient;
     FILE *fp = fopen(filename,"r");
     struct redis_stat sb;
     int old_aof_state = server.aof_state;
     long loops = ;
     off_t valid_up_to = ; /* Offset of latest well-formed command loaded. */
     off_t valid_before_multi = ; /* Offset before MULTI command loaded. */

     if (fp == NULL) {
         serverLog(LL_WARNING,"Fatal error: can't open the append log file for reading: %s",strerror(errno));
         exit();
     }

     /* Handle a zero-length AOF file as a special case. An empty AOF file
      * is a valid AOF because an empty server with AOF enabled will create
      * a zero length file at startup, that will remain like that if no write
      * operation is received. */
     if (fp && redis_fstat(fileno(fp),&sb) != - && sb.st_size == ) {
         server.aof_current_size = ;
         fclose(fp);
         return C_ERR;
     }

     /* Temporarily disable AOF, to prevent EXEC from feeding a MULTI
      * to the same file we're about to read. */
     server.aof_state = AOF_OFF;

     fakeClient = createFakeClient();
     startLoading(fp);

     /* Check if this AOF file has an RDB preamble. In that case we need to
      * load the RDB file and later continue loading the AOF tail. */
     char sig[]; /* "REDIS" */
     if (fread(sig,,,fp) !=  || memcmp(sig,"REDIS",) != ) {
         /* No RDB preamble, seek back at 0 offset. */
         if (fseek(fp,,SEEK_SET) == -) goto readerr;
     } else {
         /* RDB preamble. Pass loading the RDB functions. */
         rio rdb;

         serverLog(LL_NOTICE,"Reading RDB preamble from AOF file...");
         if (fseek(fp,,SEEK_SET) == -) goto readerr;
         rioInitWithFile(&rdb,fp);
         if (rdbLoadRio(&rdb,NULL,) != C_OK) {
             serverLog(LL_WARNING,"Error reading the RDB preamble of the AOF file, AOF loading aborted");
             goto readerr;
         } else {
             serverLog(LL_NOTICE,"Reading the remaining AOF tail...");
         }
     }

     /* Read the actual AOF file, in REPL format, command by command. */
     while() {
         int argc, j;
         unsigned long len;
         robj **argv;
         char buf[];
         sds argsds;
         struct redisCommand *cmd;

         /* Serve the clients from time to time */
         if (!(loops++ % )) {
             loadingProgress(ftello(fp));
             processEventsWhileBlocked();
         }

         if (fgets(buf,sizeof(buf),fp) == NULL) {
             if (feof(fp))
                 break;
             else
                 goto readerr;
         }
         if (buf[] != '*') goto fmterr;
         if (buf[] == '\0') goto readerr;
         argc = atoi(buf+);
         if (argc < ) goto fmterr;

         argv = zmalloc(sizeof(robj*)*argc);
         fakeClient->argc = argc;
         fakeClient->argv = argv;

         for (j = ; j < argc; j++) {
             if (fgets(buf,sizeof(buf),fp) == NULL) {
                 fakeClient->argc = j; /* Free up to j-1. */
                 freeFakeClientArgv(fakeClient);
                 goto readerr;
             }
             if (buf[] != '$') goto fmterr;
             len = strtol(buf+,NULL,);
             argsds = sdsnewlen(SDS_NOINIT,len);
             if (len && fread(argsds,len,,fp) == ) {
                 sdsfree(argsds);
                 fakeClient->argc = j; /* Free up to j-1. */
                 freeFakeClientArgv(fakeClient);
                 goto readerr;
             }
             argv[j] = createObject(OBJ_STRING,argsds);
             if (fread(buf,,,fp) == ) {
                 fakeClient->argc = j+; /* Free up to j. */
                 freeFakeClientArgv(fakeClient);
                 goto readerr; /* discard CRLF */
             }
         }

         /* Command lookup */
         cmd = lookupCommand(argv[]->ptr);
         if (!cmd) {
             serverLog(LL_WARNING,
                 "Unknown command '%s' reading the append only file",
                 (char*)argv[]->ptr);
             exit();
         }

         if (cmd == server.multiCommand) valid_before_multi = valid_up_to;

         /* Run the command in the context of a fake client */
         fakeClient->cmd = cmd;
         if (fakeClient->flags & CLIENT_MULTI &&
             fakeClient->cmd->proc != execCommand)
         {
             queueMultiCommand(fakeClient);
         } else {
             cmd->proc(fakeClient);
         }

         /* The fake client should not have a reply */
         serverAssert(fakeClient->bufpos ==  &&
                      listLength(fakeClient->reply) == );

         /* The fake client should never get blocked */
         serverAssert((fakeClient->flags & CLIENT_BLOCKED) == );

         /* Clean up. Command code may have changed argv/argc so we use the
          * argv/argc of the client instead of the local variables. */
         freeFakeClientArgv(fakeClient);
         fakeClient->cmd = NULL;
         if (server.aof_load_truncated) valid_up_to = ftello(fp);
     }

     /* This point can only be reached when EOF is reached without errors.
      * If the client is in the middle of a MULTI/EXEC, handle it as it was
      * a short read, even if technically the protocol is correct: we want
      * to remove the unprocessed tail and continue. */
     if (fakeClient->flags & CLIENT_MULTI) {
         serverLog(LL_WARNING,
             "Revert incomplete MULTI/EXEC transaction in AOF file");
         valid_up_to = valid_before_multi;
         goto uxeof;
     }

 loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */
     fclose(fp);
     freeFakeClient(fakeClient);
     server.aof_state = old_aof_state;
     stopLoading();
     aofUpdateCurrentSize();
     server.aof_rewrite_base_size = server.aof_current_size;
     return C_OK;

 readerr: /* Read error. If feof(fp) is true, fall through to unexpected EOF. */
     if (!feof(fp)) {
         if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */
         serverLog(LL_WARNING,"Unrecoverable error reading the append only file: %s", strerror(errno));
         exit();
     }

 uxeof: /* Unexpected AOF end of file. */
     if (server.aof_load_truncated) {
         serverLog(LL_WARNING,"!!! Warning: short read while loading the AOF file !!!");
         serverLog(LL_WARNING,"!!! Truncating the AOF at offset %llu !!!",
             (unsigned long long) valid_up_to);
         if (valid_up_to == - || truncate(filename,valid_up_to) == -) {
             if (valid_up_to == -) {
                 serverLog(LL_WARNING,"Last valid command offset is invalid");
             } else {
                 serverLog(LL_WARNING,"Error truncating the AOF file: %s",
                     strerror(errno));
             }
         } else {
             /* Make sure the AOF file descriptor points to the end of the
              * file after the truncate call. */
             if (server.aof_fd != - && lseek(server.aof_fd,,SEEK_END) == -) {
                 serverLog(LL_WARNING,"Can't seek the end of the AOF file: %s",
                     strerror(errno));
             } else {
                 serverLog(LL_WARNING,
                     "AOF loaded anyway because aof-load-truncated is enabled");
                 goto loaded_ok;
             }
         }
     }
     if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */
     serverLog(LL_WARNING,"Unexpected end of file reading the append only file. You can: 1) Make a backup of your AOF file, then use ./redis-check-aof --fix <filename>. 2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server.");
     exit();

 fmterr: /* Format error. */
     if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */
     serverLog(LL_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix <filename>");
     exit();
 }

loadAppendOnlyFile

RDB路线

RDB就像是一个redis快照

src/rdb.c  redis抽象出一个rio层，负责处理IO操作的，通过rio进行rdb文件恢复redis（快照保存的逆操作）。

 /* Like rdbLoadRio() but takes a filename instead of a rio stream. The
  * filename is open for reading and a rio stream object created in order
  * to do the actual loading. Moreover the ETA displayed in the INFO
  * output is initialized and finalized.
  *
  * If you pass an 'rsi' structure initialied with RDB_SAVE_OPTION_INIT, the
  * loading code will fiil the information fields in the structure. */
 int rdbLoad(char *filename, rdbSaveInfo *rsi) {
     FILE *fp;
     rio rdb;
     int retval;

     if ((fp = fopen(filename,"r")) == NULL) return C_ERR;
     startLoading(fp);
     rioInitWithFile(&rdb,fp);
     retval = rdbLoadRio(&rdb,rsi,);
     fclose(fp);
     stopLoading();
     return retval;
 }

rdbload

 /* Load an RDB file from the rio stream 'rdb'. On success C_OK is returned,
  * otherwise C_ERR is returned and 'errno' is set accordingly. */
 int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) {
     uint64_t dbid;
     int type, rdbver;
     redisDb *db = server.db+;
     char buf[];

     rdb->update_cksum = rdbLoadProgressCallback;
     rdb->max_processing_chunk = server.loading_process_events_interval_bytes;
     if (rioRead(rdb,buf,) == ) goto eoferr;
     buf[] = '\0';
     if (memcmp(buf,"REDIS",) != ) {
         serverLog(LL_WARNING,"Wrong signature trying to load DB from file");
         errno = EINVAL;
         return C_ERR;
     }
     rdbver = atoi(buf+);
     if (rdbver <  || rdbver > RDB_VERSION) {
         serverLog(LL_WARNING,"Can't handle RDB format version %d",rdbver);
         errno = EINVAL;
         return C_ERR;
     }

     /* Key-specific attributes, set by opcodes before the key type. */
     long long lru_idle = -, lfu_freq = -, expiretime = -, now = mstime();
     long long lru_clock = LRU_CLOCK();

     while() {
         robj *key, *val;

         /* Read type. */
         if ((type = rdbLoadType(rdb)) == -) goto eoferr;

         /* Handle special types. */
         if (type == RDB_OPCODE_EXPIRETIME) {
             /* EXPIRETIME: load an expire associated with the next key
              * to load. Note that after loading an expire we need to
              * load the actual type, and continue. */
             expiretime = rdbLoadTime(rdb);
             expiretime *= ;
             continue; /* Read next opcode. */
         } else if (type == RDB_OPCODE_EXPIRETIME_MS) {
             /* EXPIRETIME_MS: milliseconds precision expire times introduced
              * with RDB v3. Like EXPIRETIME but no with more precision. */
             expiretime = rdbLoadMillisecondTime(rdb,rdbver);
             continue; /* Read next opcode. */
         } else if (type == RDB_OPCODE_FREQ) {
             /* FREQ: LFU frequency. */
             uint8_t byte;
             if (rioRead(rdb,&byte,) == ) goto eoferr;
             lfu_freq = byte;
             continue; /* Read next opcode. */
         } else if (type == RDB_OPCODE_IDLE) {
             /* IDLE: LRU idle time. */
             uint64_t qword;
             if ((qword = rdbLoadLen(rdb,NULL)) == RDB_LENERR) goto eoferr;
             lru_idle = qword;
             continue; /* Read next opcode. */
         } else if (type == RDB_OPCODE_EOF) {
             /* EOF: End of file, exit the main loop. */
             break;
         } else if (type == RDB_OPCODE_SELECTDB) {
             /* SELECTDB: Select the specified database. */
             if ((dbid = rdbLoadLen(rdb,NULL)) == RDB_LENERR) goto eoferr;
             if (dbid >= (unsigned)server.dbnum) {
                 serverLog(LL_WARNING,
                     "FATAL: Data file was created with a Redis "
                     "server configured to handle more than %d "
                     "databases. Exiting\n", server.dbnum);
                 exit();
             }
             db = server.db+dbid;
             continue; /* Read next opcode. */
         } else if (type == RDB_OPCODE_RESIZEDB) {
             /* RESIZEDB: Hint about the size of the keys in the currently
              * selected data base, in order to avoid useless rehashing. */
             uint64_t db_size, expires_size;
             if ((db_size = rdbLoadLen(rdb,NULL)) == RDB_LENERR)
                 goto eoferr;
             if ((expires_size = rdbLoadLen(rdb,NULL)) == RDB_LENERR)
                 goto eoferr;
             dictExpand(db->dict,db_size);
             dictExpand(db->expires,expires_size);
             continue; /* Read next opcode. */
         } else if (type == RDB_OPCODE_AUX) {
             /* AUX: generic string-string fields. Use to add state to RDB
              * which is backward compatible. Implementations of RDB loading
              * are requierd to skip AUX fields they don't understand.
              *
              * An AUX field is composed of two strings: key and value. */
             robj *auxkey, *auxval;
             if ((auxkey = rdbLoadStringObject(rdb)) == NULL) goto eoferr;
             if ((auxval = rdbLoadStringObject(rdb)) == NULL) goto eoferr;

             if (((char*)auxkey->ptr)[] == '%') {
                 /* All the fields with a name staring with '%' are considered
                  * information fields and are logged at startup with a log
                  * level of NOTICE. */
                 serverLog(LL_NOTICE,"RDB '%s': %s",
                     (char*)auxkey->ptr,
                     (char*)auxval->ptr);
             } else if (!strcasecmp(auxkey->ptr,"repl-stream-db")) {
                 if (rsi) rsi->repl_stream_db = atoi(auxval->ptr);
             } else if (!strcasecmp(auxkey->ptr,"repl-id")) {
                 if (rsi && sdslen(auxval->ptr) == CONFIG_RUN_ID_SIZE) {
                     memcpy(rsi->repl_id,auxval->ptr,CONFIG_RUN_ID_SIZE+);
                     rsi->repl_id_is_set = ;
                 }
             } else if (!strcasecmp(auxkey->ptr,"repl-offset")) {
                 if (rsi) rsi->repl_offset = strtoll(auxval->ptr,NULL,);
             } else if (!strcasecmp(auxkey->ptr,"lua")) {
                 /* Load the script back in memory. */
                 if (luaCreateFunction(NULL,server.lua,auxval) == NULL) {
                     rdbExitReportCorruptRDB(
                         "Can't load Lua script from RDB file! "
                         "BODY: %s", auxval->ptr);
                 }
             } else {
                 /* We ignore fields we don't understand, as by AUX field
                  * contract. */
                 serverLog(LL_DEBUG,"Unrecognized RDB AUX field: '%s'",
                     (char*)auxkey->ptr);
             }

             decrRefCount(auxkey);
             decrRefCount(auxval);
             continue; /* Read type again. */
         } else if (type == RDB_OPCODE_MODULE_AUX) {
             /* This is just for compatibility with the future: we have plans
              * to add the ability for modules to store anything in the RDB
              * file, like data that is not related to the Redis key space.
              * Such data will potentially be stored both before and after the
              * RDB keys-values section. For this reason since RDB version 9,
              * we have the ability to read a MODULE_AUX opcode followed by an
              * identifier of the module, and a serialized value in "MODULE V2"
              * format. */
             uint64_t moduleid = rdbLoadLen(rdb,NULL);
             moduleType *mt = moduleTypeLookupModuleByID(moduleid);
             char name[];
             moduleTypeNameByID(name,moduleid);

             if (!rdbCheckMode && mt == NULL) {
                 /* Unknown module. */
                 serverLog(LL_WARNING,"The RDB file contains AUX module data I can't load: no matching module '%s'", name);
                 exit();
             } else if (!rdbCheckMode && mt != NULL) {
                 /* This version of Redis actually does not know what to do
                  * with modules AUX data... */
                 serverLog(LL_WARNING,"The RDB file contains AUX module data I can't load for the module '%s'. Probably you want to use a newer version of Redis which implements aux data callbacks", name);
                 exit();
             } else {
                 /* RDB check mode. */
                 robj *aux = rdbLoadCheckModuleValue(rdb,name);
                 decrRefCount(aux);
             }
         }

         /* Read key */
         if ((key = rdbLoadStringObject(rdb)) == NULL) goto eoferr;
         /* Read value */
         if ((val = rdbLoadObject(type,rdb)) == NULL) goto eoferr;
         /* Check if the key already expired. This function is used when loading
          * an RDB file from disk, either at startup, or when an RDB was
          * received from the master. In the latter case, the master is
          * responsible for key expiry. If we would expire keys here, the
          * snapshot taken by the master may not be reflected on the slave. */
         if (server.masterhost == NULL && !loading_aof && expiretime != - && expiretime < now) {
             decrRefCount(key);
             decrRefCount(val);
         } else {
             /* Add the new object in the hash table */
             dbAdd(db,key,val);

             /* Set the expire time if needed */
             if (expiretime != -) setExpire(NULL,db,key,expiretime);

             /* Set usage information (for eviction). */
             objectSetLRUOrLFU(val,lfu_freq,lru_idle,lru_clock);

             /* Decrement the key refcount since dbAdd() will take its
              * own reference. */
             decrRefCount(key);
         }

         /* Reset the state that is key-specified and is populated by
          * opcodes before the key, so that we start from scratch again. */
         expiretime = -;
         lfu_freq = -;
         lru_idle = -;
     }
     /* Verify the checksum if RDB version is >= 5 */
     if (rdbver >= ) {
         uint64_t cksum, expected = rdb->cksum;

         if (rioRead(rdb,&cksum,) == ) goto eoferr;
         if (server.rdb_checksum) {
             memrev64ifbe(&cksum);
             if (cksum == ) {
                 serverLog(LL_WARNING,"RDB file was saved with checksum disabled: no check performed.");
             } else if (cksum != expected) {
                 serverLog(LL_WARNING,"Wrong RDB checksum. Aborting now.");
                 rdbExitReportCorruptRDB("RDB CRC error");
             }
         }
     }
     return C_OK;

 eoferr: /* unexpected end of file is handled here with a fatal exit */
     serverLog(LL_WARNING,"Short read or OOM loading DB. Unrecoverable error, aborting now.");
     rdbExitReportCorruptRDB("Unexpected EOF reading RDB file");
     return C_ERR; /* Just to avoid warning */
 }

rdbLoadRio