react-native-update 10.34.2 → 10.34.3

package/harmony/pushy/src/main/cpp/HDiffPatch/libhsync/sync_make/sync_make.cpp

@@ -0,0 +1,521 @@
+//  sync_make.cpp
+//  sync_make
+//  Created by housisong on 2019-09-17.
+/*
+ The MIT License (MIT)
+ Copyright (c) 2019-2023 HouSisong
+ 
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+ 
+ The above copyright notice and this permission notice shall be
+ included in all copies of the Software.
+ 
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "sync_make.h"
+#include "sync_make_private.h"
+#include "sync_make_hash_clash.h"
+#include "sync_info_make.h"
+#include "match_in_new.h"
+#include "../../libParallel/parallel_channel.h"
+#include <vector>
+#include <atomic>
+using namespace hdiff_private;
+using namespace sync_private;
+namespace sync_private{
+
+struct _TCreateDatas {
+    const hpatch_TStreamInput*  newData;
+    hsync_TDictCompress*        compressPlugin;
+    hsync_THsynz*               hsynzPlugin;
+    TNewDataSyncInfo*           out_hsyni;
+    const hpatch_TStreamOutput* out_hsynz;
+    hpatch_StreamPos_t          curOutPos;
+    hpatch_checksumHandle       checkChecksum;
+    const _ICreateSync_by*      cs_by;
+};
+
+struct _TCompress{
+    inline _TCompress():compressPlugin(0),dictCompressHandle(0),kSyncBlockSize(0),
+                        kMaxCompressedSize(0),cmBuf(0),cmBufPos(0),blockEnd(0){}
+    inline _TCompress(hsync_TDictCompress* _compressPlugin,uint32_t kSyncBlockCount,
+                      uint32_t _kSyncBlockSize,TNewDataSyncInfo* out_hsyni)
+    :compressPlugin(0),dictCompressHandle(0),kSyncBlockSize(0),
+     kMaxCompressedSize(0),cmBuf(0),cmBufPos(0),blockEnd(0){ 
+        init(_compressPlugin,kSyncBlockCount,_kSyncBlockSize,out_hsyni); }
+    void init(hsync_TDictCompress* _compressPlugin,uint32_t kSyncBlockCount,
+              uint32_t _kSyncBlockSize,TNewDataSyncInfo* out_hsyni){
+        checkv(kSyncBlockSize==0);
+        compressPlugin=(kSyncBlockCount>0)?_compressPlugin:0;
+        kSyncBlockSize=_kSyncBlockSize;
+        kMaxCompressedSize=compressPlugin?(size_t)compressPlugin->maxCompressedSize(_kSyncBlockSize):0;
+        if (compressPlugin!=0){
+            dictCompressHandle=compressPlugin->dictCompressOpen(compressPlugin,kSyncBlockCount,_kSyncBlockSize);
+            checkv(dictCompressHandle!=0);
+            if (out_hsyni&&(out_hsyni->decompressInfoSize==0)&&compressPlugin->dictCompressInfo){
+                size_t decompressInfoSize=compressPlugin->dictCompressInfo(compressPlugin,dictCompressHandle,
+                                            &out_hsyni->decompressInfo[0],&out_hsyni->decompressInfo[0]+sizeof(out_hsyni->decompressInfo));
+                checkv(decompressInfoSize!=kDictCompressError);
+                if (decompressInfoSize!=kDictCompressCancel){
+                    assert(decompressInfoSize==(uint8_t)decompressInfoSize);
+                    checkv(decompressInfoSize<=sizeof(out_hsyni->decompressInfo));
+                    out_hsyni->decompressInfoSize=(uint8_t)decompressInfoSize;
+                }
+            }
+        }
+    }
+    inline ~_TCompress(){ if (dictCompressHandle) compressPlugin->dictCompressClose(compressPlugin,dictCompressHandle); }
+    inline size_t doCompress(uint32_t blockIndex,const TByte* in_data,size_t in_dataSize,size_t in_borderSize){
+        if (dictCompressHandle==0)
+            return 0;//not compress
+        blockEnd=blockIndex+1;
+        assert(cmBuf!=0);
+        hpatch_byte* curBuf=cmBuf+cmBufPos;
+        size_t result=compressPlugin->dictCompress(dictCompressHandle,blockIndex,curBuf,
+                                                   curBuf+kMaxCompressedSize,in_data,in_dataSize,in_borderSize);
+        checkv(result!=kDictCompressError);
+        if (result==kDictCompressCancel){
+            result=0; //cancel compress
+            memcpy(curBuf,in_data,in_dataSize);
+            cmBufPos+=in_dataSize;
+        }else{
+            checkv((result<=kMaxCompressedSize)
+                 &&(result<=2*kSyncBlockSize)); //for decompress memory size ctrl
+            cmBufPos+=result;
+        }
+        return result;
+    }
+    
+    inline void  resertDict(const hpatch_TStreamInput* data,hpatch_StreamPos_t curReadPos,size_t blockIndex){
+        if (dictCompressHandle==0)
+            return;//not compress
+        if (blockEnd==blockIndex)
+            return;//not need reset dict
+        size_t dictSize=0;
+        hpatch_byte* dictBuf=compressPlugin->getResetDictBuffer(dictCompressHandle,blockIndex,&dictSize);
+        if (dictSize==0) return;
+        checkv((dictBuf!=0)&&(dictSize<=curReadPos));
+        checkv(data->read(data,curReadPos-dictSize,dictBuf,dictBuf+dictSize));
+    }
+
+    hsync_TDictCompress* compressPlugin;
+    hsync_dictCompressHandle dictCompressHandle;
+    size_t          kSyncBlockSize;
+    size_t          kMaxCompressedSize;
+    hpatch_byte*    cmBuf;
+    size_t          cmBufPos;
+    uint32_t        blockEnd;
+};
+
+    struct TWorkBuf{
+        uint32_t            blockBegin;
+        uint32_t            blockEnd;
+        unsigned char*      buf;
+        size_t              in_borderSize;
+        size_t              cmSize;
+#if (_IS_USED_MULTITHREAD)
+        struct TWorkBuf*    next;
+#endif
+    };
+
+#if (_IS_USED_MULTITHREAD)
+namespace{
+
+    inline static void _insertNode(TWorkBuf** insertBuf,TWorkBuf* workBuf){
+        while ((*insertBuf)&&((*insertBuf)->blockBegin<workBuf->blockBegin))
+            insertBuf=&((*insertBuf)->next);
+        workBuf->next=*insertBuf;
+        *insertBuf=workBuf;
+    }
+    inline static TWorkBuf* _popNode(TWorkBuf** popBuf){
+        TWorkBuf* workBuf=*popBuf;
+        assert(workBuf!=0);
+        *popBuf=workBuf->next;
+        workBuf->next=0;
+        return workBuf;
+    }
+
+    struct TMt;
+    struct TThreadData{
+        CChecksum   checksumBlockData;
+        _TCompress  compress;
+    };
+    
+
+    struct TMt:public TMtByChannel{
+        _TCreateDatas& cd;
+        uint32_t bestWorkBlockCount;
+        CHLocker readLocker;
+        CHLocker writeLocker;
+        CHLocker workpLocker;
+        CHLocker checkLocker;
+        TWorkBuf* workFilished;
+        volatile uint32_t workBlockEnd;
+        volatile uint32_t savedBlockEnd;
+        CWaitValue nextCCheckBlocki;
+        CWaitValue nextReadBlocki;
+        const uint32_t kBlockCount;
+        std::vector<TThreadData> threadDataList;
+        inline TMt(size_t threadNum,_TCreateDatas& _cd,uint32_t _bestWorkBlockCount,
+                   hsync_TDictCompress* compressPlugin,uint32_t kSyncBlockCount,
+                   uint32_t kSyncBlockSize,TNewDataSyncInfo* out_hsyni)
+        :cd(_cd),bestWorkBlockCount(_bestWorkBlockCount),
+         workFilished(0),workBlockEnd(0),savedBlockEnd(0),
+         nextCCheckBlocki(0),nextReadBlocki(0),kBlockCount(kSyncBlockCount){
+            threadDataList.resize(threadNum);
+            for (size_t i=0;i<threadNum;++i){
+                TThreadData& td=threadDataList[i];
+                td.checksumBlockData.init(out_hsyni->strongChecksumPlugin,false);
+                td.compress.init(compressPlugin,kSyncBlockCount,kSyncBlockSize,out_hsyni);
+            }
+        }
+        inline TWorkBuf* getWorkPart(){
+            if (kBlockCount==workBlockEnd) return 0;
+            TWorkBuf* workBuf=(TWorkBuf*)read_chan.accept(true);
+            if ((workBuf==0)||(kBlockCount==workBlockEnd)) return 0;
+
+            CAutoLocker _auto_locker(workpLocker.locker);
+            if (kBlockCount==workBlockEnd) return 0;
+            workBuf->blockBegin=workBlockEnd;
+            workBlockEnd+=bestWorkBlockCount;
+            workBlockEnd=(workBlockEnd<=kBlockCount)?workBlockEnd:kBlockCount;
+            workBuf->blockEnd=workBlockEnd;
+            return workBuf;
+        }
+    };
+}
+#endif
+
+
+static void _saveCompressedData(_TCreateDatas& cd,TWorkBuf* workData,void* _mt=0){
+    const bool is_hsynz_readed_data=(cd.out_hsynz&&cd.hsynzPlugin&&cd.hsynzPlugin->hsynz_readed_data);
+    const uint32_t kSyncBlockSize=cd.out_hsyni->kSyncBlockSize;
+#if (_IS_USED_MULTITHREAD)
+  TMt* mt=(TMt*)_mt;
+  TWorkBuf* delWorkBufList=0;
+  {//in lock
+    CAutoLocker _autoLocker(mt?mt->writeLocker.locker:0);
+#endif //_IS_USED_MULTITHREAD
+
+    while(true){
+#if (_IS_USED_MULTITHREAD)
+        if (mt){//save data order by blockBegin
+            if ((workData!=0)&&(workData->blockBegin!=mt->savedBlockEnd)){
+                _insertNode(&mt->workFilished,workData);
+                workData=0;
+            }
+            if ((workData==0)&&(mt->workFilished!=0)&&(mt->workFilished->blockBegin==mt->savedBlockEnd))
+                workData=_popNode(&mt->workFilished);
+        }
+#endif 
+        if (workData==0) break;
+
+        size_t backZeroLen=0;
+        const uint32_t kSyncBlockCount=(workData->blockEnd-workData->blockBegin);
+        const size_t dataLens=(size_t)kSyncBlockSize*kSyncBlockCount;
+        {
+            const hpatch_StreamPos_t curReadPos=(hpatch_StreamPos_t)workData->blockBegin*kSyncBlockSize;
+            if (curReadPos+dataLens>cd.newData->streamSize)
+                backZeroLen=(size_t)(curReadPos+dataLens-cd.newData->streamSize);
+        }
+        if (is_hsynz_readed_data)
+            cd.hsynzPlugin->hsynz_readed_data(cd.hsynzPlugin,workData->buf,dataLens-backZeroLen);
+        if (cd.out_hsynz){
+            size_t writeSize=workData->cmSize;
+            if (writeSize>0){
+                hpatch_byte* cmBuf=workData->buf+dataLens+workData->in_borderSize;
+                writeStream(cd.out_hsynz,cd.curOutPos,cmBuf,writeSize);
+            }else{
+                writeSize=dataLens-backZeroLen;
+                writeStream(cd.out_hsynz,cd.curOutPos,workData->buf,writeSize);
+            }
+        }
+#if (_IS_USED_MULTITHREAD)
+        if (mt){
+            assert(mt->savedBlockEnd==workData->blockBegin);
+            mt->savedBlockEnd=workData->blockEnd;
+            workData->blockBegin=0;
+            _insertNode(&delWorkBufList,workData);
+        }
+#endif
+        workData=0;
+    }
+#if (_IS_USED_MULTITHREAD)
+  }//out lock
+    if (mt){
+        while (delWorkBufList){
+            workData=_popNode(&delWorkBufList);
+            if (!mt->read_chan.send(workData,true)){ mt->on_error(); break; }
+        }
+    }
+#endif
+}
+
+static void _create_sync_data_part(_TCreateDatas& cd,TWorkBuf* workData,
+                                   CChecksum& checksumBlockData,_TCompress& compress,void* _mt=0){
+    TNewDataSyncInfo*   out_hsyni=cd.out_hsyni;
+    const uint32_t      kSyncBlockSize=out_hsyni->kSyncBlockSize;
+    const uint32_t      kSyncBlockCount=(workData->blockEnd-workData->blockBegin);
+    hpatch_StreamPos_t curReadPos=(hpatch_StreamPos_t)workData->blockBegin*kSyncBlockSize;
+    const bool isCCheckByOrder=_mt&&cd.out_hsyni->isNotCChecksumNewMTParallel;
+
+    size_t backZeroLen=0;
+    const size_t dataLens=(size_t)kSyncBlockSize*kSyncBlockCount;
+    compress.cmBuf=workData->buf+dataLens+workData->in_borderSize;
+    compress.cmBufPos=0;
+    {//read data
+        if (curReadPos+dataLens>cd.newData->streamSize){
+            backZeroLen=(size_t)(curReadPos+dataLens-cd.newData->streamSize);
+            assert(backZeroLen<kSyncBlockSize);
+        }
+        {
+            size_t readLen=dataLens-backZeroLen+workData->in_borderSize;
+            if (curReadPos+readLen>cd.newData->streamSize)
+                    readLen=(size_t)(cd.newData->streamSize-curReadPos);
+        #if (_IS_USED_MULTITHREAD)
+            TMt* mt=(TMt*)_mt;
+            if (isCCheckByOrder){
+                if (!mt->nextReadBlocki.wait(workData->blockBegin,mt->is_on_error_by,mt))
+                    return;
+            }
+            CAutoLocker _autoLocker((mt&&(!isCCheckByOrder))?mt->readLocker.locker:0);
+            if (mt)
+                compress.resertDict(cd.newData,curReadPos,workData->blockBegin);
+        #endif
+            checkv(cd.newData->read(cd.newData,curReadPos,workData->buf,workData->buf+readLen));
+        #if (_IS_USED_MULTITHREAD)
+            if (isCCheckByOrder)
+                mt->nextReadBlocki.store(workData->blockEnd);
+        #endif
+        }
+        if (backZeroLen)
+            memset(workData->buf+dataLens-backZeroLen,0,backZeroLen);
+    }
+
+    if (cd.out_hsyni->isNotCChecksumNewMTParallel){//check new data by order(like single thread), & not used checksumBlockData  
+    #if (_IS_USED_MULTITHREAD)
+        TMt* mt=(TMt*)_mt;
+        if (mt){
+            if (!mt->nextCCheckBlocki.wait(workData->blockBegin,mt->is_on_error_by,mt))
+                return;
+        }
+    #endif
+        cd.cs_by->checkChecksumAppendData(cd.out_hsyni->savedNewDataCheckChecksum,workData->blockBegin,
+                                          out_hsyni->fileChecksumPlugin,cd.checkChecksum,0,workData->buf,dataLens-backZeroLen);
+    #if (_IS_USED_MULTITHREAD)
+        if (mt)
+            mt->nextCCheckBlocki.store(workData->blockEnd);
+    #endif
+    }
+
+    hpatch_byte* dataBuf=workData->buf;
+    for (uint32_t i=workData->blockBegin;i<workData->blockEnd;++i,dataBuf+=kSyncBlockSize,curReadPos+=kSyncBlockSize) {
+        size_t srcDataLen=(i+1<workData->blockEnd)?kSyncBlockSize:kSyncBlockSize-backZeroLen;
+        size_t cur_borderSize=workData->in_borderSize;
+        if (curReadPos+srcDataLen+cur_borderSize>cd.newData->streamSize)
+            cur_borderSize=(size_t)(cd.newData->streamSize-(curReadPos+srcDataLen));
+        //compress
+        size_t compressedSize=compress.doCompress(i,dataBuf,srcDataLen,cur_borderSize);
+        checkv(compressedSize==(uint32_t)compressedSize);
+        if (out_hsyni->savedSizes) //save compressedSize
+            out_hsyni->savedSizes[i]=(uint32_t)compressedSize;
+
+        const uint64_t rollHash=cd.cs_by->roll_hash_start(dataBuf,kSyncBlockSize);
+        //strong hash
+        checksumBlockData.appendBegin();
+        checksumBlockData.append(dataBuf,dataBuf+kSyncBlockSize);
+        checksumBlockData.appendEnd();
+        {//save hash
+            const uint64_t _partRollHash=cd.cs_by->toSavedPartRollHash(rollHash,out_hsyni->savedRollHashBits);
+            writeRollHashBytes(out_hsyni->rollHashs+i*out_hsyni->savedRollHashByteSize,
+                               _partRollHash,out_hsyni->savedRollHashByteSize);
+            if (!cd.out_hsyni->isNotCChecksumNewMTParallel){
+            #if (_IS_USED_MULTITHREAD)
+                TMt* mt=(TMt*)_mt;
+                CAutoLocker _autoLocker(mt?mt->checkLocker.locker:0);
+            #endif
+                cd.cs_by->checkChecksumAppendData(cd.out_hsyni->savedNewDataCheckChecksum,i,
+                                                  out_hsyni->fileChecksumPlugin,cd.checkChecksum,
+                                                  checksumBlockData.checksum.data(),0,0);
+            }
+            toPartChecksum(checksumBlockData.checksum.data(),out_hsyni->savedStrongChecksumBits,
+                           checksumBlockData.checksum.data(),out_hsyni->strongChecksumPlugin->checksumByteSize());
+            memcpy(out_hsyni->partChecksums+i*out_hsyni->savedStrongChecksumByteSize,
+                   checksumBlockData.checksum.data(),out_hsyni->savedStrongChecksumByteSize);
+        }
+    }
+
+    workData->cmSize=compress.cmBufPos;
+    _saveCompressedData(cd,workData,_mt);
+}
+
+#if (_IS_USED_MULTITHREAD)
+static void _create_sync_data_mt(int threadIndex,void* _mt){
+    TMt& mt=*(TMt*)_mt;
+    TMtByChannel::TAutoThreadEnd __auto_thread_end(mt);
+    TThreadData& td=mt.threadDataList[threadIndex];
+    while (true){
+        TWorkBuf* workData=mt.getWorkPart();
+        if (workData==0)
+            break; //work finish
+        try{
+            mt.cd.cs_by->create_sync_data_part(mt.cd,workData,td.checksumBlockData,td.compress,&mt);
+        }catch(...){
+            mt.on_error();
+        }
+    }
+}
+#endif
+
+void _create_sync_data_by(_ICreateSync_by* cs_by,TNewDataSyncInfo* newSyncInfo,
+                          const hpatch_TStreamInput* newData,const hpatch_TStreamOutput* out_hsynz,
+                          hsync_TDictCompress* compressPlugin,hsync_THsynz* hsynzPlugin,size_t threadNum){
+    const uint32_t kSyncBlockSize= newSyncInfo->kSyncBlockSize;
+    hpatch_TChecksum* checksumPlugin=newSyncInfo->fileChecksumPlugin;
+    checkv(kSyncBlockSize>=_kSyncBlockSize_min_limit);
+    if (compressPlugin) checkv(out_hsynz!=0);
+    hsync_THsynz _hsynzPlugin;
+    if ((hsynzPlugin==0)&&((compressPlugin!=0)||newSyncInfo->isDirSyncInfo)){
+        getHsynzPluginDefault(&_hsynzPlugin);
+        hsynzPlugin=&_hsynzPlugin;
+    }
+    {//check checksumByteSize
+        const size_t checksumByteSize=checksumPlugin->checksumByteSize();
+        checkv((checksumByteSize<=_kStrongChecksumByteSize_max_limit)&&(checksumByteSize>=kStrongChecksumByteSize_min));
+    }
+
+    assert(cs_by->create_sync_data_part==0);
+    cs_by->create_sync_data_part=_create_sync_data_part;
+    CChecksum      _checkChecksum(checksumPlugin,false);
+    _TCreateDatas  createDatas;
+    createDatas.newData=newData;
+    createDatas.compressPlugin=compressPlugin;
+    createDatas.out_hsyni=newSyncInfo;
+    createDatas.out_hsynz=out_hsynz;
+    createDatas.hsynzPlugin=hsynzPlugin;
+    createDatas.curOutPos=0;
+    createDatas.checkChecksum=_checkChecksum._handle;
+    createDatas.cs_by=cs_by;
+    const bool is_hsynzPlugin=(out_hsynz&&hsynzPlugin);
+    if (is_hsynzPlugin){
+        createDatas.curOutPos=hsynzPlugin->hsynz_write_head(hsynzPlugin,out_hsynz,createDatas.curOutPos,newSyncInfo->isDirSyncInfo,
+                                                            newData->streamSize,kSyncBlockSize,checksumPlugin,compressPlugin);
+        newSyncInfo->newSyncDataOffsert=createDatas.curOutPos;
+    }
+    const uint32_t kBlockCount=(uint32_t)getSyncBlockCount(newData->streamSize,kSyncBlockSize);
+    newSyncInfo->dictSize=compressPlugin?compressPlugin->limitDictSizeByData(compressPlugin,kBlockCount,kSyncBlockSize):0;
+
+    cs_by->checkChecksumInit(createDatas.out_hsyni->savedNewDataCheckChecksum,
+                             checksumPlugin->checksumByteSize());
+    
+    const size_t in_borderSize=(compressPlugin&&compressPlugin->getDictCompressBorder)?compressPlugin->getDictCompressBorder():0;
+    const hpatch_StreamPos_t kMaxCompressedSize=compressPlugin?compressPlugin->maxCompressedSize(kSyncBlockSize):0;
+    const size_t _kBestWorkBufSize=2*(1<<20);
+    uint32_t bestWorkBlockCount=(uint32_t)((_kBestWorkBufSize+kSyncBlockSize+kMaxCompressedSize-1)
+                                          /(kSyncBlockSize+kMaxCompressedSize));
+    bestWorkBlockCount=(bestWorkBlockCount<=kBlockCount)?bestWorkBlockCount:kBlockCount;
+ #if (_IS_USED_MULTITHREAD)
+    threadNum=std::min(threadNum,(compressPlugin?threadNum:4));
+    while ((hpatch_StreamPos_t)bestWorkBlockCount*threadNum>kBlockCount)
+        --threadNum;
+    if ((threadNum>1)&&(bestWorkBlockCount<=kBlockCount/2)){
+        if (compressPlugin)
+            bestWorkBlockCount=(uint32_t)compressPlugin->getBestWorkBlockCount(compressPlugin,kBlockCount,kSyncBlockSize,bestWorkBlockCount);
+        const uint32_t maxWorkBlockCount=(uint32_t)((kBlockCount+threadNum-1)/threadNum);
+        bestWorkBlockCount=(bestWorkBlockCount<=maxWorkBlockCount)?bestWorkBlockCount:maxWorkBlockCount;
+        
+        const size_t kWorkBufCount=threadNum+(threadNum/4)+1;
+        const size_t kWorkMemBufSize=(kSyncBlockSize+(size_t)kMaxCompressedSize)*bestWorkBlockCount+in_borderSize;
+        TAutoMem    membuf((sizeof(TWorkBuf)+kWorkMemBufSize)*kWorkBufCount);
+
+        TMt mt(threadNum,createDatas,bestWorkBlockCount,compressPlugin,
+               kBlockCount,kSyncBlockSize,createDatas.out_hsyni);
+        { //mem buf
+            TWorkBuf* workBufList=(TWorkBuf*)membuf.data();
+            hpatch_byte* workMemBuf=(hpatch_byte*)(workBufList+kWorkBufCount);
+            memset(workBufList,0,sizeof(TWorkBuf)*kWorkBufCount);
+            for (size_t i=0;i<kWorkBufCount;++i,workBufList++,workMemBuf+=kWorkMemBufSize){
+                workBufList->buf=workMemBuf;
+                workBufList->in_borderSize=in_borderSize;
+                checkv(mt.read_chan.send(workBufList,true));
+            }
+        }
+
+        checkv(mt.start_threads((int)threadNum,_create_sync_data_mt,&mt,true));
+        mt.wait_all_thread_end();
+        checkv(!mt.is_on_error());
+        checkv(mt.savedBlockEnd==mt.kBlockCount);
+    }else
+#endif
+    {
+        TAutoMem    membuf((kSyncBlockSize+(size_t)kMaxCompressedSize)*bestWorkBlockCount+in_borderSize);
+        CChecksum   checksumBlockData(newSyncInfo->strongChecksumPlugin,false);
+        _TCompress  compress(compressPlugin,kBlockCount,kSyncBlockSize,createDatas.out_hsyni);
+        TWorkBuf    workData={0};
+        workData.buf=membuf.data();
+        workData.in_borderSize=in_borderSize;
+        for (uint32_t ib=0; ib<kBlockCount; ib+=bestWorkBlockCount){
+            workData.blockBegin=ib;
+            workData.blockEnd=(ib+bestWorkBlockCount<=kBlockCount)?ib+bestWorkBlockCount:kBlockCount;
+            cs_by->create_sync_data_part(createDatas,&workData,checksumBlockData,compress,0);
+        }
+    }
+    cs_by->checkChecksumEndTo(createDatas.out_hsyni->savedNewDataCheckChecksum,createDatas.out_hsyni->savedNewDataCheckChecksum,
+                              checksumPlugin,createDatas.checkChecksum);
+    if (is_hsynzPlugin){
+        createDatas.curOutPos=hsynzPlugin->hsynz_write_foot(hsynzPlugin,out_hsynz,createDatas.curOutPos,
+                                createDatas.out_hsyni->savedNewDataCheckChecksum,checksumPlugin->checksumByteSize());
+    }
+    newSyncInfo->newSyncDataSize=createDatas.curOutPos;
+}
+
+void _private_create_sync_data(TNewDataSyncInfo* newSyncInfo, const hpatch_TStreamInput*  newData,
+                               const hpatch_TStreamOutput* out_hsyni, const hpatch_TStreamOutput* out_hsynz,
+                               hsync_TDictCompress* compressPlugin, hsync_THsynz* hsynzPlugin,size_t threadNum){
+    _ICreateSync_by cs_by={0};
+    cs_by.roll_hash_start=roll_hash_start;
+    cs_by.toSavedPartRollHash=toSavedPartRollHash;
+    cs_by.checkChecksumInit=checkChecksumInit;
+    cs_by.checkChecksumAppendData=checkChecksumAppendData;
+    cs_by.checkChecksumEndTo=checkChecksumEndTo;
+    _create_sync_data_by(&cs_by,newSyncInfo,newData,out_hsynz,compressPlugin,hsynzPlugin,threadNum);
+    matchNewDataInNew(newSyncInfo);
+    TNewDataSyncInfo_saveTo(newSyncInfo,out_hsyni,compressPlugin);//save to out_hsyni
+}
+
+}//namespace sync_private
+
+
+void create_sync_data(const hpatch_TStreamInput*  newData,
+                      const hpatch_TStreamOutput* out_hsyni,
+                      const hpatch_TStreamOutput* out_hsynz,
+                      hpatch_TChecksum*           strongChecksumPlugin,
+                      hsync_TDictCompress*  compressPlugin,hsync_THsynz* hsynzPlugin,
+                      uint32_t kSyncBlockSize,size_t kSafeHashClashBit,size_t threadNum){
+    CNewDataSyncInfo newSyncInfo(strongChecksumPlugin,compressPlugin,
+                                 newData->streamSize,kSyncBlockSize,kSafeHashClashBit);
+    _private_create_sync_data(&newSyncInfo,newData,out_hsyni,out_hsynz,compressPlugin,hsynzPlugin,threadNum);
+}
+
+void create_sync_data(const hpatch_TStreamInput*  newData,
+                      const hpatch_TStreamOutput* out_hsyni,
+                      hpatch_TChecksum*           strongChecksumPlugin,
+                      hsync_TDictCompress*  compressPlugin,
+                      uint32_t kSyncBlockSize,size_t kSafeHashClashBit,size_t threadNum){
+    create_sync_data(newData,out_hsyni,0,strongChecksumPlugin,compressPlugin,0,
+                     kSyncBlockSize,kSafeHashClashBit,threadNum);
+}

package/harmony/pushy/src/main/cpp/HDiffPatch/libhsync/sync_make/sync_make.h

@@ -0,0 +1,69 @@
+//  sync_make.h
+//  sync_make
+//  Created by housisong on 2019-09-17.
+/*
+ The MIT License (MIT)
+ Copyright (c) 2019-2023 HouSisong
+ 
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+ 
+ The above copyright notice and this permission notice shall be
+ included in all copies of the Software.
+ 
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef hsync_make_h
+#define hsync_make_h
+#include "../../libHDiffPatch/HPatch/checksum_plugin.h"
+#include "../../libHDiffPatch/HDiff/diff_types.h"
+#include "../sync_client/sync_client_type.h"
+#include "dict_compress_plugin.h"
+#include "hsynz_plugin.h"
+
+static const uint32_t kSyncBlockSize_default   = 1024*2;
+static const uint32_t kSyncBlockSize_min       = 128; // >= kSyncBlockSize_min_limit
+static const size_t   kSafeHashClashBit_default= 24;
+
+//create out_hsyni
+//  all clients need download out_hsyni, and dowload part of 
+//    newData which is not found in client's oldData;
+//  client's oldData can be different for different clients;
+//  out_hsyni will be loaded into memory when client sync_patch;
+//  out_hsyni's size becomes smaller when kSyncBlockSize increases,
+//    but the part of newData's size that need download becomes larger;
+void create_sync_data(const hpatch_TStreamInput*  newData,
+                      const hpatch_TStreamOutput* out_hsyni,
+                      hpatch_TChecksum*           strongChecksumPlugin,
+                      hsync_TDictCompress* compressPlugin=0,
+                      uint32_t kSyncBlockSize=kSyncBlockSize_default,
+                      size_t kSafeHashClashBit=kSafeHashClashBit_default,
+                      size_t threadNum=1);
+
+//create out_hsyni & out_hsynz
+// out_hsynz: out compressed newData by compressPlugin
+//   client download compressed part of newData from out_hsynz;
+void create_sync_data(const hpatch_TStreamInput*  newData,
+                      const hpatch_TStreamOutput* out_hsyni,
+                      const hpatch_TStreamOutput* out_hsynz,
+                      hpatch_TChecksum*           strongChecksumPlugin,
+                      hsync_TDictCompress*        compressPlugin,
+                      hsync_THsynz* hsynzPlugin=0,
+                      uint32_t kSyncBlockSize=kSyncBlockSize_default,
+                      size_t kSafeHashClashBit=kSafeHashClashBit_default,
+                      size_t threadNum=1);
+
+#endif // hsync_make_h