react-native-update 10.34.2 → 10.34.3

package/harmony/pushy/src/main/cpp/HDiffPatch/libhsync/sync_client/match_in_old.cpp

@@ -0,0 +1,332 @@
+//  match_in_old.cpp
+//  sync_client
+//  Created by housisong on 2019-09-22.
+/*
+ The MIT License (MIT)
+ Copyright (c) 2019-2023 HouSisong
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "match_in_old.h"
+#include "match_in_types.h"
+#include "sync_client_private.h"
+namespace sync_private{
+
+    #define check(value,info) { if (!(value)) { throw std::runtime_error(info); } }
+    #define checkv(value)     check(value,"check "#value" error!")
+
+    TStreamDataCache_base::TStreamDataCache_base(const hpatch_TStreamInput* baseStream,hpatch_StreamPos_t streamRollBegin,
+                          hpatch_StreamPos_t streamRollEnd,size_t backupSize,size_t backZeroLen,uint32_t kSyncBlockSize,
+                          hpatch_TChecksum* strongChecksumPlugin,void* _readLocker)
+    :m_baseStream(baseStream),m_readedPos(streamRollBegin),m_streamRollEnd(streamRollEnd),
+    m_kBackupSize(backupSize),m_kBackZeroLen(backZeroLen),
+    m_strongChecksum_buf(0),m_cur(0),m_kSyncBlockSize(kSyncBlockSize),m_checksumByteSize(0),
+    m_strongChecksumPlugin(strongChecksumPlugin),m_checksumHandle(0),m_readLocker(_readLocker){
+        size_t cacheSize=(size_t)kSyncBlockSize*2+m_kBackupSize;
+        check((((cacheSize-m_kBackupSize)>>1)==kSyncBlockSize),"TStreamDataCache_base mem error!");
+        cacheSize=(cacheSize>=kBestReadSize)?cacheSize:kBestReadSize;
+        hpatch_StreamPos_t maxDataSize=(streamRollPosEnd()-streamRollBegin);
+        if (cacheSize>maxDataSize) cacheSize=(size_t)maxDataSize;
+#if (_IS_USED_MULTITHREAD)
+        if (_readLocker)
+            cacheSize=(size_t)maxDataSize;
+#endif
+
+        m_checksumHandle=strongChecksumPlugin->open(strongChecksumPlugin);
+        checkv(m_checksumHandle!=0);
+        m_checkChecksum=strongChecksumPlugin->open(strongChecksumPlugin);
+        checkv(m_checkChecksum!=0);
+        m_checksumByteSize=(uint32_t)m_strongChecksumPlugin->checksumByteSize();
+        m_cache.realloc(cacheSize+m_checksumByteSize+m_checksumByteSize);
+        m_cache.reduceSize(cacheSize);
+        m_strongChecksum_buf=m_cache.data_end();
+        m_cur=m_cache.data_end();
+        _cache();
+    }
+    TStreamDataCache_base::~TStreamDataCache_base(){
+        if (m_checksumHandle)
+            m_strongChecksumPlugin->close(m_strongChecksumPlugin,m_checksumHandle);
+        if (m_checkChecksum)
+            m_strongChecksumPlugin->close(m_strongChecksumPlugin,m_checkChecksum);
+    }
+    void TStreamDataCache_base::_cache(){
+        // all:[        streamDataSize      +     backZeroLen    ]
+        //                 ^                         ^
+        //          streamRollBegin            streamRollEnd
+        //self:            [                          ]
+        //                       ^             ^
+        //                                 readedPos
+        //
+        //cache:             [         | kBackupSize ]
+        //                        ^
+        //                       cur
+        if (isRollEnded()) return;
+        const hpatch_StreamPos_t rollPosEnd=streamRollPosEnd();
+        if (m_readedPos>=rollPosEnd) { _setRollEnded(); return; }//set end tag
+
+        size_t needLen=m_cur-m_cache.data();
+        if (m_readedPos+needLen>rollPosEnd)
+            needLen=(size_t)(rollPosEnd-m_readedPos);
+        if (m_cur-needLen+m_kSyncBlockSize+m_kBackupSize>m_cache.data_end()) { _setRollEnded(); return; } //set end tag
+
+        memmove(m_cur-needLen,m_cur,m_cache.data_end()-m_cur);
+        size_t readLen=needLen;
+        if (m_readedPos+readLen>m_baseStream->streamSize){
+            if (m_readedPos<m_baseStream->streamSize)
+                readLen=(size_t)(m_baseStream->streamSize-m_readedPos);
+            else
+                readLen=0;
+        }
+        if (readLen>0){
+            TByte* buf=m_cache.data_end()-needLen;
+#if (_IS_USED_MULTITHREAD)
+            CAutoLocker _autoLocker((HLocker)m_readLocker);
+#endif
+            check(m_baseStream->read(m_baseStream,m_readedPos,buf,buf+readLen),
+                  "TStreamDataRoll read streamData error!");
+        }
+        size_t zerolen=needLen-readLen;
+        if (zerolen>0)
+            memset(m_cache.data_end()-zerolen,0,zerolen);
+        m_readedPos+=needLen;
+        m_cur-=needLen;
+    }
+
+    const TByte* TStreamDataCache_base::_calcPartStrongChecksum(const TByte* buf,size_t bufSize,size_t outPartBits){
+        TByte* strongChecksum=m_strongChecksum_buf+m_checksumByteSize;
+        m_strongChecksumPlugin->begin(m_checksumHandle);
+        m_strongChecksumPlugin->append(m_checksumHandle,buf,buf+bufSize);
+        m_strongChecksumPlugin->end(m_checksumHandle,strongChecksum,strongChecksum+m_checksumByteSize);
+        toPartChecksum(m_strongChecksum_buf,outPartBits,strongChecksum,m_checksumByteSize);
+        return m_strongChecksum_buf;
+    }
+
+    const uint32_t* getSortedIndexsTable(TAutoMem& _mem_table,const TNewDataSyncInfo* newSyncInfo,
+                                         const uint32_t* sorted_newIndexs,unsigned int* out_kTableHashShlBit){//optimize for std::equal_range
+        unsigned int& kTableHashShlBit=*out_kTableHashShlBit;
+        const uint32_t kBlockCount=(uint32_t)TNewDataSyncInfo_blockCount(newSyncInfo);
+        const uint32_t kMatchBlockCount=kBlockCount-newSyncInfo->samePairCount;
+        const size_t needRollHashByteSize=newSyncInfo->savedRollHashByteSize*(newSyncInfo->isSeqMatch?2:1);
+        unsigned int kTableBit =getBetterCacheBlockTableBit(kMatchBlockCount);
+        const size_t needRollHashBits=newSyncInfo->savedRollHashBits*(newSyncInfo->isSeqMatch?2:1);
+        if (kTableBit>needRollHashBits) kTableBit=(unsigned int)needRollHashBits;
+        kTableHashShlBit=(int)(needRollHashBits-kTableBit);
+        _mem_table.realloc((size_t)sizeof(uint32_t)*((1<<kTableBit)+1));
+        uint32_t* sorted_newIndexs_table=(uint32_t*)_mem_table.data();
+        {
+            TIndex_comp0 icomp0(newSyncInfo->rollHashs,newSyncInfo->savedRollHashByteSize,newSyncInfo->isSeqMatch);
+            const uint32_t* pos=sorted_newIndexs;
+            uint8_t part[sizeof(tm_roll_uint)]={0};
+            for (uint32_t i=0; i<((uint32_t)1<<kTableBit); ++i) {
+                tm_roll_uint digest=((tm_roll_uint)i)<<kTableHashShlBit;
+                writeRollHashBytes(part,digest,needRollHashByteSize);
+                TIndex_comp0::TDigest digest_value(part);
+                pos=std::lower_bound(pos,sorted_newIndexs+kMatchBlockCount,digest_value,icomp0);
+                sorted_newIndexs_table[i]=(uint32_t)(pos-sorted_newIndexs);
+            }
+            sorted_newIndexs_table[((size_t)1<<kTableBit)]=kMatchBlockCount;
+        }
+        return sorted_newIndexs_table;
+    }
+
+
+static inline hpatch_StreamPos_t _indexMapTo(uint32_t i){
+    return (hpatch_StreamPos_t)(kBlockType_needSync-1)-i; }
+static inline uint32_t _indexMapFrom(hpatch_StreamPos_t pos){
+    return (uint32_t)((kBlockType_needSync-1)-pos); }
+
+typedef volatile hpatch_StreamPos_t volStreamPos_t;
+
+bool _matchRange(hpatch_StreamPos_t* out_newBlockDataInOldPoss,const uint32_t* range_begin,const uint32_t* range_end,
+                 TStreamDataCache_base& oldData,const TNewDataSyncInfo* newSyncInfo,hpatch_StreamPos_t kMinRevSameIndex,void* _mt){
+    const TByte* oldPartStrongChecksum=0;
+    const size_t savedStrongChecksumByteSize=newSyncInfo->savedStrongChecksumByteSize;
+    bool isMatched=false;
+    int hitOutLimit=kMatchHitOutLimit;
+    do {
+        size_t newBlockIndex=*range_begin;
+        volStreamPos_t* pNewBlockDataInOldPos=&out_newBlockDataInOldPoss[newBlockIndex];
+        hpatch_StreamPos_t newBlockOldPosBack=*pNewBlockDataInOldPos;   
+        if (newBlockOldPosBack>=kMinRevSameIndex){
+            if (oldPartStrongChecksum==0)
+                oldPartStrongChecksum=oldData.calcPartStrongChecksum(newSyncInfo->savedStrongChecksumBits);
+            const TByte* newPairStrongChecksum=newSyncInfo->partChecksums+newBlockIndex*savedStrongChecksumByteSize;
+            if (0==memcmp(oldPartStrongChecksum,newPairStrongChecksum,savedStrongChecksumByteSize)){
+                isMatched=true;
+                hpatch_StreamPos_t curPos=oldData.curStreamPos();
+                {
+#if (_IS_USED_MULTITHREAD)
+                    TMt* mt=(TMt*)_mt;
+                    CAutoLocker _autoLocker(mt?mt->writeLocker.locker:0);
+                    newBlockOldPosBack=*pNewBlockDataInOldPos;
+                    if (newBlockOldPosBack<kMinRevSameIndex){// other thread done
+                        if ((--hitOutLimit)<=0) break;
+                    }else
+#endif
+                    {
+                        while(true){ //hit
+                            (*pNewBlockDataInOldPos)=curPos;
+                            if (!newSyncInfo->isNotCChecksumNewMTParallel) //note: checkChecksumAppendData() is only for hsynz
+                                checkChecksumAppendData(newSyncInfo->savedNewDataCheckChecksum,(uint32_t)newBlockIndex,
+                                                        oldData.strongChecksumPlugin(),oldData.checkChecksum(),
+                                                        oldData.strongChecksum(),0,0);
+                            if (isNeedSyncByOldPos(newBlockOldPosBack))
+                                break;
+                            //next same block
+                            newBlockIndex=_indexMapFrom(newBlockOldPosBack);
+                            //assert(newBlockIndex<kBlockCount);
+                            pNewBlockDataInOldPos=&out_newBlockDataInOldPoss[newBlockIndex];
+                            newBlockOldPosBack=*pNewBlockDataInOldPos;
+                        }
+                        //continue;
+                    }
+                }
+            }else{
+                if ((--hitOutLimit)<=0) break;
+            }
+        }else{
+            if ((--hitOutLimit)<=0) break;
+        }
+    }while ((++range_begin)!=range_end);
+    return isMatched;
+}
+
+#if (_IS_USED_MULTITHREAD)
+void _rollMatch_mt(int threadIndex,void* workData){
+    TMt& mt=*(TMt*)workData;
+    TMtByChannel::TAutoThreadEnd __auto_thread_end(mt);
+
+    hpatch_StreamPos_t clipSize=kBestMTClipSize;
+    hpatch_StreamPos_t _minClipSize=(hpatch_StreamPos_t)(mt.matchDatas.newSyncInfo->kSyncBlockSize)*4;
+    if (clipSize<_minClipSize) clipSize=_minClipSize;
+    uint32_t threadNum=mt.matchDatas.threadNum;
+    hpatch_StreamPos_t oldSize=mt.matchDatas.oldStream->streamSize;
+    hpatch_StreamPos_t _maxClipSize=(oldSize+threadNum-1)/threadNum;
+    if (clipSize>_maxClipSize) clipSize=_maxClipSize;
+    size_t workCount=(size_t)((oldSize+clipSize-1)/clipSize);
+    assert(workCount*clipSize>=oldSize);
+    try{
+    #if (_IS_USED_CPP_ATOMIC)
+        std::atomic<size_t>& workIndex=*(std::atomic<size_t>*)&mt.workIndex;
+        while (true){
+            size_t curWorkIndex=workIndex++;
+            if (curWorkIndex>=workCount) break;
+    #else
+        for (size_t curWorkIndex=0;curWorkIndex<workCount;++curWorkIndex) {
+            if (!mt.getWorki(curWorkIndex))
+                continue;
+    #endif
+            hpatch_StreamPos_t oldPosBegin=curWorkIndex*clipSize;
+            hpatch_StreamPos_t oldPosEnd = oldPosBegin+clipSize;
+            if (oldPosEnd>oldSize) oldPosEnd=oldSize;
+            mt.matchDatas.rollMatch(mt.matchDatas,oldPosBegin,oldPosEnd,&mt);
+        }
+    }catch(...){
+        mt.on_error();
+    }
+}
+#endif
+
+
+void _matchNewDataInOld(_TMatchDatas& matchDatas,int threadNum){
+    const TNewDataSyncInfo* newSyncInfo=matchDatas.newSyncInfo;
+    checkv((!newSyncInfo->isSeqMatch)||(0==newSyncInfo->samePairCount));
+    const uint32_t kBlockCount=(uint32_t)TNewDataSyncInfo_blockCount(newSyncInfo);
+    hpatch_StreamPos_t* out_newBlockDataInOldPoss=matchDatas.out_newBlockDataInOldPoss;
+    _init_newBlockDataInOldPoss(out_newBlockDataInOldPoss,kBlockCount);
+    
+    const hpatch_StreamPos_t oldDataSize=matchDatas.oldStream->streamSize;
+    if (oldDataSize==0) return;
+    checkv(oldDataSize<(hpatch_StreamPos_t)(kBlockType_needSync-1-kBlockCount));
+    
+    TAutoMem _mem_sorted;
+    const uint32_t* sorted_newIndexs=matchDatas.getSortedIndexs(_mem_sorted,newSyncInfo,matchDatas.filter);
+    {
+        uint32_t curPair=0;
+        for (uint32_t i=0;i<kBlockCount;++i){
+            if ((curPair<newSyncInfo->samePairCount)&&(i==newSyncInfo->samePairList[curPair].curIndex)){
+                uint32_t sameIndex=newSyncInfo->samePairList[curPair].sameIndex;
+                while (!isNeedSyncByOldPos(out_newBlockDataInOldPoss[sameIndex])){
+                    sameIndex=_indexMapFrom(out_newBlockDataInOldPoss[sameIndex]);
+                    assert(sameIndex<kBlockCount);
+                }
+                assert(sameIndex<i);
+                out_newBlockDataInOldPoss[sameIndex]=_indexMapTo(i);
+                ++curPair;
+            }
+        }
+        assert(curPair==newSyncInfo->samePairCount);
+    }
+    
+    TAutoMem _mem_table;
+    unsigned int kTableHashShlBit=0;
+    const uint32_t* sorted_newIndexs_table=getSortedIndexsTable(_mem_table,newSyncInfo,sorted_newIndexs,&kTableHashShlBit);
+
+    matchDatas.sorted_newIndexs=sorted_newIndexs;
+    matchDatas.sorted_newIndexs_table=sorted_newIndexs_table;
+    matchDatas.kTableHashShlBit=kTableHashShlBit;
+#if (_IS_USED_MULTITHREAD)
+    const hpatch_StreamPos_t _bestWorkCount=oldDataSize/kBestMTClipSize;
+    threadNum=(_bestWorkCount<=threadNum)?(int)_bestWorkCount:threadNum;
+    if (threadNum>1){
+        matchDatas.threadNum=threadNum;
+        TMt mt(matchDatas);
+        checkv(mt.start_threads(threadNum,_rollMatch_mt,&mt,true));
+        mt.wait_all_thread_end();
+        checkv(!mt.is_on_error());
+    }else
+#endif
+    {
+        matchDatas.rollMatch(matchDatas,0,oldDataSize,0);
+    }
+
+    _end_newBlockDataInOldPoss(out_newBlockDataInOldPoss,kBlockCount,oldDataSize);
+}
+
+static void _rollMatch(_TMatchDatas& rd,hpatch_StreamPos_t oldRollBegin,hpatch_StreamPos_t oldRollEnd,void* _mt){
+    _tm_rollMatch<TStreamDataRoll,TBloomFilter<tm_roll_uint>,false>
+            (rd,oldRollBegin,oldRollEnd,toSavedPartRollHash,_mt);
+}
+
+const uint32_t* getSortedIndexs(TAutoMem& _mem_sorted,const TNewDataSyncInfo* newSyncInfo,TBloomFilter<tm_roll_uint>& filter){
+    return _tm_getSortedIndexs(_mem_sorted,newSyncInfo,filter);
+}
+static const uint32_t* _getSortedIndexs(TAutoMem& _mem_sorted,const TNewDataSyncInfo* newSyncInfo,void* filter){
+    return getSortedIndexs(_mem_sorted,newSyncInfo,*(TBloomFilter<tm_roll_uint>*)filter);
+}
+
+void matchNewDataInOld(hpatch_StreamPos_t* out_newBlockDataInOldPoss,const TNewDataSyncInfo* newSyncInfo,
+                       const hpatch_TStreamInput* oldStream,int threadNum){
+    TBloomFilter<tm_roll_uint> filter;
+    filter.init((size_t)(TNewDataSyncInfo_blockCount(newSyncInfo)-newSyncInfo->samePairCount));
+
+    _TMatchDatas matchDatas; memset(&matchDatas,0,sizeof(matchDatas));
+    matchDatas.filter=&filter;
+    matchDatas.out_newBlockDataInOldPoss=out_newBlockDataInOldPoss;
+    matchDatas.newSyncInfo=newSyncInfo;
+    matchDatas.oldStream=oldStream;
+    matchDatas.rollMatch=_rollMatch;
+    matchDatas.getSortedIndexs=_getSortedIndexs;
+    _matchNewDataInOld(matchDatas,threadNum);
+}
+
+} //namespace sync_private

package/harmony/pushy/src/main/cpp/HDiffPatch/libhsync/sync_client/match_in_old.h

@@ -0,0 +1,48 @@
+//  match_in_old.h
+//  sync_client
+//  Created by housisong on 2019-09-22.
+/*
+ The MIT License (MIT)
+ Copyright (c) 2019-2020 HouSisong
+ 
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+ 
+ The above copyright notice and this permission notice shall be
+ included in all copies of the Software.
+ 
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef match_in_old_h
+#define match_in_old_h
+#include "sync_client_type.h"
+namespace sync_private{
+
+static const hpatch_StreamPos_t kBlockType_needSync =~(hpatch_StreamPos_t)0; //download, default
+static hpatch_force_inline bool isNeedSyncByOldPos(hpatch_StreamPos_t blockDataInOldPos){
+    return (blockDataInOldPos==kBlockType_needSync);
+}
+
+//matchNewDataInOld()
+//throw std::runtime_error on error
+//result value in out_newBlockDataInOldPoss:
+// value==kBlockType_needSync : this block need download from server;
+// 0<=value<oldDataSize : this block read from oldStream,starting at value in oldStream;
+void matchNewDataInOld(hpatch_StreamPos_t* out_newBlockDataInOldPoss,const TNewDataSyncInfo* newSyncInfo,
+                       const hpatch_TStreamInput* oldStream,int threadNum);
+
+} //namespace sync_private
+#endif // match_in_old_h

package/harmony/pushy/src/main/cpp/HDiffPatch/libhsync/sync_client/match_in_types.h

@@ -0,0 +1,198 @@
+//  match_in_types.h
+//  sync_client
+//  move from match_in_old.cpp by housisong on 2025-04-18.
+/*
+ The MIT License (MIT)
+ Copyright (c) 2019-2025 HouSisong
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef match_in_types_h
+#define match_in_types_h
+#include "string.h" //memmove
+#include <algorithm> //sort, equal_range lower_bound
+#include "sync_client_type_private.h"
+#include "../../libHDiffPatch/HDiff/private_diff/mem_buf.h"
+#include "../../libHDiffPatch/HDiff/private_diff/limit_mem_diff/bloom_filter.h"
+#include "../../libParallel/parallel_channel.h"
+#include "sync_diff_data.h"
+#if (_IS_USED_MULTITHREAD)
+# ifndef _IS_USED_CPP_ATOMIC
+#   define _IS_USED_CPP_ATOMIC  1
+# endif
+# if (_IS_USED_CPP_ATOMIC)
+#   include <atomic>
+# endif
+#endif
+using namespace hdiff_private;
+namespace sync_private{
+
+//kIsSkipMatchedBlock 0: roll byte by byte  1: skip matched block, speed++, but patchSize+
+#define     kIsSkipMatchedBlock     1
+static const int kMatchHitOutLimit =16;       //limit match deep
+static const size_t kBestReadSize  =1024*256; //for sequence read
+#if (_IS_USED_MULTITHREAD)
+static const size_t kBestMTClipSize=1*1024*1024; //for multi-thread read once
+#endif //_IS_USED_MULTITHREAD
+
+typedef unsigned char TByte;
+typedef hpatch_uint64_t tm_roll_uint;
+
+struct TIndex_comp0{
+    inline explicit TIndex_comp0(const uint8_t* _hashs,size_t _byteSize,bool isSeqMatch=false)
+    :hashs(_hashs),byteSize(_byteSize),cmpSize(_byteSize*(isSeqMatch?2:1)){ }
+    typedef uint32_t TIndex;
+    struct TDigest{
+        const uint8_t*  digests;
+        inline explicit TDigest(const uint8_t* _digests):digests(_digests){}
+    };
+    inline bool operator()(const TIndex x,const TDigest& y)const { //for equal_range
+        return _cmp(hashs+x*byteSize,y.digests,cmpSize)<0; }
+    bool operator()(const TDigest& x,const TIndex y)const { //for equal_range
+        return _cmp(x.digests,hashs+y*byteSize,cmpSize)<0; }
+    
+    inline bool operator()(const TIndex x, const TIndex y)const {//for sort
+        return _cmp(hashs+x*byteSize,hashs+y*byteSize,cmpSize)<0; }
+protected:
+    const uint8_t*  hashs;
+    size_t          byteSize;
+    size_t          cmpSize;
+public:
+    inline static int _cmp(const uint8_t* px,const uint8_t* py,size_t cmpSize){
+        const uint8_t* px_end=px+cmpSize;
+        for (;px!=px_end;++px,++py){
+            int sub=(int)(*px)-(*py);
+            if (sub!=0) return sub; //value sort
+        }
+        return 0;
+    }
+};
+
+struct TIndex_comp01{
+    inline explicit TIndex_comp01(const uint8_t* _hashs0,size_t _byteSize0,
+                                  const uint8_t* _hashs1,size_t _byteSize1,bool isSeqMatch0=false)
+    :hashs0(_hashs0),hashs1(_hashs1),byteSize0(_byteSize0),byteSize1(_byteSize1),
+        cmpSize0(_byteSize0*(isSeqMatch0?2:1)){ }
+    typedef uint32_t TIndex;
+    inline bool operator()(const TIndex x, const TIndex y)const {//for sort
+        int cmp0=TIndex_comp0::_cmp(hashs0+x*byteSize0,hashs0+y*byteSize0,cmpSize0);
+        if (cmp0!=0)
+            return cmp0<0;
+        else
+            return TIndex_comp0::_cmp(hashs1+x*byteSize1,hashs1+y*byteSize1,byteSize1)<0;
+    }
+protected:
+    const uint8_t*  hashs0;
+    const uint8_t*  hashs1;
+    size_t          byteSize0;
+    size_t          byteSize1;
+    size_t          cmpSize0;
+};
+
+
+struct TStreamDataCache_base {
+    TStreamDataCache_base(const hpatch_TStreamInput* baseStream,hpatch_StreamPos_t streamRollBegin,
+                          hpatch_StreamPos_t streamRollEnd,size_t backupSize,size_t backZeroLen,uint32_t kSyncBlockSize,
+                          hpatch_TChecksum* strongChecksumPlugin,void* _readLocker=0);
+    ~TStreamDataCache_base();
+    inline hpatch_StreamPos_t streamRollPosEnd()const{ return m_streamRollEnd+m_kBackZeroLen; }
+    void _cache();
+
+    inline bool isRollEnded()const{ return m_cur>m_cache.data_end(); }
+    inline const TByte* calcPartStrongChecksum(size_t outPartBits){
+        return _calcPartStrongChecksum(m_cur,m_kSyncBlockSize,outPartBits); }
+    inline const TByte* strongChecksum()const{//must after do calcPartStrongChecksum()
+        return m_strongChecksum_buf+m_checksumByteSize; }
+    inline size_t strongChecksumByteSize()const{ return m_checksumByteSize; }
+    inline hpatch_TChecksum* strongChecksumPlugin()const{ return m_strongChecksumPlugin; }
+    inline hpatch_checksumHandle checkChecksum()const{ return m_checkChecksum; }
+    inline hpatch_StreamPos_t curStreamPos()const{ return m_readedPos-(m_cache.data_end()-m_cur); }
+    inline hpatch_uint32_t  getSyncBlockSize()const{ return m_kSyncBlockSize; }
+    inline hpatch_StreamPos_t getStreamSize()const{ return m_baseStream->streamSize; }
+protected:
+    const hpatch_TStreamInput* m_baseStream;
+    hpatch_StreamPos_t      m_readedPos;
+    hpatch_StreamPos_t      m_streamRollEnd;
+    TAutoMem                m_cache;
+    TByte*                  m_strongChecksum_buf;
+    TByte*                  m_cur;
+    const size_t            m_kBackZeroLen; //default kSyncBlockSize-1
+    const size_t            m_kBackupSize; //default 0
+    const uint32_t          m_kSyncBlockSize;
+    uint32_t                m_checksumByteSize;
+    hpatch_TChecksum*       m_strongChecksumPlugin;
+    hpatch_checksumHandle   m_checksumHandle;
+    hpatch_checksumHandle   m_checkChecksum;
+    void*                   m_readLocker;
+
+    inline void _setRollEnded() { m_cur=m_cache.data_end()+1; }
+    const TByte* _calcPartStrongChecksum(const TByte* buf,size_t bufSize,size_t outPartBits);
+};
+
+struct TStreamDataRoll:public TStreamDataCache_base {
+    inline TStreamDataRoll(const hpatch_TStreamInput* baseStream,hpatch_StreamPos_t streamRollBegin,
+                            hpatch_StreamPos_t streamRollEnd,uint32_t kSyncBlockSize,
+                            hpatch_TChecksum* strongChecksumPlugin,void* _readLocker=0)
+            :TStreamDataCache_base(baseStream,streamRollBegin,streamRollEnd,0,kSyncBlockSize-1,kSyncBlockSize,
+                                   strongChecksumPlugin,_readLocker),m_rollHash(0){
+                if (isRollEnded()) return;
+                m_rollHash=roll_hash_start(m_cur,m_kSyncBlockSize);
+            }
+    hpatch_force_inline tm_roll_uint hashValue()const{ return m_rollHash; }
+    hpatch_force_inline bool roll(){
+        const TByte* curIn=m_cur+m_kSyncBlockSize;
+        assert(curIn>=m_cache.data());
+        if (curIn<m_cache.data_end()){
+            m_rollHash=roll_hash_roll(m_rollHash,m_kSyncBlockSize,*m_cur,*curIn);
+            ++m_cur;
+            return true;
+        }else{
+            return _cacheAndRoll();
+        }
+    }
+    bool nextBlock(){
+        m_cur+=m_kSyncBlockSize;
+        if (m_cur+m_kSyncBlockSize>m_cache.data_end()){
+            TStreamDataCache_base::_cache();
+            if (isRollEnded()) return false;
+        }
+        m_rollHash=roll_hash_start(m_cur,m_kSyncBlockSize);
+        return true;
+    }
+    inline const uint8_t* partHashNext(size_t savedRollHashByteSize){ return 0; } //not used
+protected:
+    tm_roll_uint            m_rollHash;
+
+    bool _cacheAndRoll(){
+        TStreamDataCache_base::_cache();
+        if (isRollEnded()) return false;
+        return roll();
+    }
+};
+
+const uint32_t* getSortedIndexs(TAutoMem& _mem_sorted,const TNewDataSyncInfo* newSyncInfo,
+                                TBloomFilter<tm_roll_uint>& filter);
+const uint32_t* getSortedIndexsTable(TAutoMem& _mem_table,const TNewDataSyncInfo* newSyncInfo,
+                                     const uint32_t* sorted_newIndexs,unsigned int* out_kTableHashShlBit);
+
+} //namespace sync_private
+#endif //match_in_types_h