react-native-update 10.34.3 → 10.34.5

package/harmony/pushy/src/main/cpp/HDiffPatch/libhsync/sync_client/match_in_old.cpp

@@ -1,332 +0,0 @@
-//  match_in_old.cpp
-//  sync_client
-//  Created by housisong on 2019-09-22.
-/*
- The MIT License (MIT)
- Copyright (c) 2019-2023 HouSisong
-
- Permission is hereby granted, free of charge, to any person
- obtaining a copy of this software and associated documentation
- files (the "Software"), to deal in the Software without
- restriction, including without limitation the rights to use,
- copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the
- Software is furnished to do so, subject to the following
- conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- OTHER DEALINGS IN THE SOFTWARE.
- */
-#include "match_in_old.h"
-#include "match_in_types.h"
-#include "sync_client_private.h"
-namespace sync_private{
-
-    #define check(value,info) { if (!(value)) { throw std::runtime_error(info); } }
-    #define checkv(value)     check(value,"check "#value" error!")
-
-    TStreamDataCache_base::TStreamDataCache_base(const hpatch_TStreamInput* baseStream,hpatch_StreamPos_t streamRollBegin,
-                          hpatch_StreamPos_t streamRollEnd,size_t backupSize,size_t backZeroLen,uint32_t kSyncBlockSize,
-                          hpatch_TChecksum* strongChecksumPlugin,void* _readLocker)
-    :m_baseStream(baseStream),m_readedPos(streamRollBegin),m_streamRollEnd(streamRollEnd),
-    m_kBackupSize(backupSize),m_kBackZeroLen(backZeroLen),
-    m_strongChecksum_buf(0),m_cur(0),m_kSyncBlockSize(kSyncBlockSize),m_checksumByteSize(0),
-    m_strongChecksumPlugin(strongChecksumPlugin),m_checksumHandle(0),m_readLocker(_readLocker){
-        size_t cacheSize=(size_t)kSyncBlockSize*2+m_kBackupSize;
-        check((((cacheSize-m_kBackupSize)>>1)==kSyncBlockSize),"TStreamDataCache_base mem error!");
-        cacheSize=(cacheSize>=kBestReadSize)?cacheSize:kBestReadSize;
-        hpatch_StreamPos_t maxDataSize=(streamRollPosEnd()-streamRollBegin);
-        if (cacheSize>maxDataSize) cacheSize=(size_t)maxDataSize;
-#if (_IS_USED_MULTITHREAD)
-        if (_readLocker)
-            cacheSize=(size_t)maxDataSize;
-#endif
-
-        m_checksumHandle=strongChecksumPlugin->open(strongChecksumPlugin);
-        checkv(m_checksumHandle!=0);
-        m_checkChecksum=strongChecksumPlugin->open(strongChecksumPlugin);
-        checkv(m_checkChecksum!=0);
-        m_checksumByteSize=(uint32_t)m_strongChecksumPlugin->checksumByteSize();
-        m_cache.realloc(cacheSize+m_checksumByteSize+m_checksumByteSize);
-        m_cache.reduceSize(cacheSize);
-        m_strongChecksum_buf=m_cache.data_end();
-        m_cur=m_cache.data_end();
-        _cache();
-    }
-    TStreamDataCache_base::~TStreamDataCache_base(){
-        if (m_checksumHandle)
-            m_strongChecksumPlugin->close(m_strongChecksumPlugin,m_checksumHandle);
-        if (m_checkChecksum)
-            m_strongChecksumPlugin->close(m_strongChecksumPlugin,m_checkChecksum);
-    }
-    void TStreamDataCache_base::_cache(){
-        // all:[        streamDataSize      +     backZeroLen    ]
-        //                 ^                         ^
-        //          streamRollBegin            streamRollEnd
-        //self:            [                          ]
-        //                       ^             ^
-        //                                 readedPos
-        //
-        //cache:             [         | kBackupSize ]
-        //                        ^
-        //                       cur
-        if (isRollEnded()) return;
-        const hpatch_StreamPos_t rollPosEnd=streamRollPosEnd();
-        if (m_readedPos>=rollPosEnd) { _setRollEnded(); return; }//set end tag
-
-        size_t needLen=m_cur-m_cache.data();
-        if (m_readedPos+needLen>rollPosEnd)
-            needLen=(size_t)(rollPosEnd-m_readedPos);
-        if (m_cur-needLen+m_kSyncBlockSize+m_kBackupSize>m_cache.data_end()) { _setRollEnded(); return; } //set end tag
-
-        memmove(m_cur-needLen,m_cur,m_cache.data_end()-m_cur);
-        size_t readLen=needLen;
-        if (m_readedPos+readLen>m_baseStream->streamSize){
-            if (m_readedPos<m_baseStream->streamSize)
-                readLen=(size_t)(m_baseStream->streamSize-m_readedPos);
-            else
-                readLen=0;
-        }
-        if (readLen>0){
-            TByte* buf=m_cache.data_end()-needLen;
-#if (_IS_USED_MULTITHREAD)
-            CAutoLocker _autoLocker((HLocker)m_readLocker);
-#endif
-            check(m_baseStream->read(m_baseStream,m_readedPos,buf,buf+readLen),
-                  "TStreamDataRoll read streamData error!");
-        }
-        size_t zerolen=needLen-readLen;
-        if (zerolen>0)
-            memset(m_cache.data_end()-zerolen,0,zerolen);
-        m_readedPos+=needLen;
-        m_cur-=needLen;
-    }
-
-    const TByte* TStreamDataCache_base::_calcPartStrongChecksum(const TByte* buf,size_t bufSize,size_t outPartBits){
-        TByte* strongChecksum=m_strongChecksum_buf+m_checksumByteSize;
-        m_strongChecksumPlugin->begin(m_checksumHandle);
-        m_strongChecksumPlugin->append(m_checksumHandle,buf,buf+bufSize);
-        m_strongChecksumPlugin->end(m_checksumHandle,strongChecksum,strongChecksum+m_checksumByteSize);
-        toPartChecksum(m_strongChecksum_buf,outPartBits,strongChecksum,m_checksumByteSize);
-        return m_strongChecksum_buf;
-    }
-
-    const uint32_t* getSortedIndexsTable(TAutoMem& _mem_table,const TNewDataSyncInfo* newSyncInfo,
-                                         const uint32_t* sorted_newIndexs,unsigned int* out_kTableHashShlBit){//optimize for std::equal_range
-        unsigned int& kTableHashShlBit=*out_kTableHashShlBit;
-        const uint32_t kBlockCount=(uint32_t)TNewDataSyncInfo_blockCount(newSyncInfo);
-        const uint32_t kMatchBlockCount=kBlockCount-newSyncInfo->samePairCount;
-        const size_t needRollHashByteSize=newSyncInfo->savedRollHashByteSize*(newSyncInfo->isSeqMatch?2:1);
-        unsigned int kTableBit =getBetterCacheBlockTableBit(kMatchBlockCount);
-        const size_t needRollHashBits=newSyncInfo->savedRollHashBits*(newSyncInfo->isSeqMatch?2:1);
-        if (kTableBit>needRollHashBits) kTableBit=(unsigned int)needRollHashBits;
-        kTableHashShlBit=(int)(needRollHashBits-kTableBit);
-        _mem_table.realloc((size_t)sizeof(uint32_t)*((1<<kTableBit)+1));
-        uint32_t* sorted_newIndexs_table=(uint32_t*)_mem_table.data();
-        {
-            TIndex_comp0 icomp0(newSyncInfo->rollHashs,newSyncInfo->savedRollHashByteSize,newSyncInfo->isSeqMatch);
-            const uint32_t* pos=sorted_newIndexs;
-            uint8_t part[sizeof(tm_roll_uint)]={0};
-            for (uint32_t i=0; i<((uint32_t)1<<kTableBit); ++i) {
-                tm_roll_uint digest=((tm_roll_uint)i)<<kTableHashShlBit;
-                writeRollHashBytes(part,digest,needRollHashByteSize);
-                TIndex_comp0::TDigest digest_value(part);
-                pos=std::lower_bound(pos,sorted_newIndexs+kMatchBlockCount,digest_value,icomp0);
-                sorted_newIndexs_table[i]=(uint32_t)(pos-sorted_newIndexs);
-            }
-            sorted_newIndexs_table[((size_t)1<<kTableBit)]=kMatchBlockCount;
-        }
-        return sorted_newIndexs_table;
-    }
-
-
-static inline hpatch_StreamPos_t _indexMapTo(uint32_t i){
-    return (hpatch_StreamPos_t)(kBlockType_needSync-1)-i; }
-static inline uint32_t _indexMapFrom(hpatch_StreamPos_t pos){
-    return (uint32_t)((kBlockType_needSync-1)-pos); }
-
-typedef volatile hpatch_StreamPos_t volStreamPos_t;
-
-bool _matchRange(hpatch_StreamPos_t* out_newBlockDataInOldPoss,const uint32_t* range_begin,const uint32_t* range_end,
-                 TStreamDataCache_base& oldData,const TNewDataSyncInfo* newSyncInfo,hpatch_StreamPos_t kMinRevSameIndex,void* _mt){
-    const TByte* oldPartStrongChecksum=0;
-    const size_t savedStrongChecksumByteSize=newSyncInfo->savedStrongChecksumByteSize;
-    bool isMatched=false;
-    int hitOutLimit=kMatchHitOutLimit;
-    do {
-        size_t newBlockIndex=*range_begin;
-        volStreamPos_t* pNewBlockDataInOldPos=&out_newBlockDataInOldPoss[newBlockIndex];
-        hpatch_StreamPos_t newBlockOldPosBack=*pNewBlockDataInOldPos;   
-        if (newBlockOldPosBack>=kMinRevSameIndex){
-            if (oldPartStrongChecksum==0)
-                oldPartStrongChecksum=oldData.calcPartStrongChecksum(newSyncInfo->savedStrongChecksumBits);
-            const TByte* newPairStrongChecksum=newSyncInfo->partChecksums+newBlockIndex*savedStrongChecksumByteSize;
-            if (0==memcmp(oldPartStrongChecksum,newPairStrongChecksum,savedStrongChecksumByteSize)){
-                isMatched=true;
-                hpatch_StreamPos_t curPos=oldData.curStreamPos();
-                {
-#if (_IS_USED_MULTITHREAD)
-                    TMt* mt=(TMt*)_mt;
-                    CAutoLocker _autoLocker(mt?mt->writeLocker.locker:0);
-                    newBlockOldPosBack=*pNewBlockDataInOldPos;
-                    if (newBlockOldPosBack<kMinRevSameIndex){// other thread done
-                        if ((--hitOutLimit)<=0) break;
-                    }else
-#endif
-                    {
-                        while(true){ //hit
-                            (*pNewBlockDataInOldPos)=curPos;
-                            if (!newSyncInfo->isNotCChecksumNewMTParallel) //note: checkChecksumAppendData() is only for hsynz
-                                checkChecksumAppendData(newSyncInfo->savedNewDataCheckChecksum,(uint32_t)newBlockIndex,
-                                                        oldData.strongChecksumPlugin(),oldData.checkChecksum(),
-                                                        oldData.strongChecksum(),0,0);
-                            if (isNeedSyncByOldPos(newBlockOldPosBack))
-                                break;
-                            //next same block
-                            newBlockIndex=_indexMapFrom(newBlockOldPosBack);
-                            //assert(newBlockIndex<kBlockCount);
-                            pNewBlockDataInOldPos=&out_newBlockDataInOldPoss[newBlockIndex];
-                            newBlockOldPosBack=*pNewBlockDataInOldPos;
-                        }
-                        //continue;
-                    }
-                }
-            }else{
-                if ((--hitOutLimit)<=0) break;
-            }
-        }else{
-            if ((--hitOutLimit)<=0) break;
-        }
-    }while ((++range_begin)!=range_end);
-    return isMatched;
-}
-
-#if (_IS_USED_MULTITHREAD)
-void _rollMatch_mt(int threadIndex,void* workData){
-    TMt& mt=*(TMt*)workData;
-    TMtByChannel::TAutoThreadEnd __auto_thread_end(mt);
-
-    hpatch_StreamPos_t clipSize=kBestMTClipSize;
-    hpatch_StreamPos_t _minClipSize=(hpatch_StreamPos_t)(mt.matchDatas.newSyncInfo->kSyncBlockSize)*4;
-    if (clipSize<_minClipSize) clipSize=_minClipSize;
-    uint32_t threadNum=mt.matchDatas.threadNum;
-    hpatch_StreamPos_t oldSize=mt.matchDatas.oldStream->streamSize;
-    hpatch_StreamPos_t _maxClipSize=(oldSize+threadNum-1)/threadNum;
-    if (clipSize>_maxClipSize) clipSize=_maxClipSize;
-    size_t workCount=(size_t)((oldSize+clipSize-1)/clipSize);
-    assert(workCount*clipSize>=oldSize);
-    try{
-    #if (_IS_USED_CPP_ATOMIC)
-        std::atomic<size_t>& workIndex=*(std::atomic<size_t>*)&mt.workIndex;
-        while (true){
-            size_t curWorkIndex=workIndex++;
-            if (curWorkIndex>=workCount) break;
-    #else
-        for (size_t curWorkIndex=0;curWorkIndex<workCount;++curWorkIndex) {
-            if (!mt.getWorki(curWorkIndex))
-                continue;
-    #endif
-            hpatch_StreamPos_t oldPosBegin=curWorkIndex*clipSize;
-            hpatch_StreamPos_t oldPosEnd = oldPosBegin+clipSize;
-            if (oldPosEnd>oldSize) oldPosEnd=oldSize;
-            mt.matchDatas.rollMatch(mt.matchDatas,oldPosBegin,oldPosEnd,&mt);
-        }
-    }catch(...){
-        mt.on_error();
-    }
-}
-#endif
-
-
-void _matchNewDataInOld(_TMatchDatas& matchDatas,int threadNum){
-    const TNewDataSyncInfo* newSyncInfo=matchDatas.newSyncInfo;
-    checkv((!newSyncInfo->isSeqMatch)||(0==newSyncInfo->samePairCount));
-    const uint32_t kBlockCount=(uint32_t)TNewDataSyncInfo_blockCount(newSyncInfo);
-    hpatch_StreamPos_t* out_newBlockDataInOldPoss=matchDatas.out_newBlockDataInOldPoss;
-    _init_newBlockDataInOldPoss(out_newBlockDataInOldPoss,kBlockCount);
-    
-    const hpatch_StreamPos_t oldDataSize=matchDatas.oldStream->streamSize;
-    if (oldDataSize==0) return;
-    checkv(oldDataSize<(hpatch_StreamPos_t)(kBlockType_needSync-1-kBlockCount));
-    
-    TAutoMem _mem_sorted;
-    const uint32_t* sorted_newIndexs=matchDatas.getSortedIndexs(_mem_sorted,newSyncInfo,matchDatas.filter);
-    {
-        uint32_t curPair=0;
-        for (uint32_t i=0;i<kBlockCount;++i){
-            if ((curPair<newSyncInfo->samePairCount)&&(i==newSyncInfo->samePairList[curPair].curIndex)){
-                uint32_t sameIndex=newSyncInfo->samePairList[curPair].sameIndex;
-                while (!isNeedSyncByOldPos(out_newBlockDataInOldPoss[sameIndex])){
-                    sameIndex=_indexMapFrom(out_newBlockDataInOldPoss[sameIndex]);
-                    assert(sameIndex<kBlockCount);
-                }
-                assert(sameIndex<i);
-                out_newBlockDataInOldPoss[sameIndex]=_indexMapTo(i);
-                ++curPair;
-            }
-        }
-        assert(curPair==newSyncInfo->samePairCount);
-    }
-    
-    TAutoMem _mem_table;
-    unsigned int kTableHashShlBit=0;
-    const uint32_t* sorted_newIndexs_table=getSortedIndexsTable(_mem_table,newSyncInfo,sorted_newIndexs,&kTableHashShlBit);
-
-    matchDatas.sorted_newIndexs=sorted_newIndexs;
-    matchDatas.sorted_newIndexs_table=sorted_newIndexs_table;
-    matchDatas.kTableHashShlBit=kTableHashShlBit;
-#if (_IS_USED_MULTITHREAD)
-    const hpatch_StreamPos_t _bestWorkCount=oldDataSize/kBestMTClipSize;
-    threadNum=(_bestWorkCount<=threadNum)?(int)_bestWorkCount:threadNum;
-    if (threadNum>1){
-        matchDatas.threadNum=threadNum;
-        TMt mt(matchDatas);
-        checkv(mt.start_threads(threadNum,_rollMatch_mt,&mt,true));
-        mt.wait_all_thread_end();
-        checkv(!mt.is_on_error());
-    }else
-#endif
-    {
-        matchDatas.rollMatch(matchDatas,0,oldDataSize,0);
-    }
-
-    _end_newBlockDataInOldPoss(out_newBlockDataInOldPoss,kBlockCount,oldDataSize);
-}
-
-static void _rollMatch(_TMatchDatas& rd,hpatch_StreamPos_t oldRollBegin,hpatch_StreamPos_t oldRollEnd,void* _mt){
-    _tm_rollMatch<TStreamDataRoll,TBloomFilter<tm_roll_uint>,false>
-            (rd,oldRollBegin,oldRollEnd,toSavedPartRollHash,_mt);
-}
-
-const uint32_t* getSortedIndexs(TAutoMem& _mem_sorted,const TNewDataSyncInfo* newSyncInfo,TBloomFilter<tm_roll_uint>& filter){
-    return _tm_getSortedIndexs(_mem_sorted,newSyncInfo,filter);
-}
-static const uint32_t* _getSortedIndexs(TAutoMem& _mem_sorted,const TNewDataSyncInfo* newSyncInfo,void* filter){
-    return getSortedIndexs(_mem_sorted,newSyncInfo,*(TBloomFilter<tm_roll_uint>*)filter);
-}
-
-void matchNewDataInOld(hpatch_StreamPos_t* out_newBlockDataInOldPoss,const TNewDataSyncInfo* newSyncInfo,
-                       const hpatch_TStreamInput* oldStream,int threadNum){
-    TBloomFilter<tm_roll_uint> filter;
-    filter.init((size_t)(TNewDataSyncInfo_blockCount(newSyncInfo)-newSyncInfo->samePairCount));
-
-    _TMatchDatas matchDatas; memset(&matchDatas,0,sizeof(matchDatas));
-    matchDatas.filter=&filter;
-    matchDatas.out_newBlockDataInOldPoss=out_newBlockDataInOldPoss;
-    matchDatas.newSyncInfo=newSyncInfo;
-    matchDatas.oldStream=oldStream;
-    matchDatas.rollMatch=_rollMatch;
-    matchDatas.getSortedIndexs=_getSortedIndexs;
-    _matchNewDataInOld(matchDatas,threadNum);
-}
-
-} //namespace sync_private

package/harmony/pushy/src/main/cpp/HDiffPatch/libhsync/sync_client/match_in_old.h

@@ -1,48 +0,0 @@
-//  match_in_old.h
-//  sync_client
-//  Created by housisong on 2019-09-22.
-/*
- The MIT License (MIT)
- Copyright (c) 2019-2020 HouSisong
- 
- Permission is hereby granted, free of charge, to any person
- obtaining a copy of this software and associated documentation
- files (the "Software"), to deal in the Software without
- restriction, including without limitation the rights to use,
- copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the
- Software is furnished to do so, subject to the following
- conditions:
- 
- The above copyright notice and this permission notice shall be
- included in all copies of the Software.
- 
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef match_in_old_h
-#define match_in_old_h
-#include "sync_client_type.h"
-namespace sync_private{
-
-static const hpatch_StreamPos_t kBlockType_needSync =~(hpatch_StreamPos_t)0; //download, default
-static hpatch_force_inline bool isNeedSyncByOldPos(hpatch_StreamPos_t blockDataInOldPos){
-    return (blockDataInOldPos==kBlockType_needSync);
-}
-
-//matchNewDataInOld()
-//throw std::runtime_error on error
-//result value in out_newBlockDataInOldPoss:
-// value==kBlockType_needSync : this block need download from server;
-// 0<=value<oldDataSize : this block read from oldStream,starting at value in oldStream;
-void matchNewDataInOld(hpatch_StreamPos_t* out_newBlockDataInOldPoss,const TNewDataSyncInfo* newSyncInfo,
-                       const hpatch_TStreamInput* oldStream,int threadNum);
-
-} //namespace sync_private
-#endif // match_in_old_h

package/harmony/pushy/src/main/cpp/HDiffPatch/libhsync/sync_client/match_in_types.h

@@ -1,198 +0,0 @@
-//  match_in_types.h
-//  sync_client
-//  move from match_in_old.cpp by housisong on 2025-04-18.
-/*
- The MIT License (MIT)
- Copyright (c) 2019-2025 HouSisong
-
- Permission is hereby granted, free of charge, to any person
- obtaining a copy of this software and associated documentation
- files (the "Software"), to deal in the Software without
- restriction, including without limitation the rights to use,
- copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the
- Software is furnished to do so, subject to the following
- conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef match_in_types_h
-#define match_in_types_h
-#include "string.h" //memmove
-#include <algorithm> //sort, equal_range lower_bound
-#include "sync_client_type_private.h"
-#include "../../libHDiffPatch/HDiff/private_diff/mem_buf.h"
-#include "../../libHDiffPatch/HDiff/private_diff/limit_mem_diff/bloom_filter.h"
-#include "../../libParallel/parallel_channel.h"
-#include "sync_diff_data.h"
-#if (_IS_USED_MULTITHREAD)
-# ifndef _IS_USED_CPP_ATOMIC
-#   define _IS_USED_CPP_ATOMIC  1
-# endif
-# if (_IS_USED_CPP_ATOMIC)
-#   include <atomic>
-# endif
-#endif
-using namespace hdiff_private;
-namespace sync_private{
-
-//kIsSkipMatchedBlock 0: roll byte by byte  1: skip matched block, speed++, but patchSize+
-#define     kIsSkipMatchedBlock     1
-static const int kMatchHitOutLimit =16;       //limit match deep
-static const size_t kBestReadSize  =1024*256; //for sequence read
-#if (_IS_USED_MULTITHREAD)
-static const size_t kBestMTClipSize=1*1024*1024; //for multi-thread read once
-#endif //_IS_USED_MULTITHREAD
-
-typedef unsigned char TByte;
-typedef hpatch_uint64_t tm_roll_uint;
-
-struct TIndex_comp0{
-    inline explicit TIndex_comp0(const uint8_t* _hashs,size_t _byteSize,bool isSeqMatch=false)
-    :hashs(_hashs),byteSize(_byteSize),cmpSize(_byteSize*(isSeqMatch?2:1)){ }
-    typedef uint32_t TIndex;
-    struct TDigest{
-        const uint8_t*  digests;
-        inline explicit TDigest(const uint8_t* _digests):digests(_digests){}
-    };
-    inline bool operator()(const TIndex x,const TDigest& y)const { //for equal_range
-        return _cmp(hashs+x*byteSize,y.digests,cmpSize)<0; }
-    bool operator()(const TDigest& x,const TIndex y)const { //for equal_range
-        return _cmp(x.digests,hashs+y*byteSize,cmpSize)<0; }
-    
-    inline bool operator()(const TIndex x, const TIndex y)const {//for sort
-        return _cmp(hashs+x*byteSize,hashs+y*byteSize,cmpSize)<0; }
-protected:
-    const uint8_t*  hashs;
-    size_t          byteSize;
-    size_t          cmpSize;
-public:
-    inline static int _cmp(const uint8_t* px,const uint8_t* py,size_t cmpSize){
-        const uint8_t* px_end=px+cmpSize;
-        for (;px!=px_end;++px,++py){
-            int sub=(int)(*px)-(*py);
-            if (sub!=0) return sub; //value sort
-        }
-        return 0;
-    }
-};
-
-struct TIndex_comp01{
-    inline explicit TIndex_comp01(const uint8_t* _hashs0,size_t _byteSize0,
-                                  const uint8_t* _hashs1,size_t _byteSize1,bool isSeqMatch0=false)
-    :hashs0(_hashs0),hashs1(_hashs1),byteSize0(_byteSize0),byteSize1(_byteSize1),
-        cmpSize0(_byteSize0*(isSeqMatch0?2:1)){ }
-    typedef uint32_t TIndex;
-    inline bool operator()(const TIndex x, const TIndex y)const {//for sort
-        int cmp0=TIndex_comp0::_cmp(hashs0+x*byteSize0,hashs0+y*byteSize0,cmpSize0);
-        if (cmp0!=0)
-            return cmp0<0;
-        else
-            return TIndex_comp0::_cmp(hashs1+x*byteSize1,hashs1+y*byteSize1,byteSize1)<0;
-    }
-protected:
-    const uint8_t*  hashs0;
-    const uint8_t*  hashs1;
-    size_t          byteSize0;
-    size_t          byteSize1;
-    size_t          cmpSize0;
-};
-
-
-struct TStreamDataCache_base {
-    TStreamDataCache_base(const hpatch_TStreamInput* baseStream,hpatch_StreamPos_t streamRollBegin,
-                          hpatch_StreamPos_t streamRollEnd,size_t backupSize,size_t backZeroLen,uint32_t kSyncBlockSize,
-                          hpatch_TChecksum* strongChecksumPlugin,void* _readLocker=0);
-    ~TStreamDataCache_base();
-    inline hpatch_StreamPos_t streamRollPosEnd()const{ return m_streamRollEnd+m_kBackZeroLen; }
-    void _cache();
-
-    inline bool isRollEnded()const{ return m_cur>m_cache.data_end(); }
-    inline const TByte* calcPartStrongChecksum(size_t outPartBits){
-        return _calcPartStrongChecksum(m_cur,m_kSyncBlockSize,outPartBits); }
-    inline const TByte* strongChecksum()const{//must after do calcPartStrongChecksum()
-        return m_strongChecksum_buf+m_checksumByteSize; }
-    inline size_t strongChecksumByteSize()const{ return m_checksumByteSize; }
-    inline hpatch_TChecksum* strongChecksumPlugin()const{ return m_strongChecksumPlugin; }
-    inline hpatch_checksumHandle checkChecksum()const{ return m_checkChecksum; }
-    inline hpatch_StreamPos_t curStreamPos()const{ return m_readedPos-(m_cache.data_end()-m_cur); }
-    inline hpatch_uint32_t  getSyncBlockSize()const{ return m_kSyncBlockSize; }
-    inline hpatch_StreamPos_t getStreamSize()const{ return m_baseStream->streamSize; }
-protected:
-    const hpatch_TStreamInput* m_baseStream;
-    hpatch_StreamPos_t      m_readedPos;
-    hpatch_StreamPos_t      m_streamRollEnd;
-    TAutoMem                m_cache;
-    TByte*                  m_strongChecksum_buf;
-    TByte*                  m_cur;
-    const size_t            m_kBackZeroLen; //default kSyncBlockSize-1
-    const size_t            m_kBackupSize; //default 0
-    const uint32_t          m_kSyncBlockSize;
-    uint32_t                m_checksumByteSize;
-    hpatch_TChecksum*       m_strongChecksumPlugin;
-    hpatch_checksumHandle   m_checksumHandle;
-    hpatch_checksumHandle   m_checkChecksum;
-    void*                   m_readLocker;
-
-    inline void _setRollEnded() { m_cur=m_cache.data_end()+1; }
-    const TByte* _calcPartStrongChecksum(const TByte* buf,size_t bufSize,size_t outPartBits);
-};
-
-struct TStreamDataRoll:public TStreamDataCache_base {
-    inline TStreamDataRoll(const hpatch_TStreamInput* baseStream,hpatch_StreamPos_t streamRollBegin,
-                            hpatch_StreamPos_t streamRollEnd,uint32_t kSyncBlockSize,
-                            hpatch_TChecksum* strongChecksumPlugin,void* _readLocker=0)
-            :TStreamDataCache_base(baseStream,streamRollBegin,streamRollEnd,0,kSyncBlockSize-1,kSyncBlockSize,
-                                   strongChecksumPlugin,_readLocker),m_rollHash(0){
-                if (isRollEnded()) return;
-                m_rollHash=roll_hash_start(m_cur,m_kSyncBlockSize);
-            }
-    hpatch_force_inline tm_roll_uint hashValue()const{ return m_rollHash; }
-    hpatch_force_inline bool roll(){
-        const TByte* curIn=m_cur+m_kSyncBlockSize;
-        assert(curIn>=m_cache.data());
-        if (curIn<m_cache.data_end()){
-            m_rollHash=roll_hash_roll(m_rollHash,m_kSyncBlockSize,*m_cur,*curIn);
-            ++m_cur;
-            return true;
-        }else{
-            return _cacheAndRoll();
-        }
-    }
-    bool nextBlock(){
-        m_cur+=m_kSyncBlockSize;
-        if (m_cur+m_kSyncBlockSize>m_cache.data_end()){
-            TStreamDataCache_base::_cache();
-            if (isRollEnded()) return false;
-        }
-        m_rollHash=roll_hash_start(m_cur,m_kSyncBlockSize);
-        return true;
-    }
-    inline const uint8_t* partHashNext(size_t savedRollHashByteSize){ return 0; } //not used
-protected:
-    tm_roll_uint            m_rollHash;
-
-    bool _cacheAndRoll(){
-        TStreamDataCache_base::_cache();
-        if (isRollEnded()) return false;
-        return roll();
-    }
-};
-
-const uint32_t* getSortedIndexs(TAutoMem& _mem_sorted,const TNewDataSyncInfo* newSyncInfo,
-                                TBloomFilter<tm_roll_uint>& filter);
-const uint32_t* getSortedIndexsTable(TAutoMem& _mem_table,const TNewDataSyncInfo* newSyncInfo,
-                                     const uint32_t* sorted_newIndexs,unsigned int* out_kTableHashShlBit);
-
-} //namespace sync_private
-#endif //match_in_types_h