­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ ­ 3 ybpP@s ddlZddlZddlZddlZddlZddlZddlmZmZm Z ddl mZddl m Z m Z ddl mZdd Zd d ZGd d d eZGdddeZGdddeZGdddeZGdddeZGdddeZGdddeZGdddeZGdddeZGdddeZdS) N)NSIndex ChunkIndexChunkIndexEntry) hashindex)IntegrityCheckedFileFileIntegrityError) BaseTestCasecCstd|dS)Nz%-0.32dascii)bytes)xr!/usr/lib64/python3.6/hashindex.pyHsrcCstjt|jS)N)hashlibsha256rdigest)r rrrH2src@sDeZdZddZddZddZddZd d Zd d Zd dZ dS)HashIndexTestCasec Cs|}|jt|dx tdD]}|||t|<q W|jt|dx(tdD]}|j|t|||qRWx$tdD]}||d|t|<q|W|jt|dx,tdD] }|j|t|||dqWxtdD]}|t|=qWx"tddD]}t||kstqWx$tdD]}t||ks tq Wx&tdD]}|jt|jt|qFW|jt|dtj }|j |j ~t |j d }|jt j|jj|WdQRX|j|j }|jt|dx0tddD]"}|j|t|||dqW|j|jt|d|j |j ~|jt|j|j d|}|jtd|dtd|kstt|td|dkst|jtd|dtd|kst|td|dkst|jtd|d|dkst|jtd|d|dkst~dS) Nrdr2rb*r ) assert_equallenrangerAssertionError assert_raisesKeyError __delitem__tempfileNamedTemporaryFilewritenameopenrrreadZ hexdigestclear setdefault)selfclsZ make_valueZshaidxr idx_namefdrrr _generic_testsZ  $ " ""zHashIndexTestCase._generic_testcCs|jtddddS)NcSs||fS)Nr)r rrr[sz0HashIndexTestCase.test_nsindex..Z@85f72b036c692c8266e4f51ccf0cff2147204282b5e316ae508d30a448d88fef)r/r)r*rrr test_nsindexZs zHashIndexTestCase.test_nsindexcCs|jtddddS)NcSs |||fS)Nr)r rrrr0_sz3HashIndexTestCase.test_chunkindex..Z@c83fdf33755fc37879285f2ecfc5d1f63b97577494902126b6fb6f3e4d852488)r/r)r*rrrtest_chunkindex^s z!HashIndexTestCase.test_chunkindexcCsd}tj}t}|j|jtjj|j}|jt |dx t |D]}||f|t |<qFW|j|j|tjj|jkst xt |D]}|t |=qW|jt |d|j|j|j|tjj|jdS)Nir) r"r#rr$r%ospathgetsizerrrrr)r*nr-r,Z initial_sizer rrr test_resizebs   zHashIndexTestCase.test_resizecCst}x tdD]}||f|t|<qW|j}t|}|jt|d|jtt |t|j|ddd}|jt|d|j||dddS)Nr1r)Zmarkerr) rrr iteritemslistrrr StopIterationnext)r*r,r iteratorallZ second_halfrrrtest_iteritemsssz HashIndexTestCase.test_iteritemscCst}d |td<d |td<d|td<t}d|td<d|td<d|td<|j||tddksrt|tddkst|tddkst|tddkstdS)Nr rr,)r rr)rr@r@)rArBrB)rCrr)rDr@r@)rErFrF)rDrr)rGr@r@)rArBrB)rErFrF)rrmerger)r*idx1idx2rrrtest_chunkindex_merges       z'HashIndexTestCase.test_chunkindex_mergecCst}d |td<d |td<d |td<|j\}}}}}}|dksJt|dksVt|dksbt|dksnt|dkszt|dkstdS)Nr rrr@rA ,)r rLr)rrMr@)rArNrOiii(#i6iiiixi ipi,iXrArE)rrZ summarizer)r*r,sizeZcsizeZ unique_sizeZ unique_csizeZ unique_chunksZchunksrrrtest_chunkindex_summarizes        z+HashIndexTestCase.test_chunkindex_summarizeN) __name__ __module__ __qualname__r/r1r2r7r?rKrQrrrrrs= rc@seZdZdZddZdS)HashIndexExtraTestCasezRThese tests are separate because they should not become part of the selftest. cCs.tdtjd}t|}ddt|D}|dd|d|d|dd}}x"t|D]\}}|||f||<q`Wx"t|D]\}}|||f||<qWx|D] }||=qWx*t|D]\}}|||||fkstqWx|D]}|j|dkstqWx|D] }||=qWt|jgks*tdS)Ni cSsg|]}tjt|jqSr)rrrr).0krrr sz=HashIndexExtraTestCase.test_chunk_indexer..rrrA) intrMAX_LOAD_FACTORr enumeratergetr:r9)r*Z key_countindexZall_keyskeysZto_delete_keysikeyrrrtest_chunk_indexers"*     z)HashIndexExtraTestCase.test_chunk_indexerN)rRrSrT__doc__rbrrrrrUsrUc@seZdZddZddZdS)HashIndexSizeTestCasecCst}|jd kstdS) Ni rArC ,i4iF)rrPr)r*r,rrrtest_size_on_disksz'HashIndexSizeTestCase.test_size_on_diskc Cstt}x*tdD]}||d|df|t|<qWtj }|j|jtjj |j}WdQRX|j |kspt dS)NirrA) rrrr"r#r$r%r3r4r5rPr)r*r,r`filerPrrrtest_size_on_disk_accurates  z0HashIndexSizeTestCase.test_size_on_disk_accurateN)rRrSrTrirkrrrrrdsrdc@steZdZddZddZddZddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ ddZddZdS)HashIndexRefcountingTestCasecCst}tjdddf|td<x0tdD]$}|jtd^}}|tjks(tq(Wx0tdD]$}|jtd^}}|tjksZtqZWdS)Nr rrD)r MAX_VALUErrincrefrdecref)r*r,r`refcount_rrrtest_chunkindex_limitsz2HashIndexRefcountingTestCase.test_chunkindex_limitcCs(dd}|||}||||ks$t|S)NcSsNt}|ddf|td<t}|ddf|td<|j||td^}}|S)Nr r)rrrH)Z refcount1Z refcount2rIrJrprqrrrrHs z2HashIndexRefcountingTestCase._merge..merge)r)r*Z refcountaZ refcountbrHresultrrr_merges z#HashIndexRefcountingTestCase._mergecCs(tjd}|j||tjdks$tdS)Nrr )rrmrtr)r*halfrrrtest_chunkindex_merge_limit1s z9HashIndexRefcountingTestCase.test_chunkindex_merge_limit1cCs|jddtjkstdS)Nl^ei5w)rtrrmr)r*rrrtest_chunkindex_merge_limit2sz9HashIndexRefcountingTestCase.test_chunkindex_merge_limit2cCs(tjd}|j|d|tjks$tdS)Nrr )rrmrtr)r*rurrrtest_chunkindex_merge_limit3s z9HashIndexRefcountingTestCase.test_chunkindex_merge_limit3cCsFtjd}|j|d|tjks$t|j|d|dtjksBtdS)Nrr )rrmrtr)r*rurrrtest_chunkindex_merge_limit4s z9HashIndexRefcountingTestCase.test_chunkindex_merge_limit4cCsZt}|jtdddd|tddks.t|jtdddd|tddksVtdS) Nr rDrErGrrA)rDrErG)rErrA)raddrr)r*rIrrrtest_chunkindex_adds z0HashIndexRefcountingTestCase.test_chunkindex_addcCsJt}tjddf|td<|jtd|td^}}|tjksFtdS)NrErGr )rrmrrnr)r*rIrprqrrrtest_incref_limits z.HashIndexRefcountingTestCase.test_incref_limitcCsJt}tjddf|td<|jtd|td^}}|tjksFtdS)NrErGr )rrmrror)r*rIrprqrrrtest_decref_limit s z.HashIndexRefcountingTestCase.test_decref_limitc Cs:t}d|td<|jt|jtdWdQRXdS)Nrr )rrr)rrrrro)r*rIrrrtest_decref_zeros  z-HashIndexRefcountingTestCase.test_decref_zerocCsvt}|jtdddd|tddks.t|jtd|tddksPt|jtd|tddksrtdS)Nr rDrErG)rDrErG)rErErG)rDrErG)rrzrrrnro)r*rIrrrtest_incref_decrefsz/HashIndexRefcountingTestCase.test_incref_decrefc Cs8t}|jttjdddf|td<WdQRXdS)Nr r)rrrrmr)r*rIrrrtest_setitem_raises#s z0HashIndexRefcountingTestCase.test_setitem_raisesc!Cst}|jt|jtdWdQRX|jt|jtdWdQRX|jt|tdWdQRX|jt|jtddddWdQRXdS)Nr r)rrr rnrro OverflowErrorrz)r*r,rrr test_keyerror(s    z*HashIndexRefcountingTestCase.test_keyerrorN)rRrSrTrrrtrvrwrxryr{r|r}r~rrrrrrrrls  rlc@s@eZdZdZddZddZddZdd Zd d Zd d Z dS)HashIndexDataTestCases(eJzt0L0NgmAUhtHLT0LDEI6AuAEhMVYmVnSuYefC7AB3Aj9KNedJbnfyFne6P67P27w0EdG1Eac+Cm1ZybAsy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7Isy7LsL9nhc+cqTZ3XlO2Ys++Du5fX+l1/YFmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVmWZVn2/+0O2rYccw==cCsPtj>}tjj|d}|j|t|d}|j|jSQRXWdQRXdS)Nr,r) r"TemporaryDirectoryr3r4joinr$r&_packr')r*r,tempdirrjfrrr_serialize_hashindex:s    z*HashIndexDataTestCase._serialize_hashindexcCsRtj@}tjj|d}t|d}|j|j|WdQRXtj |SQRXdS)Nr,wb) r"rr3r4rr&r$_unpackrr')r* bytestringrrjrrrr_deserialize_hashindexAs   z,HashIndexDataTestCase._deserialize_hashindexcCstjtj|S)N)base64Z b64encodezlibcompress)r*rrrrrHszHashIndexDataTestCase._packcCstjtj|S)N)rZ decompressrZ b64decode)r*rrrrrKszHashIndexDataTestCase._unpackcCsRt}d|td<d |td<d |td<|j|}|j||j|jksNtdS) Nr rrArxl)r rrAl)rrr)rrr)rrrr HASHINDEXr)r*rIZ serializedrrrtest_identical_creationNs     z-HashIndexDataTestCase.test_identical_creationcCs|j|j}|tdd ks t|tddks4t|tddksHtt}d|td<|j||tdtjd d fkstdS)Nr rrArrxlrfi@rErG)r rrAl)rrr)rrrl)rrErG)rrrrrrHrm)r*rIrJrrrtest_read_known_goodWs   z*HashIndexDataTestCase.test_read_known_goodN) rRrSrTrrrrrrrrrrrr4s rc@seZdZddZddZdS)HashIndexIntegrityTestCasec Csb|j|j}tjj|d}t|dd}|j|WdQRX|j}d|ksNtd|ksZt||fS)Nr,T)r4r$finalZ HashHeader) rrr3r4rrr$integrity_datar)r*rr,rjr.rrrrwrite_integrity_checked_indexds   z8HashIndexIntegrityTestCase.write_integrity_checked_indexcCstjn}|j|\}}t|d}|jdWdQRX|jt*t|d|d}tj |WdQRXWdQRXWdQRXdS)Nzr+bsFooF)r4r$r) r"rrr&r$rrrrr')r*rrjrr.rrrtest_integrity_checked_filens   z6HashIndexIntegrityTestCase.test_integrity_checked_fileN)rRrSrTrrrrrrrcs rc@steZdZddZddZddZddZd d Zd d Zd dZ ddZ ddZ ddZ ddZ ddZddZdS)HashIndexCompactTestCasecCsdtj}|jd|j|jdd|j|jdd|jdjdd|jdjdd||_dS)NsBORG_IDXrClittlerfr rArg)ioBytesIOr$to_bytes index_data)r* num_entries num_bucketsrrrrr^ys zHashIndexCompactTestCase.indexcCs|jjdtj|j}|S)Nr)rseekrr')r*r^rrrindex_from_datas  z(HashIndexCompactTestCase.index_from_datacCstj}|j||jS)N)rrr$getvalue)r*r^datarrr index_to_datas z&HashIndexCompactTestCase.index_to_datacCs|j}|j|j|}|S)N)rcompactr)r*r^ compact_indexrrrindex_from_data_compact_to_datas z8HashIndexCompactTestCase.index_from_data_compact_to_datacGs2|jj|x |D]}|jj|jddqWdS)NrCr)rr$r)r*ravaluesvaluerrr write_entrys  z$HashIndexCompactTestCase.write_entrycCs|j|ddddS)Nlr)r)r*rarrr write_emptysz$HashIndexCompactTestCase.write_emptycCs|j|ddddS)Nlr)r)r*rarrr write_deletedsz&HashIndexCompactTestCase.write_deletedcCs|jddd|jtdddd|jtd|jtd|jtdddd|jtd d d d |jtd|j}|jddd|jtdddd|jtdddd|jtd d d d ||jjkstdS) NrArE)rrrr rrDrGrC rV) r^rrrrrrrr)r*rrrr test_simplesz$HashIndexCompactTestCase.test_simplecCs|jddd|jtd|jtdddd|jtd|jtdddd|jtd d d d |jtd|j}|jddd|jtdddd|jtdddd|jtd d d d ||jjkstdS) NrArE)rrr rrrDrGrCrrrV) r^rrrrrrrr)r*rrrrtest_first_emptysz)HashIndexCompactTestCase.test_first_emptycCs|jddd|jtd|jtdddd|jtd|jtdddd|jtd|jtd d d d |j}|jddd|jtdddd|jtdddd|jtd d d d ||jjkstdS) NrArE)rrr rrrDrGrCrrrV) r^rrrrrrrr)r*rrrrtest_last_usedsz'HashIndexCompactTestCase.test_last_usedcCs|jddd|jtd|jtdddd|jtdddd|jtd|jtd|jtd d d d |j}|jddd|jtdddd|jtdddd|jtd d d d ||jjkstdS) NrArE)rrr rrrDrGrCrrrV) r^rrrrrrrr)r*rrrrtest_too_few_empty_slotssz1HashIndexCompactTestCase.test_too_few_empty_slotscCs|jddd|jtd|jtd|jtd|jtd|jtd|jtd|j}|jddd||jjkstdS) NrrE)rrr rArrDrC)r^rrrrrrr)r*rrrr test_emptysz#HashIndexCompactTestCase.test_emptycCst}t}d |td<d |td<d |td<|j|jdksHt|j||tddksft|tddkszt|tddkstdS)Nr rrr@rA,rerfrC)r rr)rr@r@)rArrrgrh)r rr)rr@r@)rArr)rrrrPrrH)r*ZmasterrIrrr test_merges    z#HashIndexCompactTestCase.test_mergeN)rRrSrTr^rrrrrrrrrrrrrrrrrxsrc@seZdZddZdS)NSIndexTestCasec Csht}|jttjddf|td<WdQRXtd|ksBttjdf|td<td|ksdtdS)Nr rr)rrrrmr)r*r,rrrtest_nsindex_segment_limits   z*NSIndexTestCase.test_nsindex_segment_limitN)rRrSrTrrrrrrsrc@seZdZddZdS)AllIndexTestCasecCs tjdksttjdkstdS)Ng?)rr[rr)r*rrrtest_max_load_factorsz%AllIndexTestCase.test_max_load_factorN)rRrSrTrrrrrr src@seZdZddZdS)IndexCorruptionTestCasecs(ddlmfddtx"tdD]}d|fd|<q(WjdksRtxtd D]}d|=q\Wx"td D]}d |fd |<qzWfd d td dDdd td dDkstfdd td Ddd td Dkstfdd td Ddd td Dks$tdS)Nr)packcsd||dddS)Nz.HHii(reiiJiXcsg|]}jd|qS)r)r])rWr)rr,rrrY5sz9IndexCorruptionTestCase.test_bug_4829..cSsg|] }d|fqS)rr)rWrrrrrY5scsg|]}d|kqS)rr)rWr)rr,rrrY7scSsg|]}dqS)Fr)rWrrrrrY7scsg|]}jd|qS)iX)r])rWr)rr,rrrY:scSsg|] }d|fqS)iXr)rWrrrrrY:sii*)structrrrrPr)r*rr)rr,rr test_bug_4829s  2.z%IndexCorruptionTestCase.test_bug_4829N)rRrSrTrrrrrrsr)rrrr3r"rrrrrZcrypto.file_integrityrrr rrrrUrdrlrrrrrrrrrrs.  f/